LLVM 19.0.0git
IRTranslator.cpp
Go to the documentation of this file.
1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallSet.h"
21#include "llvm/Analysis/Loads.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/CFG.h"
53#include "llvm/IR/Constant.h"
54#include "llvm/IR/Constants.h"
55#include "llvm/IR/DataLayout.h"
58#include "llvm/IR/Function.h"
60#include "llvm/IR/InlineAsm.h"
61#include "llvm/IR/InstrTypes.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/IntrinsicsAMDGPU.h"
66#include "llvm/IR/LLVMContext.h"
67#include "llvm/IR/Metadata.h"
69#include "llvm/IR/Statepoint.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/User.h"
72#include "llvm/IR/Value.h"
74#include "llvm/MC/MCContext.h"
75#include "llvm/Pass.h"
78#include "llvm/Support/Debug.h"
86#include <algorithm>
87#include <cassert>
88#include <cstdint>
89#include <iterator>
90#include <optional>
91#include <string>
92#include <utility>
93#include <vector>
94
95#define DEBUG_TYPE "irtranslator"
96
97using namespace llvm;
98
99static cl::opt<bool>
100 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
101 cl::desc("Should enable CSE in irtranslator"),
102 cl::Optional, cl::init(false));
103char IRTranslator::ID = 0;
104
105INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
106 false, false)
114
119 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
120
121 // Print the function name explicitly if we don't have a debug location (which
122 // makes the diagnostic less useful) or if we're going to emit a raw error.
123 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
124 R << (" (in function: " + MF.getName() + ")").str();
125
126 if (TPC.isGlobalISelAbortEnabled())
127 report_fatal_error(Twine(R.getMsg()));
128 else
129 ORE.emit(R);
130}
131
133 : MachineFunctionPass(ID), OptLevel(optlevel) {}
134
135#ifndef NDEBUG
136namespace {
137/// Verify that every instruction created has the same DILocation as the
138/// instruction being translated.
139class DILocationVerifier : public GISelChangeObserver {
140 const Instruction *CurrInst = nullptr;
141
142public:
143 DILocationVerifier() = default;
144 ~DILocationVerifier() = default;
145
146 const Instruction *getCurrentInst() const { return CurrInst; }
147 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
148
149 void erasingInstr(MachineInstr &MI) override {}
150 void changingInstr(MachineInstr &MI) override {}
151 void changedInstr(MachineInstr &MI) override {}
152
153 void createdInstr(MachineInstr &MI) override {
154 assert(getCurrentInst() && "Inserted instruction without a current MI");
155
156 // Only print the check message if we're actually checking it.
157#ifndef NDEBUG
158 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
159 << " was copied to " << MI);
160#endif
161 // We allow insts in the entry block to have no debug loc because
162 // they could have originated from constants, and we don't want a jumpy
163 // debug experience.
164 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
165 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
166 (MI.isDebugInstr())) &&
167 "Line info was not transferred to all instructions");
168 }
169};
170} // namespace
171#endif // ifndef NDEBUG
172
173
179 if (OptLevel != CodeGenOptLevel::None) {
182 }
187}
188
190IRTranslator::allocateVRegs(const Value &Val) {
191 auto VRegsIt = VMap.findVRegs(Val);
192 if (VRegsIt != VMap.vregs_end())
193 return *VRegsIt->second;
194 auto *Regs = VMap.getVRegs(Val);
195 auto *Offsets = VMap.getOffsets(Val);
196 SmallVector<LLT, 4> SplitTys;
197 computeValueLLTs(*DL, *Val.getType(), SplitTys,
198 Offsets->empty() ? Offsets : nullptr);
199 for (unsigned i = 0; i < SplitTys.size(); ++i)
200 Regs->push_back(0);
201 return *Regs;
202}
203
204ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
205 auto VRegsIt = VMap.findVRegs(Val);
206 if (VRegsIt != VMap.vregs_end())
207 return *VRegsIt->second;
208
209 if (Val.getType()->isVoidTy())
210 return *VMap.getVRegs(Val);
211
212 // Create entry for this type.
213 auto *VRegs = VMap.getVRegs(Val);
214 auto *Offsets = VMap.getOffsets(Val);
215
216 assert(Val.getType()->isSized() &&
217 "Don't know how to create an empty vreg");
218
219 SmallVector<LLT, 4> SplitTys;
220 computeValueLLTs(*DL, *Val.getType(), SplitTys,
221 Offsets->empty() ? Offsets : nullptr);
222
223 if (!isa<Constant>(Val)) {
224 for (auto Ty : SplitTys)
225 VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
226 return *VRegs;
227 }
228
229 if (Val.getType()->isAggregateType()) {
230 // UndefValue, ConstantAggregateZero
231 auto &C = cast<Constant>(Val);
232 unsigned Idx = 0;
233 while (auto Elt = C.getAggregateElement(Idx++)) {
234 auto EltRegs = getOrCreateVRegs(*Elt);
235 llvm::copy(EltRegs, std::back_inserter(*VRegs));
236 }
237 } else {
238 assert(SplitTys.size() == 1 && "unexpectedly split LLT");
239 VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
240 bool Success = translate(cast<Constant>(Val), VRegs->front());
241 if (!Success) {
242 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
244 &MF->getFunction().getEntryBlock());
245 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
246 reportTranslationError(*MF, *TPC, *ORE, R);
247 return *VRegs;
248 }
249 }
250
251 return *VRegs;
252}
253
254int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
255 auto MapEntry = FrameIndices.find(&AI);
256 if (MapEntry != FrameIndices.end())
257 return MapEntry->second;
258
259 uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
260 uint64_t Size =
261 ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
262
263 // Always allocate at least one byte.
264 Size = std::max<uint64_t>(Size, 1u);
265
266 int &FI = FrameIndices[&AI];
267 FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
268 return FI;
269}
270
271Align IRTranslator::getMemOpAlign(const Instruction &I) {
272 if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
273 return SI->getAlign();
274 if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
275 return LI->getAlign();
276 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
277 return AI->getAlign();
278 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
279 return AI->getAlign();
280
281 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
282 R << "unable to translate memop: " << ore::NV("Opcode", &I);
283 reportTranslationError(*MF, *TPC, *ORE, R);
284 return Align(1);
285}
286
287MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
288 MachineBasicBlock *&MBB = BBToMBB[&BB];
289 assert(MBB && "BasicBlock was not encountered before");
290 return *MBB;
291}
292
293void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
294 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
295 MachinePreds[Edge].push_back(NewPred);
296}
297
298bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
299 MachineIRBuilder &MIRBuilder) {
300 // Get or create a virtual register for each value.
301 // Unless the value is a Constant => loadimm cst?
302 // or inline constant each time?
303 // Creation of a virtual register needs to have a size.
304 Register Op0 = getOrCreateVReg(*U.getOperand(0));
305 Register Op1 = getOrCreateVReg(*U.getOperand(1));
306 Register Res = getOrCreateVReg(U);
307 uint32_t Flags = 0;
308 if (isa<Instruction>(U)) {
309 const Instruction &I = cast<Instruction>(U);
311 }
312
313 MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
314 return true;
315}
316
317bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
318 MachineIRBuilder &MIRBuilder) {
319 Register Op0 = getOrCreateVReg(*U.getOperand(0));
320 Register Res = getOrCreateVReg(U);
321 uint32_t Flags = 0;
322 if (isa<Instruction>(U)) {
323 const Instruction &I = cast<Instruction>(U);
325 }
326 MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);
327 return true;
328}
329
330bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
331 return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
332}
333
334bool IRTranslator::translateCompare(const User &U,
335 MachineIRBuilder &MIRBuilder) {
336 auto *CI = dyn_cast<CmpInst>(&U);
337 Register Op0 = getOrCreateVReg(*U.getOperand(0));
338 Register Op1 = getOrCreateVReg(*U.getOperand(1));
339 Register Res = getOrCreateVReg(U);
340 CmpInst::Predicate Pred =
341 CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
342 cast<ConstantExpr>(U).getPredicate());
343 if (CmpInst::isIntPredicate(Pred))
344 MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
345 else if (Pred == CmpInst::FCMP_FALSE)
346 MIRBuilder.buildCopy(
347 Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
348 else if (Pred == CmpInst::FCMP_TRUE)
349 MIRBuilder.buildCopy(
350 Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
351 else {
352 uint32_t Flags = 0;
353 if (CI)
355 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
356 }
357
358 return true;
359}
360
361bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
362 const ReturnInst &RI = cast<ReturnInst>(U);
363 const Value *Ret = RI.getReturnValue();
364 if (Ret && DL->getTypeStoreSize(Ret->getType()).isZero())
365 Ret = nullptr;
366
367 ArrayRef<Register> VRegs;
368 if (Ret)
369 VRegs = getOrCreateVRegs(*Ret);
370
371 Register SwiftErrorVReg = 0;
372 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
373 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
374 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
375 }
376
377 // The target may mess up with the insertion point, but
378 // this is not important as a return is the last instruction
379 // of the block anyway.
380 return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
381}
382
383void IRTranslator::emitBranchForMergedCondition(
385 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
386 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
387 // If the leaf of the tree is a comparison, merge the condition into
388 // the caseblock.
389 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
390 CmpInst::Predicate Condition;
391 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
392 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
393 } else {
394 const FCmpInst *FC = cast<FCmpInst>(Cond);
395 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
396 }
397
398 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),
399 BOp->getOperand(1), nullptr, TBB, FBB, CurBB,
400 CurBuilder->getDebugLoc(), TProb, FProb);
401 SL->SwitchCases.push_back(CB);
402 return;
403 }
404
405 // Create a CaseBlock record representing this branch.
408 Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),
409 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
410 SL->SwitchCases.push_back(CB);
411}
412
413static bool isValInBlock(const Value *V, const BasicBlock *BB) {
414 if (const Instruction *I = dyn_cast<Instruction>(V))
415 return I->getParent() == BB;
416 return true;
417}
418
419void IRTranslator::findMergedConditions(
421 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
423 BranchProbability FProb, bool InvertCond) {
424 using namespace PatternMatch;
425 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
426 "Expected Opc to be AND/OR");
427 // Skip over not part of the tree and remember to invert op and operands at
428 // next level.
429 Value *NotCond;
430 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
431 isValInBlock(NotCond, CurBB->getBasicBlock())) {
432 findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
433 !InvertCond);
434 return;
435 }
436
437 const Instruction *BOp = dyn_cast<Instruction>(Cond);
438 const Value *BOpOp0, *BOpOp1;
439 // Compute the effective opcode for Cond, taking into account whether it needs
440 // to be inverted, e.g.
441 // and (not (or A, B)), C
442 // gets lowered as
443 // and (and (not A, not B), C)
445 if (BOp) {
446 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
447 ? Instruction::And
448 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
449 ? Instruction::Or
451 if (InvertCond) {
452 if (BOpc == Instruction::And)
453 BOpc = Instruction::Or;
454 else if (BOpc == Instruction::Or)
455 BOpc = Instruction::And;
456 }
457 }
458
459 // If this node is not part of the or/and tree, emit it as a branch.
460 // Note that all nodes in the tree should have same opcode.
461 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
462 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
463 !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||
464 !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {
465 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
466 InvertCond);
467 return;
468 }
469
470 // Create TmpBB after CurBB.
471 MachineFunction::iterator BBI(CurBB);
472 MachineBasicBlock *TmpBB =
474 CurBB->getParent()->insert(++BBI, TmpBB);
475
476 if (Opc == Instruction::Or) {
477 // Codegen X | Y as:
478 // BB1:
479 // jmp_if_X TBB
480 // jmp TmpBB
481 // TmpBB:
482 // jmp_if_Y TBB
483 // jmp FBB
484 //
485
486 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
487 // The requirement is that
488 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
489 // = TrueProb for original BB.
490 // Assuming the original probabilities are A and B, one choice is to set
491 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
492 // A/(1+B) and 2B/(1+B). This choice assumes that
493 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
494 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
495 // TmpBB, but the math is more complicated.
496
497 auto NewTrueProb = TProb / 2;
498 auto NewFalseProb = TProb / 2 + FProb;
499 // Emit the LHS condition.
500 findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
501 NewFalseProb, InvertCond);
502
503 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
504 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
505 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
506 // Emit the RHS condition into TmpBB.
507 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
508 Probs[1], InvertCond);
509 } else {
510 assert(Opc == Instruction::And && "Unknown merge op!");
511 // Codegen X & Y as:
512 // BB1:
513 // jmp_if_X TmpBB
514 // jmp FBB
515 // TmpBB:
516 // jmp_if_Y TBB
517 // jmp FBB
518 //
519 // This requires creation of TmpBB after CurBB.
520
521 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
522 // The requirement is that
523 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
524 // = FalseProb for original BB.
525 // Assuming the original probabilities are A and B, one choice is to set
526 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
527 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
528 // TrueProb for BB1 * FalseProb for TmpBB.
529
530 auto NewTrueProb = TProb + FProb / 2;
531 auto NewFalseProb = FProb / 2;
532 // Emit the LHS condition.
533 findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
534 NewFalseProb, InvertCond);
535
536 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
537 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
538 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
539 // Emit the RHS condition into TmpBB.
540 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
541 Probs[1], InvertCond);
542 }
543}
544
545bool IRTranslator::shouldEmitAsBranches(
546 const std::vector<SwitchCG::CaseBlock> &Cases) {
547 // For multiple cases, it's better to emit as branches.
548 if (Cases.size() != 2)
549 return true;
550
551 // If this is two comparisons of the same values or'd or and'd together, they
552 // will get folded into a single comparison, so don't emit two blocks.
553 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
554 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
555 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
556 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
557 return false;
558 }
559
560 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
561 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
562 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
563 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
564 isa<Constant>(Cases[0].CmpRHS) &&
565 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
566 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
567 Cases[0].TrueBB == Cases[1].ThisBB)
568 return false;
569 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
570 Cases[0].FalseBB == Cases[1].ThisBB)
571 return false;
572 }
573
574 return true;
575}
576
577bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
578 const BranchInst &BrInst = cast<BranchInst>(U);
579 auto &CurMBB = MIRBuilder.getMBB();
580 auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));
581
582 if (BrInst.isUnconditional()) {
583 // If the unconditional target is the layout successor, fallthrough.
584 if (OptLevel == CodeGenOptLevel::None ||
585 !CurMBB.isLayoutSuccessor(Succ0MBB))
586 MIRBuilder.buildBr(*Succ0MBB);
587
588 // Link successors.
589 for (const BasicBlock *Succ : successors(&BrInst))
590 CurMBB.addSuccessor(&getMBB(*Succ));
591 return true;
592 }
593
594 // If this condition is one of the special cases we handle, do special stuff
595 // now.
596 const Value *CondVal = BrInst.getCondition();
597 MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));
598
599 const auto &TLI = *MF->getSubtarget().getTargetLowering();
600
601 // If this is a series of conditions that are or'd or and'd together, emit
602 // this as a sequence of branches instead of setcc's with and/or operations.
603 // As long as jumps are not expensive (exceptions for multi-use logic ops,
604 // unpredictable branches, and vector extracts because those jumps are likely
605 // expensive for any target), this should improve performance.
606 // For example, instead of something like:
607 // cmp A, B
608 // C = seteq
609 // cmp D, E
610 // F = setle
611 // or C, F
612 // jnz foo
613 // Emit:
614 // cmp A, B
615 // je foo
616 // cmp D, E
617 // jle foo
618 using namespace PatternMatch;
619 const Instruction *CondI = dyn_cast<Instruction>(CondVal);
620 if (!TLI.isJumpExpensive() && CondI && CondI->hasOneUse() &&
621 !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {
623 Value *Vec;
624 const Value *BOp0, *BOp1;
625 if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
626 Opcode = Instruction::And;
627 else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
628 Opcode = Instruction::Or;
629
630 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
631 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
632 findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
633 getEdgeProbability(&CurMBB, Succ0MBB),
634 getEdgeProbability(&CurMBB, Succ1MBB),
635 /*InvertCond=*/false);
636 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
637
638 // Allow some cases to be rejected.
639 if (shouldEmitAsBranches(SL->SwitchCases)) {
640 // Emit the branch for this block.
641 emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
642 SL->SwitchCases.erase(SL->SwitchCases.begin());
643 return true;
644 }
645
646 // Okay, we decided not to do this, remove any inserted MBB's and clear
647 // SwitchCases.
648 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
649 MF->erase(SL->SwitchCases[I].ThisBB);
650
651 SL->SwitchCases.clear();
652 }
653 }
654
655 // Create a CaseBlock record representing this branch.
656 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
658 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
659 CurBuilder->getDebugLoc());
660
661 // Use emitSwitchCase to actually insert the fast branch sequence for this
662 // cond branch.
663 emitSwitchCase(CB, &CurMBB, *CurBuilder);
664 return true;
665}
666
667void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
669 BranchProbability Prob) {
670 if (!FuncInfo.BPI) {
671 Src->addSuccessorWithoutProb(Dst);
672 return;
673 }
674 if (Prob.isUnknown())
675 Prob = getEdgeProbability(Src, Dst);
676 Src->addSuccessor(Dst, Prob);
677}
678
680IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
681 const MachineBasicBlock *Dst) const {
682 const BasicBlock *SrcBB = Src->getBasicBlock();
683 const BasicBlock *DstBB = Dst->getBasicBlock();
684 if (!FuncInfo.BPI) {
685 // If BPI is not available, set the default probability as 1 / N, where N is
686 // the number of successors.
687 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
688 return BranchProbability(1, SuccSize);
689 }
690 return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
691}
692
693bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
694 using namespace SwitchCG;
695 // Extract cases from the switch.
696 const SwitchInst &SI = cast<SwitchInst>(U);
697 BranchProbabilityInfo *BPI = FuncInfo.BPI;
698 CaseClusterVector Clusters;
699 Clusters.reserve(SI.getNumCases());
700 for (const auto &I : SI.cases()) {
701 MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
702 assert(Succ && "Could not find successor mbb in mapping");
703 const ConstantInt *CaseVal = I.getCaseValue();
704 BranchProbability Prob =
705 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
706 : BranchProbability(1, SI.getNumCases() + 1);
707 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
708 }
709
710 MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
711
712 // Cluster adjacent cases with the same destination. We do this at all
713 // optimization levels because it's cheap to do and will make codegen faster
714 // if there are many clusters.
715 sortAndRangeify(Clusters);
716
717 MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
718
719 // If there is only the default destination, jump there directly.
720 if (Clusters.empty()) {
721 SwitchMBB->addSuccessor(DefaultMBB);
722 if (DefaultMBB != SwitchMBB->getNextNode())
723 MIB.buildBr(*DefaultMBB);
724 return true;
725 }
726
727 SL->findJumpTables(Clusters, &SI, std::nullopt, DefaultMBB, nullptr, nullptr);
728 SL->findBitTestClusters(Clusters, &SI);
729
730 LLVM_DEBUG({
731 dbgs() << "Case clusters: ";
732 for (const CaseCluster &C : Clusters) {
733 if (C.Kind == CC_JumpTable)
734 dbgs() << "JT:";
735 if (C.Kind == CC_BitTests)
736 dbgs() << "BT:";
737
738 C.Low->getValue().print(dbgs(), true);
739 if (C.Low != C.High) {
740 dbgs() << '-';
741 C.High->getValue().print(dbgs(), true);
742 }
743 dbgs() << ' ';
744 }
745 dbgs() << '\n';
746 });
747
748 assert(!Clusters.empty());
749 SwitchWorkList WorkList;
750 CaseClusterIt First = Clusters.begin();
751 CaseClusterIt Last = Clusters.end() - 1;
752 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
753 WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
754
755 while (!WorkList.empty()) {
756 SwitchWorkListItem W = WorkList.pop_back_val();
757
758 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
759 // For optimized builds, lower large range as a balanced binary tree.
760 if (NumClusters > 3 &&
762 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
763 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB, MIB);
764 continue;
765 }
766
767 if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
768 return false;
769 }
770 return true;
771}
772
773void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
775 Value *Cond, MachineBasicBlock *SwitchMBB,
776 MachineIRBuilder &MIB) {
777 using namespace SwitchCG;
778 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
779 "Clusters not sorted?");
780 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
781
782 auto [LastLeft, FirstRight, LeftProb, RightProb] =
783 SL->computeSplitWorkItemInfo(W);
784
785 // Use the first element on the right as pivot since we will make less-than
786 // comparisons against it.
787 CaseClusterIt PivotCluster = FirstRight;
788 assert(PivotCluster > W.FirstCluster);
789 assert(PivotCluster <= W.LastCluster);
790
791 CaseClusterIt FirstLeft = W.FirstCluster;
792 CaseClusterIt LastRight = W.LastCluster;
793
794 const ConstantInt *Pivot = PivotCluster->Low;
795
796 // New blocks will be inserted immediately after the current one.
798 ++BBI;
799
800 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
801 // we can branch to its destination directly if it's squeezed exactly in
802 // between the known lower bound and Pivot - 1.
803 MachineBasicBlock *LeftMBB;
804 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
805 FirstLeft->Low == W.GE &&
806 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
807 LeftMBB = FirstLeft->MBB;
808 } else {
809 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
810 FuncInfo.MF->insert(BBI, LeftMBB);
811 WorkList.push_back(
812 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
813 }
814
815 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
816 // single cluster, RHS.Low == Pivot, and we can branch to its destination
817 // directly if RHS.High equals the current upper bound.
818 MachineBasicBlock *RightMBB;
819 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
820 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
821 RightMBB = FirstRight->MBB;
822 } else {
823 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
824 FuncInfo.MF->insert(BBI, RightMBB);
825 WorkList.push_back(
826 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
827 }
828
829 // Create the CaseBlock record that will be used to lower the branch.
830 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
831 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
832 RightProb);
833
834 if (W.MBB == SwitchMBB)
835 emitSwitchCase(CB, SwitchMBB, MIB);
836 else
837 SL->SwitchCases.push_back(CB);
838}
839
840void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
842 // Emit the code for the jump table
843 assert(JT.Reg != -1U && "Should lower JT Header first!");
845 MIB.setMBB(*MBB);
846 MIB.setDebugLoc(CurBuilder->getDebugLoc());
847
849 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
850
851 auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
852 MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
853}
854
855bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
857 MachineBasicBlock *HeaderBB) {
858 MachineIRBuilder MIB(*HeaderBB->getParent());
859 MIB.setMBB(*HeaderBB);
860 MIB.setDebugLoc(CurBuilder->getDebugLoc());
861
862 const Value &SValue = *JTH.SValue;
863 // Subtract the lowest switch case value from the value being switched on.
864 const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
865 Register SwitchOpReg = getOrCreateVReg(SValue);
866 auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
867 auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
868
869 // This value may be smaller or larger than the target's pointer type, and
870 // therefore require extension or truncating.
871 auto *PtrIRTy = PointerType::getUnqual(SValue.getContext());
872 const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
873 Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
874
875 JT.Reg = Sub.getReg(0);
876
877 if (JTH.FallthroughUnreachable) {
878 if (JT.MBB != HeaderBB->getNextNode())
879 MIB.buildBr(*JT.MBB);
880 return true;
881 }
882
883 // Emit the range check for the jump table, and branch to the default block
884 // for the switch statement if the value being switched on exceeds the
885 // largest case in the switch.
886 auto Cst = getOrCreateVReg(
887 *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
888 Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
889 auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
890
891 auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
892
893 // Avoid emitting unnecessary branches to the next block.
894 if (JT.MBB != HeaderBB->getNextNode())
895 BrCond = MIB.buildBr(*JT.MBB);
896 return true;
897}
898
899void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
900 MachineBasicBlock *SwitchBB,
901 MachineIRBuilder &MIB) {
902 Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
904 DebugLoc OldDbgLoc = MIB.getDebugLoc();
905 MIB.setDebugLoc(CB.DbgLoc);
906 MIB.setMBB(*CB.ThisBB);
907
908 if (CB.PredInfo.NoCmp) {
909 // Branch or fall through to TrueBB.
910 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
911 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
912 CB.ThisBB);
914 if (CB.TrueBB != CB.ThisBB->getNextNode())
915 MIB.buildBr(*CB.TrueBB);
916 MIB.setDebugLoc(OldDbgLoc);
917 return;
918 }
919
920 const LLT i1Ty = LLT::scalar(1);
921 // Build the compare.
922 if (!CB.CmpMHS) {
923 const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);
924 // For conditional branch lowering, we might try to do something silly like
925 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
926 // just re-use the existing condition vreg.
927 if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
929 Cond = CondLHS;
930 } else {
931 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
933 Cond =
934 MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
935 else
936 Cond =
937 MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
938 }
939 } else {
941 "Can only handle SLE ranges");
942
943 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
944 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
945
946 Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
947 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
948 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
949 Cond =
950 MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
951 } else {
952 const LLT CmpTy = MRI->getType(CmpOpReg);
953 auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
954 auto Diff = MIB.buildConstant(CmpTy, High - Low);
955 Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
956 }
957 }
958
959 // Update successor info
960 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
961
962 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
963 CB.ThisBB);
964
965 // TrueBB and FalseBB are always different unless the incoming IR is
966 // degenerate. This only happens when running llc on weird IR.
967 if (CB.TrueBB != CB.FalseBB)
968 addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
970
971 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
972 CB.ThisBB);
973
974 MIB.buildBrCond(Cond, *CB.TrueBB);
975 MIB.buildBr(*CB.FalseBB);
976 MIB.setDebugLoc(OldDbgLoc);
977}
978
979bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
980 MachineBasicBlock *SwitchMBB,
981 MachineBasicBlock *CurMBB,
982 MachineBasicBlock *DefaultMBB,
983 MachineIRBuilder &MIB,
985 BranchProbability UnhandledProbs,
987 MachineBasicBlock *Fallthrough,
988 bool FallthroughUnreachable) {
989 using namespace SwitchCG;
990 MachineFunction *CurMF = SwitchMBB->getParent();
991 // FIXME: Optimize away range check based on pivot comparisons.
992 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
993 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
994 BranchProbability DefaultProb = W.DefaultProb;
995
996 // The jump block hasn't been inserted yet; insert it here.
997 MachineBasicBlock *JumpMBB = JT->MBB;
998 CurMF->insert(BBI, JumpMBB);
999
1000 // Since the jump table block is separate from the switch block, we need
1001 // to keep track of it as a machine predecessor to the default block,
1002 // otherwise we lose the phi edges.
1003 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1004 CurMBB);
1005 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1006 JumpMBB);
1007
1008 auto JumpProb = I->Prob;
1009 auto FallthroughProb = UnhandledProbs;
1010
1011 // If the default statement is a target of the jump table, we evenly
1012 // distribute the default probability to successors of CurMBB. Also
1013 // update the probability on the edge from JumpMBB to Fallthrough.
1014 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1015 SE = JumpMBB->succ_end();
1016 SI != SE; ++SI) {
1017 if (*SI == DefaultMBB) {
1018 JumpProb += DefaultProb / 2;
1019 FallthroughProb -= DefaultProb / 2;
1020 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
1021 JumpMBB->normalizeSuccProbs();
1022 } else {
1023 // Also record edges from the jump table block to it's successors.
1024 addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1025 JumpMBB);
1026 }
1027 }
1028
1029 if (FallthroughUnreachable)
1030 JTH->FallthroughUnreachable = true;
1031
1032 if (!JTH->FallthroughUnreachable)
1033 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
1034 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
1035 CurMBB->normalizeSuccProbs();
1036
1037 // The jump table header will be inserted in our current block, do the
1038 // range check, and fall through to our fallthrough block.
1039 JTH->HeaderBB = CurMBB;
1040 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1041
1042 // If we're in the right place, emit the jump table header right now.
1043 if (CurMBB == SwitchMBB) {
1044 if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
1045 return false;
1046 JTH->Emitted = true;
1047 }
1048 return true;
1049}
1050bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
1051 Value *Cond,
1052 MachineBasicBlock *Fallthrough,
1053 bool FallthroughUnreachable,
1054 BranchProbability UnhandledProbs,
1055 MachineBasicBlock *CurMBB,
1056 MachineIRBuilder &MIB,
1057 MachineBasicBlock *SwitchMBB) {
1058 using namespace SwitchCG;
1059 const Value *RHS, *LHS, *MHS;
1060 CmpInst::Predicate Pred;
1061 if (I->Low == I->High) {
1062 // Check Cond == I->Low.
1063 Pred = CmpInst::ICMP_EQ;
1064 LHS = Cond;
1065 RHS = I->Low;
1066 MHS = nullptr;
1067 } else {
1068 // Check I->Low <= Cond <= I->High.
1069 Pred = CmpInst::ICMP_SLE;
1070 LHS = I->Low;
1071 MHS = Cond;
1072 RHS = I->High;
1073 }
1074
1075 // If Fallthrough is unreachable, fold away the comparison.
1076 // The false probability is the sum of all unhandled cases.
1077 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1078 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1079
1080 emitSwitchCase(CB, SwitchMBB, MIB);
1081 return true;
1082}
1083
1084void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1085 MachineBasicBlock *SwitchBB) {
1086 MachineIRBuilder &MIB = *CurBuilder;
1087 MIB.setMBB(*SwitchBB);
1088
1089 // Subtract the minimum value.
1090 Register SwitchOpReg = getOrCreateVReg(*B.SValue);
1091
1092 LLT SwitchOpTy = MRI->getType(SwitchOpReg);
1093 Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);
1094 auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1095
1097 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1098
1099 LLT MaskTy = SwitchOpTy;
1100 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1101 !llvm::has_single_bit<uint32_t>(MaskTy.getSizeInBits()))
1102 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1103 else {
1104 // Ensure that the type will fit the mask value.
1105 for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1106 if (!isUIntN(SwitchOpTy.getSizeInBits(), B.Cases[I].Mask)) {
1107 // Switch table case range are encoded into series of masks.
1108 // Just use pointer type, it's guaranteed to fit.
1109 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1110 break;
1111 }
1112 }
1113 }
1114 Register SubReg = RangeSub.getReg(0);
1115 if (SwitchOpTy != MaskTy)
1116 SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);
1117
1118 B.RegVT = getMVTForLLT(MaskTy);
1119 B.Reg = SubReg;
1120
1121 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1122
1123 if (!B.FallthroughUnreachable)
1124 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
1125 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
1126
1127 SwitchBB->normalizeSuccProbs();
1128
1129 if (!B.FallthroughUnreachable) {
1130 // Conditional branch to the default block.
1131 auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);
1132 auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),
1133 RangeSub, RangeCst);
1134 MIB.buildBrCond(RangeCmp, *B.Default);
1135 }
1136
1137 // Avoid emitting unnecessary branches to the next block.
1138 if (MBB != SwitchBB->getNextNode())
1139 MIB.buildBr(*MBB);
1140}
1141
1142void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1143 MachineBasicBlock *NextMBB,
1144 BranchProbability BranchProbToNext,
1146 MachineBasicBlock *SwitchBB) {
1147 MachineIRBuilder &MIB = *CurBuilder;
1148 MIB.setMBB(*SwitchBB);
1149
1150 LLT SwitchTy = getLLTForMVT(BB.RegVT);
1151 Register Cmp;
1152 unsigned PopCount = llvm::popcount(B.Mask);
1153 if (PopCount == 1) {
1154 // Testing for a single bit; just compare the shift count with what it
1155 // would need to be to shift a 1 bit in that position.
1156 auto MaskTrailingZeros =
1157 MIB.buildConstant(SwitchTy, llvm::countr_zero(B.Mask));
1158 Cmp =
1159 MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
1160 .getReg(0);
1161 } else if (PopCount == BB.Range) {
1162 // There is only one zero bit in the range, test for it directly.
1163 auto MaskTrailingOnes =
1164 MIB.buildConstant(SwitchTy, llvm::countr_one(B.Mask));
1165 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
1166 .getReg(0);
1167 } else {
1168 // Make desired shift.
1169 auto CstOne = MIB.buildConstant(SwitchTy, 1);
1170 auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);
1171
1172 // Emit bit tests and jumps.
1173 auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);
1174 auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);
1175 auto CstZero = MIB.buildConstant(SwitchTy, 0);
1176 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)
1177 .getReg(0);
1178 }
1179
1180 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1181 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
1182 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1183 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1184 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1185 // one as they are relative probabilities (and thus work more like weights),
1186 // and hence we need to normalize them to let the sum of them become one.
1187 SwitchBB->normalizeSuccProbs();
1188
1189 // Record the fact that the IR edge from the header to the bit test target
1190 // will go through our new block. Neeeded for PHIs to have nodes added.
1191 addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1192 SwitchBB);
1193
1194 MIB.buildBrCond(Cmp, *B.TargetBB);
1195
1196 // Avoid emitting unnecessary branches to the next block.
1197 if (NextMBB != SwitchBB->getNextNode())
1198 MIB.buildBr(*NextMBB);
1199}
1200
1201bool IRTranslator::lowerBitTestWorkItem(
1203 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1205 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1207 bool FallthroughUnreachable) {
1208 using namespace SwitchCG;
1209 MachineFunction *CurMF = SwitchMBB->getParent();
1210 // FIXME: Optimize away range check based on pivot comparisons.
1211 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1212 // The bit test blocks haven't been inserted yet; insert them here.
1213 for (BitTestCase &BTC : BTB->Cases)
1214 CurMF->insert(BBI, BTC.ThisBB);
1215
1216 // Fill in fields of the BitTestBlock.
1217 BTB->Parent = CurMBB;
1218 BTB->Default = Fallthrough;
1219
1220 BTB->DefaultProb = UnhandledProbs;
1221 // If the cases in bit test don't form a contiguous range, we evenly
1222 // distribute the probability on the edge to Fallthrough to two
1223 // successors of CurMBB.
1224 if (!BTB->ContiguousRange) {
1225 BTB->Prob += DefaultProb / 2;
1226 BTB->DefaultProb -= DefaultProb / 2;
1227 }
1228
1229 if (FallthroughUnreachable)
1230 BTB->FallthroughUnreachable = true;
1231
1232 // If we're in the right place, emit the bit test header right now.
1233 if (CurMBB == SwitchMBB) {
1234 emitBitTestHeader(*BTB, SwitchMBB);
1235 BTB->Emitted = true;
1236 }
1237 return true;
1238}
1239
1240bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1241 Value *Cond,
1242 MachineBasicBlock *SwitchMBB,
1243 MachineBasicBlock *DefaultMBB,
1244 MachineIRBuilder &MIB) {
1245 using namespace SwitchCG;
1246 MachineFunction *CurMF = FuncInfo.MF;
1247 MachineBasicBlock *NextMBB = nullptr;
1249 if (++BBI != FuncInfo.MF->end())
1250 NextMBB = &*BBI;
1251
1252 if (EnableOpts) {
1253 // Here, we order cases by probability so the most likely case will be
1254 // checked first. However, two clusters can have the same probability in
1255 // which case their relative ordering is non-deterministic. So we use Low
1256 // as a tie-breaker as clusters are guaranteed to never overlap.
1257 llvm::sort(W.FirstCluster, W.LastCluster + 1,
1258 [](const CaseCluster &a, const CaseCluster &b) {
1259 return a.Prob != b.Prob
1260 ? a.Prob > b.Prob
1261 : a.Low->getValue().slt(b.Low->getValue());
1262 });
1263
1264 // Rearrange the case blocks so that the last one falls through if possible
1265 // without changing the order of probabilities.
1266 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1267 --I;
1268 if (I->Prob > W.LastCluster->Prob)
1269 break;
1270 if (I->Kind == CC_Range && I->MBB == NextMBB) {
1271 std::swap(*I, *W.LastCluster);
1272 break;
1273 }
1274 }
1275 }
1276
1277 // Compute total probability.
1278 BranchProbability DefaultProb = W.DefaultProb;
1279 BranchProbability UnhandledProbs = DefaultProb;
1280 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1281 UnhandledProbs += I->Prob;
1282
1283 MachineBasicBlock *CurMBB = W.MBB;
1284 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1285 bool FallthroughUnreachable = false;
1286 MachineBasicBlock *Fallthrough;
1287 if (I == W.LastCluster) {
1288 // For the last cluster, fall through to the default destination.
1289 Fallthrough = DefaultMBB;
1290 FallthroughUnreachable = isa<UnreachableInst>(
1291 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1292 } else {
1293 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
1294 CurMF->insert(BBI, Fallthrough);
1295 }
1296 UnhandledProbs -= I->Prob;
1297
1298 switch (I->Kind) {
1299 case CC_BitTests: {
1300 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1301 DefaultProb, UnhandledProbs, I, Fallthrough,
1302 FallthroughUnreachable)) {
1303 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1304 return false;
1305 }
1306 break;
1307 }
1308
1309 case CC_JumpTable: {
1310 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1311 UnhandledProbs, I, Fallthrough,
1312 FallthroughUnreachable)) {
1313 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1314 return false;
1315 }
1316 break;
1317 }
1318 case CC_Range: {
1319 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1320 FallthroughUnreachable, UnhandledProbs,
1321 CurMBB, MIB, SwitchMBB)) {
1322 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1323 return false;
1324 }
1325 break;
1326 }
1327 }
1328 CurMBB = Fallthrough;
1329 }
1330
1331 return true;
1332}
1333
1334bool IRTranslator::translateIndirectBr(const User &U,
1335 MachineIRBuilder &MIRBuilder) {
1336 const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
1337
1338 const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
1339 MIRBuilder.buildBrIndirect(Tgt);
1340
1341 // Link successors.
1343 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1344 for (const BasicBlock *Succ : successors(&BrInst)) {
1345 // It's legal for indirectbr instructions to have duplicate blocks in the
1346 // destination list. We don't allow this in MIR. Skip anything that's
1347 // already a successor.
1348 if (!AddedSuccessors.insert(Succ).second)
1349 continue;
1350 CurBB.addSuccessor(&getMBB(*Succ));
1351 }
1352
1353 return true;
1354}
1355
1356static bool isSwiftError(const Value *V) {
1357 if (auto Arg = dyn_cast<Argument>(V))
1358 return Arg->hasSwiftErrorAttr();
1359 if (auto AI = dyn_cast<AllocaInst>(V))
1360 return AI->isSwiftError();
1361 return false;
1362}
1363
1364bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1365 const LoadInst &LI = cast<LoadInst>(U);
1366
1367 unsigned StoreSize = DL->getTypeStoreSize(LI.getType());
1368 if (StoreSize == 0)
1369 return true;
1370
1371 ArrayRef<Register> Regs = getOrCreateVRegs(LI);
1372 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
1373 Register Base = getOrCreateVReg(*LI.getPointerOperand());
1374 AAMDNodes AAInfo = LI.getAAMetadata();
1375
1376 const Value *Ptr = LI.getPointerOperand();
1377 Type *OffsetIRTy = DL->getIndexType(Ptr->getType());
1378 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1379
1380 if (CLI->supportSwiftError() && isSwiftError(Ptr)) {
1381 assert(Regs.size() == 1 && "swifterror should be single pointer");
1382 Register VReg =
1383 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1384 MIRBuilder.buildCopy(Regs[0], VReg);
1385 return true;
1386 }
1387
1388 auto &TLI = *MF->getSubtarget().getTargetLowering();
1390 TLI.getLoadMemOperandFlags(LI, *DL, AC, LibInfo);
1391 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1392 if (AA->pointsToConstantMemory(
1393 MemoryLocation(Ptr, LocationSize::precise(StoreSize), AAInfo))) {
1395 }
1396 }
1397
1398 const MDNode *Ranges =
1399 Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
1400 for (unsigned i = 0; i < Regs.size(); ++i) {
1401 Register Addr;
1402 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1403
1404 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
1405 Align BaseAlign = getMemOpAlign(LI);
1406 auto MMO = MF->getMachineMemOperand(
1407 Ptr, Flags, MRI->getType(Regs[i]),
1408 commonAlignment(BaseAlign, Offsets[i] / 8), AAInfo, Ranges,
1409 LI.getSyncScopeID(), LI.getOrdering());
1410 MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
1411 }
1412
1413 return true;
1414}
1415
1416bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1417 const StoreInst &SI = cast<StoreInst>(U);
1418 if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
1419 return true;
1420
1421 ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
1422 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
1423 Register Base = getOrCreateVReg(*SI.getPointerOperand());
1424
1425 Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());
1426 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1427
1428 if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
1429 assert(Vals.size() == 1 && "swifterror should be single pointer");
1430
1431 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1432 SI.getPointerOperand());
1433 MIRBuilder.buildCopy(VReg, Vals[0]);
1434 return true;
1435 }
1436
1437 auto &TLI = *MF->getSubtarget().getTargetLowering();
1438 MachineMemOperand::Flags Flags = TLI.getStoreMemOperandFlags(SI, *DL);
1439
1440 for (unsigned i = 0; i < Vals.size(); ++i) {
1441 Register Addr;
1442 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1443
1444 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1445 Align BaseAlign = getMemOpAlign(SI);
1446 auto MMO = MF->getMachineMemOperand(
1447 Ptr, Flags, MRI->getType(Vals[i]),
1448 commonAlignment(BaseAlign, Offsets[i] / 8), SI.getAAMetadata(), nullptr,
1449 SI.getSyncScopeID(), SI.getOrdering());
1450 MIRBuilder.buildStore(Vals[i], Addr, *MMO);
1451 }
1452 return true;
1453}
1454
1456 const Value *Src = U.getOperand(0);
1457 Type *Int32Ty = Type::getInt32Ty(U.getContext());
1458
1459 // getIndexedOffsetInType is designed for GEPs, so the first index is the
1460 // usual array element rather than looking into the actual aggregate.
1462 Indices.push_back(ConstantInt::get(Int32Ty, 0));
1463
1464 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
1465 for (auto Idx : EVI->indices())
1466 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1467 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1468 for (auto Idx : IVI->indices())
1469 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1470 } else {
1471 for (unsigned i = 1; i < U.getNumOperands(); ++i)
1472 Indices.push_back(U.getOperand(i));
1473 }
1474
1475 return 8 * static_cast<uint64_t>(
1476 DL.getIndexedOffsetInType(Src->getType(), Indices));
1477}
1478
1479bool IRTranslator::translateExtractValue(const User &U,
1480 MachineIRBuilder &MIRBuilder) {
1481 const Value *Src = U.getOperand(0);
1483 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1484 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
1485 unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
1486 auto &DstRegs = allocateVRegs(U);
1487
1488 for (unsigned i = 0; i < DstRegs.size(); ++i)
1489 DstRegs[i] = SrcRegs[Idx++];
1490
1491 return true;
1492}
1493
1494bool IRTranslator::translateInsertValue(const User &U,
1495 MachineIRBuilder &MIRBuilder) {
1496 const Value *Src = U.getOperand(0);
1498 auto &DstRegs = allocateVRegs(U);
1499 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
1500 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1501 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
1502 auto *InsertedIt = InsertedRegs.begin();
1503
1504 for (unsigned i = 0; i < DstRegs.size(); ++i) {
1505 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1506 DstRegs[i] = *InsertedIt++;
1507 else
1508 DstRegs[i] = SrcRegs[i];
1509 }
1510
1511 return true;
1512}
1513
1514bool IRTranslator::translateSelect(const User &U,
1515 MachineIRBuilder &MIRBuilder) {
1516 Register Tst = getOrCreateVReg(*U.getOperand(0));
1517 ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1518 ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1519 ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1520
1521 uint32_t Flags = 0;
1522 if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
1524
1525 for (unsigned i = 0; i < ResRegs.size(); ++i) {
1526 MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
1527 }
1528
1529 return true;
1530}
1531
1532bool IRTranslator::translateCopy(const User &U, const Value &V,
1533 MachineIRBuilder &MIRBuilder) {
1534 Register Src = getOrCreateVReg(V);
1535 auto &Regs = *VMap.getVRegs(U);
1536 if (Regs.empty()) {
1537 Regs.push_back(Src);
1538 VMap.getOffsets(U)->push_back(0);
1539 } else {
1540 // If we already assigned a vreg for this instruction, we can't change that.
1541 // Emit a copy to satisfy the users we already emitted.
1542 MIRBuilder.buildCopy(Regs[0], Src);
1543 }
1544 return true;
1545}
1546
1547bool IRTranslator::translateBitCast(const User &U,
1548 MachineIRBuilder &MIRBuilder) {
1549 // If we're bitcasting to the source type, we can reuse the source vreg.
1550 if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1551 getLLTForType(*U.getType(), *DL)) {
1552 // If the source is a ConstantInt then it was probably created by
1553 // ConstantHoisting and we should leave it alone.
1554 if (isa<ConstantInt>(U.getOperand(0)))
1555 return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1556 MIRBuilder);
1557 return translateCopy(U, *U.getOperand(0), MIRBuilder);
1558 }
1559
1560 return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1561}
1562
1563bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1564 MachineIRBuilder &MIRBuilder) {
1565 if (U.getType()->getScalarType()->isBFloatTy() ||
1566 U.getOperand(0)->getType()->getScalarType()->isBFloatTy())
1567 return false;
1568 Register Op = getOrCreateVReg(*U.getOperand(0));
1569 Register Res = getOrCreateVReg(U);
1570 MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1571 return true;
1572}
1573
1574bool IRTranslator::translateGetElementPtr(const User &U,
1575 MachineIRBuilder &MIRBuilder) {
1576 Value &Op0 = *U.getOperand(0);
1577 Register BaseReg = getOrCreateVReg(Op0);
1578 Type *PtrIRTy = Op0.getType();
1579 LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1580 Type *OffsetIRTy = DL->getIndexType(PtrIRTy);
1581 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1582
1583 uint32_t Flags = 0;
1584 if (isa<Instruction>(U)) {
1585 const Instruction &I = cast<Instruction>(U);
1587 }
1588
1589 // Normalize Vector GEP - all scalar operands should be converted to the
1590 // splat vector.
1591 unsigned VectorWidth = 0;
1592
1593 // True if we should use a splat vector; using VectorWidth alone is not
1594 // sufficient.
1595 bool WantSplatVector = false;
1596 if (auto *VT = dyn_cast<VectorType>(U.getType())) {
1597 VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1598 // We don't produce 1 x N vectors; those are treated as scalars.
1599 WantSplatVector = VectorWidth > 1;
1600 }
1601
1602 // We might need to splat the base pointer into a vector if the offsets
1603 // are vectors.
1604 if (WantSplatVector && !PtrTy.isVector()) {
1605 BaseReg =
1606 MIRBuilder
1607 .buildSplatVector(LLT::fixed_vector(VectorWidth, PtrTy), BaseReg)
1608 .getReg(0);
1609 PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
1610 PtrTy = getLLTForType(*PtrIRTy, *DL);
1611 OffsetIRTy = DL->getIndexType(PtrIRTy);
1612 OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1613 }
1614
1615 int64_t Offset = 0;
1616 for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1617 GTI != E; ++GTI) {
1618 const Value *Idx = GTI.getOperand();
1619 if (StructType *StTy = GTI.getStructTypeOrNull()) {
1620 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1622 continue;
1623 } else {
1624 uint64_t ElementSize = GTI.getSequentialElementStride(*DL);
1625
1626 // If this is a scalar constant or a splat vector of constants,
1627 // handle it quickly.
1628 if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1629 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1630 Offset += ElementSize * *Val;
1631 continue;
1632 }
1633 }
1634
1635 if (Offset != 0) {
1636 auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1637 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1638 .getReg(0);
1639 Offset = 0;
1640 }
1641
1642 Register IdxReg = getOrCreateVReg(*Idx);
1643 LLT IdxTy = MRI->getType(IdxReg);
1644 if (IdxTy != OffsetTy) {
1645 if (!IdxTy.isVector() && WantSplatVector) {
1646 IdxReg = MIRBuilder.buildSplatVector(
1647 OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0);
1648 }
1649
1650 IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1651 }
1652
1653 // N = N + Idx * ElementSize;
1654 // Avoid doing it for ElementSize of 1.
1655 Register GepOffsetReg;
1656 if (ElementSize != 1) {
1657 auto ElementSizeMIB = MIRBuilder.buildConstant(
1658 getLLTForType(*OffsetIRTy, *DL), ElementSize);
1659 GepOffsetReg =
1660 MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
1661 } else
1662 GepOffsetReg = IdxReg;
1663
1664 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1665 }
1666 }
1667
1668 if (Offset != 0) {
1669 auto OffsetMIB =
1670 MIRBuilder.buildConstant(OffsetTy, Offset);
1671
1672 if (int64_t(Offset) >= 0 && cast<GEPOperator>(U).isInBounds())
1674
1675 MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0),
1676 Flags);
1677 return true;
1678 }
1679
1680 MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1681 return true;
1682}
1683
1684bool IRTranslator::translateMemFunc(const CallInst &CI,
1685 MachineIRBuilder &MIRBuilder,
1686 unsigned Opcode) {
1687 const Value *SrcPtr = CI.getArgOperand(1);
1688 // If the source is undef, then just emit a nop.
1689 if (isa<UndefValue>(SrcPtr))
1690 return true;
1691
1693
1694 unsigned MinPtrSize = UINT_MAX;
1695 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {
1696 Register SrcReg = getOrCreateVReg(**AI);
1697 LLT SrcTy = MRI->getType(SrcReg);
1698 if (SrcTy.isPointer())
1699 MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);
1700 SrcRegs.push_back(SrcReg);
1701 }
1702
1703 LLT SizeTy = LLT::scalar(MinPtrSize);
1704
1705 // The size operand should be the minimum of the pointer sizes.
1706 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1707 if (MRI->getType(SizeOpReg) != SizeTy)
1708 SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);
1709
1710 auto ICall = MIRBuilder.buildInstr(Opcode);
1711 for (Register SrcReg : SrcRegs)
1712 ICall.addUse(SrcReg);
1713
1714 Align DstAlign;
1715 Align SrcAlign;
1716 unsigned IsVol =
1717 cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();
1718
1719 ConstantInt *CopySize = nullptr;
1720
1721 if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1722 DstAlign = MCI->getDestAlign().valueOrOne();
1723 SrcAlign = MCI->getSourceAlign().valueOrOne();
1724 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1725 } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
1726 DstAlign = MCI->getDestAlign().valueOrOne();
1727 SrcAlign = MCI->getSourceAlign().valueOrOne();
1728 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1729 } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1730 DstAlign = MMI->getDestAlign().valueOrOne();
1731 SrcAlign = MMI->getSourceAlign().valueOrOne();
1732 CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2));
1733 } else {
1734 auto *MSI = cast<MemSetInst>(&CI);
1735 DstAlign = MSI->getDestAlign().valueOrOne();
1736 }
1737
1738 if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1739 // We need to propagate the tail call flag from the IR inst as an argument.
1740 // Otherwise, we have to pessimize and assume later that we cannot tail call
1741 // any memory intrinsics.
1742 ICall.addImm(CI.isTailCall() ? 1 : 0);
1743 }
1744
1745 // Create mem operands to store the alignment and volatile info.
1748 if (IsVol) {
1749 LoadFlags |= MachineMemOperand::MOVolatile;
1750 StoreFlags |= MachineMemOperand::MOVolatile;
1751 }
1752
1753 AAMDNodes AAInfo = CI.getAAMetadata();
1754 if (AA && CopySize &&
1756 SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) {
1757 LoadFlags |= MachineMemOperand::MOInvariant;
1758
1759 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1760 // but the previous usage implied it did. Probably should check
1761 // isDereferenceableAndAlignedPointer.
1763 }
1764
1765 ICall.addMemOperand(
1767 StoreFlags, 1, DstAlign, AAInfo));
1768 if (Opcode != TargetOpcode::G_MEMSET)
1769 ICall.addMemOperand(MF->getMachineMemOperand(
1770 MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
1771
1772 return true;
1773}
1774
1775void IRTranslator::getStackGuard(Register DstReg,
1776 MachineIRBuilder &MIRBuilder) {
1778 MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1779 auto MIB =
1780 MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1781
1782 auto &TLI = *MF->getSubtarget().getTargetLowering();
1783 Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
1784 if (!Global)
1785 return;
1786
1787 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1788 LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
1789
1790 MachinePointerInfo MPInfo(Global);
1794 MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
1795 MIB.setMemRefs({MemRef});
1796}
1797
1798bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1799 MachineIRBuilder &MIRBuilder) {
1800 ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1801 MIRBuilder.buildInstr(
1802 Op, {ResRegs[0], ResRegs[1]},
1803 {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});
1804
1805 return true;
1806}
1807
1808bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1809 MachineIRBuilder &MIRBuilder) {
1810 Register Dst = getOrCreateVReg(CI);
1811 Register Src0 = getOrCreateVReg(*CI.getOperand(0));
1812 Register Src1 = getOrCreateVReg(*CI.getOperand(1));
1813 uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
1814 MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
1815 return true;
1816}
1817
1818unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1819 switch (ID) {
1820 default:
1821 break;
1822 case Intrinsic::bswap:
1823 return TargetOpcode::G_BSWAP;
1824 case Intrinsic::bitreverse:
1825 return TargetOpcode::G_BITREVERSE;
1826 case Intrinsic::fshl:
1827 return TargetOpcode::G_FSHL;
1828 case Intrinsic::fshr:
1829 return TargetOpcode::G_FSHR;
1830 case Intrinsic::ceil:
1831 return TargetOpcode::G_FCEIL;
1832 case Intrinsic::cos:
1833 return TargetOpcode::G_FCOS;
1834 case Intrinsic::ctpop:
1835 return TargetOpcode::G_CTPOP;
1836 case Intrinsic::exp:
1837 return TargetOpcode::G_FEXP;
1838 case Intrinsic::exp2:
1839 return TargetOpcode::G_FEXP2;
1840 case Intrinsic::exp10:
1841 return TargetOpcode::G_FEXP10;
1842 case Intrinsic::fabs:
1843 return TargetOpcode::G_FABS;
1844 case Intrinsic::copysign:
1845 return TargetOpcode::G_FCOPYSIGN;
1846 case Intrinsic::minnum:
1847 return TargetOpcode::G_FMINNUM;
1848 case Intrinsic::maxnum:
1849 return TargetOpcode::G_FMAXNUM;
1850 case Intrinsic::minimum:
1851 return TargetOpcode::G_FMINIMUM;
1852 case Intrinsic::maximum:
1853 return TargetOpcode::G_FMAXIMUM;
1854 case Intrinsic::canonicalize:
1855 return TargetOpcode::G_FCANONICALIZE;
1856 case Intrinsic::floor:
1857 return TargetOpcode::G_FFLOOR;
1858 case Intrinsic::fma:
1859 return TargetOpcode::G_FMA;
1860 case Intrinsic::log:
1861 return TargetOpcode::G_FLOG;
1862 case Intrinsic::log2:
1863 return TargetOpcode::G_FLOG2;
1864 case Intrinsic::log10:
1865 return TargetOpcode::G_FLOG10;
1866 case Intrinsic::ldexp:
1867 return TargetOpcode::G_FLDEXP;
1868 case Intrinsic::nearbyint:
1869 return TargetOpcode::G_FNEARBYINT;
1870 case Intrinsic::pow:
1871 return TargetOpcode::G_FPOW;
1872 case Intrinsic::powi:
1873 return TargetOpcode::G_FPOWI;
1874 case Intrinsic::rint:
1875 return TargetOpcode::G_FRINT;
1876 case Intrinsic::round:
1877 return TargetOpcode::G_INTRINSIC_ROUND;
1878 case Intrinsic::roundeven:
1879 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1880 case Intrinsic::sin:
1881 return TargetOpcode::G_FSIN;
1882 case Intrinsic::sqrt:
1883 return TargetOpcode::G_FSQRT;
1884 case Intrinsic::trunc:
1885 return TargetOpcode::G_INTRINSIC_TRUNC;
1886 case Intrinsic::readcyclecounter:
1887 return TargetOpcode::G_READCYCLECOUNTER;
1888 case Intrinsic::readsteadycounter:
1889 return TargetOpcode::G_READSTEADYCOUNTER;
1890 case Intrinsic::ptrmask:
1891 return TargetOpcode::G_PTRMASK;
1892 case Intrinsic::lrint:
1893 return TargetOpcode::G_INTRINSIC_LRINT;
1894 // FADD/FMUL require checking the FMF, so are handled elsewhere.
1895 case Intrinsic::vector_reduce_fmin:
1896 return TargetOpcode::G_VECREDUCE_FMIN;
1897 case Intrinsic::vector_reduce_fmax:
1898 return TargetOpcode::G_VECREDUCE_FMAX;
1899 case Intrinsic::vector_reduce_fminimum:
1900 return TargetOpcode::G_VECREDUCE_FMINIMUM;
1901 case Intrinsic::vector_reduce_fmaximum:
1902 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
1903 case Intrinsic::vector_reduce_add:
1904 return TargetOpcode::G_VECREDUCE_ADD;
1905 case Intrinsic::vector_reduce_mul:
1906 return TargetOpcode::G_VECREDUCE_MUL;
1907 case Intrinsic::vector_reduce_and:
1908 return TargetOpcode::G_VECREDUCE_AND;
1909 case Intrinsic::vector_reduce_or:
1910 return TargetOpcode::G_VECREDUCE_OR;
1911 case Intrinsic::vector_reduce_xor:
1912 return TargetOpcode::G_VECREDUCE_XOR;
1913 case Intrinsic::vector_reduce_smax:
1914 return TargetOpcode::G_VECREDUCE_SMAX;
1915 case Intrinsic::vector_reduce_smin:
1916 return TargetOpcode::G_VECREDUCE_SMIN;
1917 case Intrinsic::vector_reduce_umax:
1918 return TargetOpcode::G_VECREDUCE_UMAX;
1919 case Intrinsic::vector_reduce_umin:
1920 return TargetOpcode::G_VECREDUCE_UMIN;
1921 case Intrinsic::lround:
1922 return TargetOpcode::G_LROUND;
1923 case Intrinsic::llround:
1924 return TargetOpcode::G_LLROUND;
1925 case Intrinsic::get_fpenv:
1926 return TargetOpcode::G_GET_FPENV;
1927 case Intrinsic::get_fpmode:
1928 return TargetOpcode::G_GET_FPMODE;
1929 }
1931}
1932
1933bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1935 MachineIRBuilder &MIRBuilder) {
1936
1937 unsigned Op = getSimpleIntrinsicOpcode(ID);
1938
1939 // Is this a simple intrinsic?
1941 return false;
1942
1943 // Yes. Let's translate it.
1945 for (const auto &Arg : CI.args())
1946 VRegs.push_back(getOrCreateVReg(*Arg));
1947
1948 MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1950 return true;
1951}
1952
1953// TODO: Include ConstainedOps.def when all strict instructions are defined.
1955 switch (ID) {
1956 case Intrinsic::experimental_constrained_fadd:
1957 return TargetOpcode::G_STRICT_FADD;
1958 case Intrinsic::experimental_constrained_fsub:
1959 return TargetOpcode::G_STRICT_FSUB;
1960 case Intrinsic::experimental_constrained_fmul:
1961 return TargetOpcode::G_STRICT_FMUL;
1962 case Intrinsic::experimental_constrained_fdiv:
1963 return TargetOpcode::G_STRICT_FDIV;
1964 case Intrinsic::experimental_constrained_frem:
1965 return TargetOpcode::G_STRICT_FREM;
1966 case Intrinsic::experimental_constrained_fma:
1967 return TargetOpcode::G_STRICT_FMA;
1968 case Intrinsic::experimental_constrained_sqrt:
1969 return TargetOpcode::G_STRICT_FSQRT;
1970 case Intrinsic::experimental_constrained_ldexp:
1971 return TargetOpcode::G_STRICT_FLDEXP;
1972 default:
1973 return 0;
1974 }
1975}
1976
1977bool IRTranslator::translateConstrainedFPIntrinsic(
1978 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
1980
1981 unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
1982 if (!Opcode)
1983 return false;
1984
1988
1990 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(0)));
1991 if (!FPI.isUnaryOp())
1992 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(1)));
1993 if (FPI.isTernaryOp())
1994 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(2)));
1995
1996 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
1997 return true;
1998}
1999
2000std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
2001 auto VRegs = getOrCreateVRegs(Arg);
2002 if (VRegs.size() != 1)
2003 return std::nullopt;
2004
2005 // Arguments are lowered as a copy of a livein physical register.
2006 auto *VRegDef = MF->getRegInfo().getVRegDef(VRegs[0]);
2007 if (!VRegDef || !VRegDef->isCopy())
2008 return std::nullopt;
2009 return VRegDef->getOperand(1).getReg().asMCReg();
2010}
2011
2012bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,
2013 const DILocalVariable *Var,
2014 const DIExpression *Expr,
2015 const DebugLoc &DL,
2016 MachineIRBuilder &MIRBuilder) {
2017 auto *Arg = dyn_cast<Argument>(Val);
2018 if (!Arg)
2019 return false;
2020
2021 if (!Expr->isEntryValue())
2022 return false;
2023
2024 std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
2025 if (!PhysReg) {
2026 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2027 << ": expression is entry_value but "
2028 << "couldn't find a physical register\n");
2029 LLVM_DEBUG(dbgs() << *Var << "\n");
2030 return true;
2031 }
2032
2033 if (isDeclare) {
2034 // Append an op deref to account for the fact that this is a dbg_declare.
2035 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
2036 MF->setVariableDbgInfo(Var, Expr, *PhysReg, DL);
2037 } else {
2038 MIRBuilder.buildDirectDbgValue(*PhysReg, Var, Expr);
2039 }
2040
2041 return true;
2042}
2043
2044bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
2045 MachineIRBuilder &MIRBuilder) {
2046 if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
2047 if (ORE->enabled()) {
2048 if (MemoryOpRemark::canHandle(MI, *LibInfo)) {
2049 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2050 R.visit(MI);
2051 }
2052 }
2053 }
2054
2055 // If this is a simple intrinsic (that is, we just need to add a def of
2056 // a vreg, and uses for each arg operand, then translate it.
2057 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
2058 return true;
2059
2060 switch (ID) {
2061 default:
2062 break;
2063 case Intrinsic::lifetime_start:
2064 case Intrinsic::lifetime_end: {
2065 // No stack colouring in O0, discard region information.
2067 return true;
2068
2069 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2070 : TargetOpcode::LIFETIME_END;
2071
2072 // Get the underlying objects for the location passed on the lifetime
2073 // marker.
2075 getUnderlyingObjects(CI.getArgOperand(1), Allocas);
2076
2077 // Iterate over each underlying object, creating lifetime markers for each
2078 // static alloca. Quit if we find a non-static alloca.
2079 for (const Value *V : Allocas) {
2080 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
2081 if (!AI)
2082 continue;
2083
2084 if (!AI->isStaticAlloca())
2085 return true;
2086
2087 MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
2088 }
2089 return true;
2090 }
2091 case Intrinsic::dbg_declare: {
2092 const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
2093 assert(DI.getVariable() && "Missing variable");
2094 translateDbgDeclareRecord(DI.getAddress(), DI.hasArgList(), DI.getVariable(),
2095 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2096 return true;
2097 }
2098 case Intrinsic::dbg_label: {
2099 const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
2100 assert(DI.getLabel() && "Missing label");
2101
2103 MIRBuilder.getDebugLoc()) &&
2104 "Expected inlined-at fields to agree");
2105
2106 MIRBuilder.buildDbgLabel(DI.getLabel());
2107 return true;
2108 }
2109 case Intrinsic::vaend:
2110 // No target I know of cares about va_end. Certainly no in-tree target
2111 // does. Simplest intrinsic ever!
2112 return true;
2113 case Intrinsic::vastart: {
2114 auto &TLI = *MF->getSubtarget().getTargetLowering();
2115 Value *Ptr = CI.getArgOperand(0);
2116 unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
2117 Align Alignment = getKnownAlignment(Ptr, *DL);
2118
2119 MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
2120 .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),
2122 ListSize, Alignment));
2123 return true;
2124 }
2125 case Intrinsic::dbg_assign:
2126 // A dbg.assign is a dbg.value with more information about stack locations,
2127 // typically produced during optimisation of variables with leaked
2128 // addresses. We can treat it like a normal dbg_value intrinsic here; to
2129 // benefit from the full analysis of stack/SSA locations, GlobalISel would
2130 // need to register for and use the AssignmentTrackingAnalysis pass.
2132 case Intrinsic::dbg_value: {
2133 // This form of DBG_VALUE is target-independent.
2134 const DbgValueInst &DI = cast<DbgValueInst>(CI);
2135 translateDbgValueRecord(DI.getValue(), DI.hasArgList(), DI.getVariable(),
2136 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2137 return true;
2138 }
2139 case Intrinsic::uadd_with_overflow:
2140 return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
2141 case Intrinsic::sadd_with_overflow:
2142 return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
2143 case Intrinsic::usub_with_overflow:
2144 return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
2145 case Intrinsic::ssub_with_overflow:
2146 return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
2147 case Intrinsic::umul_with_overflow:
2148 return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
2149 case Intrinsic::smul_with_overflow:
2150 return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
2151 case Intrinsic::uadd_sat:
2152 return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
2153 case Intrinsic::sadd_sat:
2154 return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
2155 case Intrinsic::usub_sat:
2156 return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
2157 case Intrinsic::ssub_sat:
2158 return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
2159 case Intrinsic::ushl_sat:
2160 return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
2161 case Intrinsic::sshl_sat:
2162 return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
2163 case Intrinsic::umin:
2164 return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
2165 case Intrinsic::umax:
2166 return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
2167 case Intrinsic::smin:
2168 return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
2169 case Intrinsic::smax:
2170 return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
2171 case Intrinsic::abs:
2172 // TODO: Preserve "int min is poison" arg in GMIR?
2173 return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
2174 case Intrinsic::smul_fix:
2175 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2176 case Intrinsic::umul_fix:
2177 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2178 case Intrinsic::smul_fix_sat:
2179 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2180 case Intrinsic::umul_fix_sat:
2181 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2182 case Intrinsic::sdiv_fix:
2183 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2184 case Intrinsic::udiv_fix:
2185 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2186 case Intrinsic::sdiv_fix_sat:
2187 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2188 case Intrinsic::udiv_fix_sat:
2189 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2190 case Intrinsic::fmuladd: {
2191 const TargetMachine &TM = MF->getTarget();
2192 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2193 Register Dst = getOrCreateVReg(CI);
2194 Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
2195 Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
2196 Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
2197 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2199 TLI.getValueType(*DL, CI.getType()))) {
2200 // TODO: Revisit this to see if we should move this part of the
2201 // lowering to the combiner.
2202 MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,
2204 } else {
2205 LLT Ty = getLLTForType(*CI.getType(), *DL);
2206 auto FMul = MIRBuilder.buildFMul(
2207 Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));
2208 MIRBuilder.buildFAdd(Dst, FMul, Op2,
2210 }
2211 return true;
2212 }
2213 case Intrinsic::convert_from_fp16:
2214 // FIXME: This intrinsic should probably be removed from the IR.
2215 MIRBuilder.buildFPExt(getOrCreateVReg(CI),
2216 getOrCreateVReg(*CI.getArgOperand(0)),
2218 return true;
2219 case Intrinsic::convert_to_fp16:
2220 // FIXME: This intrinsic should probably be removed from the IR.
2221 MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
2222 getOrCreateVReg(*CI.getArgOperand(0)),
2224 return true;
2225 case Intrinsic::frexp: {
2226 ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
2227 MIRBuilder.buildFFrexp(VRegs[0], VRegs[1],
2228 getOrCreateVReg(*CI.getArgOperand(0)),
2230 return true;
2231 }
2232 case Intrinsic::memcpy_inline:
2233 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
2234 case Intrinsic::memcpy:
2235 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2236 case Intrinsic::memmove:
2237 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2238 case Intrinsic::memset:
2239 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2240 case Intrinsic::eh_typeid_for: {
2242 Register Reg = getOrCreateVReg(CI);
2243 unsigned TypeID = MF->getTypeIDFor(GV);
2244 MIRBuilder.buildConstant(Reg, TypeID);
2245 return true;
2246 }
2247 case Intrinsic::objectsize:
2248 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2249
2250 case Intrinsic::is_constant:
2251 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2252
2253 case Intrinsic::stackguard:
2254 getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2255 return true;
2256 case Intrinsic::stackprotector: {
2257 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2258 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2259 Register GuardVal;
2260 if (TLI.useLoadStackGuardNode()) {
2261 GuardVal = MRI->createGenericVirtualRegister(PtrTy);
2262 getStackGuard(GuardVal, MIRBuilder);
2263 } else
2264 GuardVal = getOrCreateVReg(*CI.getArgOperand(0)); // The guard's value.
2265
2266 AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
2267 int FI = getOrCreateFrameIndex(*Slot);
2269
2270 MIRBuilder.buildStore(
2271 GuardVal, getOrCreateVReg(*Slot),
2275 PtrTy, Align(8)));
2276 return true;
2277 }
2278 case Intrinsic::stacksave: {
2279 MIRBuilder.buildInstr(TargetOpcode::G_STACKSAVE, {getOrCreateVReg(CI)}, {});
2280 return true;
2281 }
2282 case Intrinsic::stackrestore: {
2283 MIRBuilder.buildInstr(TargetOpcode::G_STACKRESTORE, {},
2284 {getOrCreateVReg(*CI.getArgOperand(0))});
2285 return true;
2286 }
2287 case Intrinsic::cttz:
2288 case Intrinsic::ctlz: {
2289 ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
2290 bool isTrailing = ID == Intrinsic::cttz;
2291 unsigned Opcode = isTrailing
2292 ? Cst->isZero() ? TargetOpcode::G_CTTZ
2293 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2294 : Cst->isZero() ? TargetOpcode::G_CTLZ
2295 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2296 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
2297 {getOrCreateVReg(*CI.getArgOperand(0))});
2298 return true;
2299 }
2300 case Intrinsic::invariant_start: {
2301 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2303 MIRBuilder.buildUndef(Undef);
2304 return true;
2305 }
2306 case Intrinsic::invariant_end:
2307 return true;
2308 case Intrinsic::expect:
2309 case Intrinsic::annotation:
2310 case Intrinsic::ptr_annotation:
2311 case Intrinsic::launder_invariant_group:
2312 case Intrinsic::strip_invariant_group: {
2313 // Drop the intrinsic, but forward the value.
2314 MIRBuilder.buildCopy(getOrCreateVReg(CI),
2315 getOrCreateVReg(*CI.getArgOperand(0)));
2316 return true;
2317 }
2318 case Intrinsic::assume:
2319 case Intrinsic::experimental_noalias_scope_decl:
2320 case Intrinsic::var_annotation:
2321 case Intrinsic::sideeffect:
2322 // Discard annotate attributes, assumptions, and artificial side-effects.
2323 return true;
2324 case Intrinsic::read_volatile_register:
2325 case Intrinsic::read_register: {
2326 Value *Arg = CI.getArgOperand(0);
2327 MIRBuilder
2328 .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2329 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
2330 return true;
2331 }
2332 case Intrinsic::write_register: {
2333 Value *Arg = CI.getArgOperand(0);
2334 MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
2335 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
2336 .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
2337 return true;
2338 }
2339 case Intrinsic::localescape: {
2340 MachineBasicBlock &EntryMBB = MF->front();
2342
2343 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2344 // is the same on all targets.
2345 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2347 if (isa<ConstantPointerNull>(Arg))
2348 continue; // Skip null pointers. They represent a hole in index space.
2349
2350 int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
2351 MCSymbol *FrameAllocSym =
2352 MF->getMMI().getContext().getOrCreateFrameAllocSymbol(EscapedName,
2353 Idx);
2354
2355 // This should be inserted at the start of the entry block.
2356 auto LocalEscape =
2357 MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)
2358 .addSym(FrameAllocSym)
2359 .addFrameIndex(FI);
2360
2361 EntryMBB.insert(EntryMBB.begin(), LocalEscape);
2362 }
2363
2364 return true;
2365 }
2366 case Intrinsic::vector_reduce_fadd:
2367 case Intrinsic::vector_reduce_fmul: {
2368 // Need to check for the reassoc flag to decide whether we want a
2369 // sequential reduction opcode or not.
2370 Register Dst = getOrCreateVReg(CI);
2371 Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));
2372 Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));
2373 unsigned Opc = 0;
2374 if (!CI.hasAllowReassoc()) {
2375 // The sequential ordering case.
2376 Opc = ID == Intrinsic::vector_reduce_fadd
2377 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2378 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2379 MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2381 return true;
2382 }
2383 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2384 // since the associativity doesn't matter.
2385 unsigned ScalarOpc;
2386 if (ID == Intrinsic::vector_reduce_fadd) {
2387 Opc = TargetOpcode::G_VECREDUCE_FADD;
2388 ScalarOpc = TargetOpcode::G_FADD;
2389 } else {
2390 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2391 ScalarOpc = TargetOpcode::G_FMUL;
2392 }
2393 LLT DstTy = MRI->getType(Dst);
2394 auto Rdx = MIRBuilder.buildInstr(
2395 Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));
2396 MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2398
2399 return true;
2400 }
2401 case Intrinsic::trap:
2402 case Intrinsic::debugtrap:
2403 case Intrinsic::ubsantrap: {
2404 StringRef TrapFuncName =
2405 CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
2406 if (TrapFuncName.empty())
2407 break; // Use the default handling.
2409 if (ID == Intrinsic::ubsantrap) {
2410 Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
2411 CI.getArgOperand(0)->getType(), 0});
2412 }
2413 Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
2414 Info.CB = &CI;
2415 Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
2416 return CLI->lowerCall(MIRBuilder, Info);
2417 }
2418 case Intrinsic::amdgcn_cs_chain:
2419 return translateCallBase(CI, MIRBuilder);
2420 case Intrinsic::fptrunc_round: {
2422
2423 // Convert the metadata argument to a constant integer
2424 Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata();
2425 std::optional<RoundingMode> RoundMode =
2426 convertStrToRoundingMode(cast<MDString>(MD)->getString());
2427
2428 // Add the Rounding mode as an integer
2429 MIRBuilder
2430 .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2431 {getOrCreateVReg(CI)},
2432 {getOrCreateVReg(*CI.getArgOperand(0))}, Flags)
2433 .addImm((int)*RoundMode);
2434
2435 return true;
2436 }
2437 case Intrinsic::is_fpclass: {
2438 Value *FpValue = CI.getOperand(0);
2439 ConstantInt *TestMaskValue = cast<ConstantInt>(CI.getOperand(1));
2440
2441 MIRBuilder
2442 .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},
2443 {getOrCreateVReg(*FpValue)})
2444 .addImm(TestMaskValue->getZExtValue());
2445
2446 return true;
2447 }
2448 case Intrinsic::set_fpenv: {
2449 Value *FPEnv = CI.getOperand(0);
2450 MIRBuilder.buildInstr(TargetOpcode::G_SET_FPENV, {},
2451 {getOrCreateVReg(*FPEnv)});
2452 return true;
2453 }
2454 case Intrinsic::reset_fpenv: {
2455 MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPENV, {}, {});
2456 return true;
2457 }
2458 case Intrinsic::set_fpmode: {
2459 Value *FPState = CI.getOperand(0);
2460 MIRBuilder.buildInstr(TargetOpcode::G_SET_FPMODE, {},
2461 { getOrCreateVReg(*FPState) });
2462 return true;
2463 }
2464 case Intrinsic::reset_fpmode: {
2465 MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {});
2466 return true;
2467 }
2468 case Intrinsic::prefetch: {
2469 Value *Addr = CI.getOperand(0);
2470 unsigned RW = cast<ConstantInt>(CI.getOperand(1))->getZExtValue();
2471 unsigned Locality = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
2472 unsigned CacheType = cast<ConstantInt>(CI.getOperand(3))->getZExtValue();
2473
2475 auto &MMO = *MF->getMachineMemOperand(MachinePointerInfo(Addr), Flags,
2476 LLT(), Align());
2477
2478 MIRBuilder.buildPrefetch(getOrCreateVReg(*Addr), RW, Locality, CacheType,
2479 MMO);
2480
2481 return true;
2482 }
2483#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2484 case Intrinsic::INTRINSIC:
2485#include "llvm/IR/ConstrainedOps.def"
2486 return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2487 MIRBuilder);
2488
2489 }
2490 return false;
2491}
2492
2493bool IRTranslator::translateInlineAsm(const CallBase &CB,
2494 MachineIRBuilder &MIRBuilder) {
2495
2497
2498 if (!ALI) {
2499 LLVM_DEBUG(
2500 dbgs() << "Inline asm lowering is not supported for this target yet\n");
2501 return false;
2502 }
2503
2504 return ALI->lowerInlineAsm(
2505 MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });
2506}
2507
2508bool IRTranslator::translateCallBase(const CallBase &CB,
2509 MachineIRBuilder &MIRBuilder) {
2510 ArrayRef<Register> Res = getOrCreateVRegs(CB);
2511
2513 Register SwiftInVReg = 0;
2514 Register SwiftErrorVReg = 0;
2515 for (const auto &Arg : CB.args()) {
2516 if (CLI->supportSwiftError() && isSwiftError(Arg)) {
2517 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2518 LLT Ty = getLLTForType(*Arg->getType(), *DL);
2519 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2520 MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
2521 &CB, &MIRBuilder.getMBB(), Arg));
2522 Args.emplace_back(ArrayRef(SwiftInVReg));
2523 SwiftErrorVReg =
2524 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2525 continue;
2526 }
2527 Args.push_back(getOrCreateVRegs(*Arg));
2528 }
2529
2530 if (auto *CI = dyn_cast<CallInst>(&CB)) {
2531 if (ORE->enabled()) {
2532 if (MemoryOpRemark::canHandle(CI, *LibInfo)) {
2533 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2534 R.visit(CI);
2535 }
2536 }
2537 }
2538
2539 // We don't set HasCalls on MFI here yet because call lowering may decide to
2540 // optimize into tail calls. Instead, we defer that to selection where a final
2541 // scan is done to check if any instructions are calls.
2542 bool Success =
2543 CLI->lowerCall(MIRBuilder, CB, Res, Args, SwiftErrorVReg,
2544 [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
2545
2546 // Check if we just inserted a tail call.
2547 if (Success) {
2548 assert(!HasTailCall && "Can't tail call return twice from block?");
2550 HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
2551 }
2552
2553 return Success;
2554}
2555
2556bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2557 const CallInst &CI = cast<CallInst>(U);
2558 auto TII = MF->getTarget().getIntrinsicInfo();
2559 const Function *F = CI.getCalledFunction();
2560
2561 // FIXME: support Windows dllimport function calls and calls through
2562 // weak symbols.
2563 if (F && (F->hasDLLImportStorageClass() ||
2565 F->hasExternalWeakLinkage())))
2566 return false;
2567
2568 // FIXME: support control flow guard targets.
2570 return false;
2571
2572 // FIXME: support statepoints and related.
2573 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
2574 return false;
2575
2576 if (CI.isInlineAsm())
2577 return translateInlineAsm(CI, MIRBuilder);
2578
2579 diagnoseDontCall(CI);
2580
2582 if (F && F->isIntrinsic()) {
2583 ID = F->getIntrinsicID();
2585 ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2586 }
2587
2588 if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2589 return translateCallBase(CI, MIRBuilder);
2590
2591 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2592
2593 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2594 return true;
2595
2596 ArrayRef<Register> ResultRegs;
2597 if (!CI.getType()->isVoidTy())
2598 ResultRegs = getOrCreateVRegs(CI);
2599
2600 // Ignore the callsite attributes. Backend code is most likely not expecting
2601 // an intrinsic to sometimes have side effects and sometimes not.
2602 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);
2603 if (isa<FPMathOperator>(CI))
2604 MIB->copyIRFlags(CI);
2605
2606 for (const auto &Arg : enumerate(CI.args())) {
2607 // If this is required to be an immediate, don't materialize it in a
2608 // register.
2609 if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2610 if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
2611 // imm arguments are more convenient than cimm (and realistically
2612 // probably sufficient), so use them.
2613 assert(CI->getBitWidth() <= 64 &&
2614 "large intrinsic immediates not handled");
2615 MIB.addImm(CI->getSExtValue());
2616 } else {
2617 MIB.addFPImm(cast<ConstantFP>(Arg.value()));
2618 }
2619 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {
2620 auto *MD = MDVal->getMetadata();
2621 auto *MDN = dyn_cast<MDNode>(MD);
2622 if (!MDN) {
2623 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))
2624 MDN = MDNode::get(MF->getFunction().getContext(), ConstMD);
2625 else // This was probably an MDString.
2626 return false;
2627 }
2628 MIB.addMetadata(MDN);
2629 } else {
2630 ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
2631 if (VRegs.size() > 1)
2632 return false;
2633 MIB.addUse(VRegs[0]);
2634 }
2635 }
2636
2637 // Add a MachineMemOperand if it is a target mem intrinsic.
2638 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2640 // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2641 if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2642 Align Alignment = Info.align.value_or(
2643 DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
2644 LLT MemTy = Info.memVT.isSimple()
2645 ? getLLTForMVT(Info.memVT.getSimpleVT())
2646 : LLT::scalar(Info.memVT.getStoreSizeInBits());
2647
2648 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
2649 // didn't yield anything useful.
2651 if (Info.ptrVal)
2652 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2653 else if (Info.fallbackAddressSpace)
2654 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2655 MIB.addMemOperand(
2656 MF->getMachineMemOperand(MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata()));
2657 }
2658
2659 return true;
2660}
2661
2662bool IRTranslator::findUnwindDestinations(
2663 const BasicBlock *EHPadBB,
2664 BranchProbability Prob,
2665 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2666 &UnwindDests) {
2668 EHPadBB->getParent()->getFunction().getPersonalityFn());
2669 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2670 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2671 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2672 bool IsSEH = isAsynchronousEHPersonality(Personality);
2673
2674 if (IsWasmCXX) {
2675 // Ignore this for now.
2676 return false;
2677 }
2678
2679 while (EHPadBB) {
2680 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2681 BasicBlock *NewEHPadBB = nullptr;
2682 if (isa<LandingPadInst>(Pad)) {
2683 // Stop on landingpads. They are not funclets.
2684 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2685 break;
2686 }
2687 if (isa<CleanupPadInst>(Pad)) {
2688 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2689 // personalities.
2690 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2691 UnwindDests.back().first->setIsEHScopeEntry();
2692 UnwindDests.back().first->setIsEHFuncletEntry();
2693 break;
2694 }
2695 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2696 // Add the catchpad handlers to the possible destinations.
2697 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2698 UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2699 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2700 if (IsMSVCCXX || IsCoreCLR)
2701 UnwindDests.back().first->setIsEHFuncletEntry();
2702 if (!IsSEH)
2703 UnwindDests.back().first->setIsEHScopeEntry();
2704 }
2705 NewEHPadBB = CatchSwitch->getUnwindDest();
2706 } else {
2707 continue;
2708 }
2709
2710 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2711 if (BPI && NewEHPadBB)
2712 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2713 EHPadBB = NewEHPadBB;
2714 }
2715 return true;
2716}
2717
2718bool IRTranslator::translateInvoke(const User &U,
2719 MachineIRBuilder &MIRBuilder) {
2720 const InvokeInst &I = cast<InvokeInst>(U);
2721 MCContext &Context = MF->getContext();
2722
2723 const BasicBlock *ReturnBB = I.getSuccessor(0);
2724 const BasicBlock *EHPadBB = I.getSuccessor(1);
2725
2726 const Function *Fn = I.getCalledFunction();
2727
2728 // FIXME: support invoking patchpoint and statepoint intrinsics.
2729 if (Fn && Fn->isIntrinsic())
2730 return false;
2731
2732 // FIXME: support whatever these are.
2733 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
2734 return false;
2735
2736 // FIXME: support control flow guard targets.
2737 if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
2738 return false;
2739
2740 // FIXME: support Windows exception handling.
2741 if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHI()))
2742 return false;
2743
2744 // FIXME: support Windows dllimport function calls and calls through
2745 // weak symbols.
2746 if (Fn && (Fn->hasDLLImportStorageClass() ||
2748 Fn->hasExternalWeakLinkage())))
2749 return false;
2750
2751 bool LowerInlineAsm = I.isInlineAsm();
2752 bool NeedEHLabel = true;
2753
2754 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2755 // the region covered by the try.
2756 MCSymbol *BeginSymbol = nullptr;
2757 if (NeedEHLabel) {
2758 MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START);
2759 BeginSymbol = Context.createTempSymbol();
2760 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
2761 }
2762
2763 if (LowerInlineAsm) {
2764 if (!translateInlineAsm(I, MIRBuilder))
2765 return false;
2766 } else if (!translateCallBase(I, MIRBuilder))
2767 return false;
2768
2769 MCSymbol *EndSymbol = nullptr;
2770 if (NeedEHLabel) {
2771 EndSymbol = Context.createTempSymbol();
2772 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
2773 }
2774
2776 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2777 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2778 BranchProbability EHPadBBProb =
2779 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2781
2782 if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2783 return false;
2784
2785 MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
2786 &ReturnMBB = getMBB(*ReturnBB);
2787 // Update successor info.
2788 addSuccessorWithProb(InvokeMBB, &ReturnMBB);
2789 for (auto &UnwindDest : UnwindDests) {
2790 UnwindDest.first->setIsEHPad();
2791 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2792 }
2793 InvokeMBB->normalizeSuccProbs();
2794
2795 if (NeedEHLabel) {
2796 assert(BeginSymbol && "Expected a begin symbol!");
2797 assert(EndSymbol && "Expected an end symbol!");
2798 MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
2799 }
2800
2801 MIRBuilder.buildBr(ReturnMBB);
2802 return true;
2803}
2804
2805bool IRTranslator::translateCallBr(const User &U,
2806 MachineIRBuilder &MIRBuilder) {
2807 // FIXME: Implement this.
2808 return false;
2809}
2810
2811bool IRTranslator::translateLandingPad(const User &U,
2812 MachineIRBuilder &MIRBuilder) {
2813 const LandingPadInst &LP = cast<LandingPadInst>(U);
2814
2815 MachineBasicBlock &MBB = MIRBuilder.getMBB();
2816
2817 MBB.setIsEHPad();
2818
2819 // If there aren't registers to copy the values into (e.g., during SjLj
2820 // exceptions), then don't bother.
2821 auto &TLI = *MF->getSubtarget().getTargetLowering();
2822 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
2823 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2824 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2825 return true;
2826
2827 // If landingpad's return type is token type, we don't create DAG nodes
2828 // for its exception pointer and selector value. The extraction of exception
2829 // pointer or selector value from token type landingpads is not currently
2830 // supported.
2831 if (LP.getType()->isTokenTy())
2832 return true;
2833
2834 // Add a label to mark the beginning of the landing pad. Deletion of the
2835 // landing pad can thus be detected via the MachineModuleInfo.
2836 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
2837 .addSym(MF->addLandingPad(&MBB));
2838
2839 // If the unwinder does not preserve all registers, ensure that the
2840 // function marks the clobbered registers as used.
2842 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
2844
2845 LLT Ty = getLLTForType(*LP.getType(), *DL);
2847 MIRBuilder.buildUndef(Undef);
2848
2850 for (Type *Ty : cast<StructType>(LP.getType())->elements())
2851 Tys.push_back(getLLTForType(*Ty, *DL));
2852 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
2853
2854 // Mark exception register as live in.
2855 Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
2856 if (!ExceptionReg)
2857 return false;
2858
2859 MBB.addLiveIn(ExceptionReg);
2860 ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
2861 MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
2862
2863 Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
2864 if (!SelectorReg)
2865 return false;
2866
2867 MBB.addLiveIn(SelectorReg);
2868 Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
2869 MIRBuilder.buildCopy(PtrVReg, SelectorReg);
2870 MIRBuilder.buildCast(ResRegs[1], PtrVReg);
2871
2872 return true;
2873}
2874
2875bool IRTranslator::translateAlloca(const User &U,
2876 MachineIRBuilder &MIRBuilder) {
2877 auto &AI = cast<AllocaInst>(U);
2878
2879 if (AI.isSwiftError())
2880 return true;
2881
2882 if (AI.isStaticAlloca()) {
2883 Register Res = getOrCreateVReg(AI);
2884 int FI = getOrCreateFrameIndex(AI);
2885 MIRBuilder.buildFrameIndex(Res, FI);
2886 return true;
2887 }
2888
2889 // FIXME: support stack probing for Windows.
2891 return false;
2892
2893 // Now we're in the harder dynamic case.
2894 Register NumElts = getOrCreateVReg(*AI.getArraySize());
2895 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
2896 LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
2897 if (MRI->getType(NumElts) != IntPtrTy) {
2898 Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
2899 MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
2900 NumElts = ExtElts;
2901 }
2902
2903 Type *Ty = AI.getAllocatedType();
2904
2905 Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
2906 Register TySize =
2907 getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
2908 MIRBuilder.buildMul(AllocSize, NumElts, TySize);
2909
2910 // Round the size of the allocation up to the stack alignment size
2911 // by add SA-1 to the size. This doesn't overflow because we're computing
2912 // an address inside an alloca.
2913 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
2914 auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
2915 auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
2917 auto AlignCst =
2918 MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
2919 auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
2920
2921 Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
2922 if (Alignment <= StackAlign)
2923 Alignment = Align(1);
2924 MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
2925
2926 MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
2928 return true;
2929}
2930
2931bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
2932 // FIXME: We may need more info about the type. Because of how LLT works,
2933 // we're completely discarding the i64/double distinction here (amongst
2934 // others). Fortunately the ABIs I know of where that matters don't use va_arg
2935 // anyway but that's not guaranteed.
2936 MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
2937 {getOrCreateVReg(*U.getOperand(0)),
2938 DL->getABITypeAlign(U.getType()).value()});
2939 return true;
2940}
2941
2942bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
2944 return true;
2945
2946 auto &UI = cast<UnreachableInst>(U);
2947 // We may be able to ignore unreachable behind a noreturn call.
2949 const BasicBlock &BB = *UI.getParent();
2950 if (&UI != &BB.front()) {
2952 std::prev(BasicBlock::const_iterator(UI));
2953 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2954 if (Call->doesNotReturn())
2955 return true;
2956 }
2957 }
2958 }
2959
2960 MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>());
2961 return true;
2962}
2963
2964bool IRTranslator::translateInsertElement(const User &U,
2965 MachineIRBuilder &MIRBuilder) {
2966 // If it is a <1 x Ty> vector, use the scalar as it is
2967 // not a legal vector type in LLT.
2968 if (auto *FVT = dyn_cast<FixedVectorType>(U.getType());
2969 FVT && FVT->getNumElements() == 1)
2970 return translateCopy(U, *U.getOperand(1), MIRBuilder);
2971
2972 Register Res = getOrCreateVReg(U);
2973 Register Val = getOrCreateVReg(*U.getOperand(0));
2974 Register Elt = getOrCreateVReg(*U.getOperand(1));
2975 Register Idx = getOrCreateVReg(*U.getOperand(2));
2976 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
2977 return true;
2978}
2979
2980bool IRTranslator::translateExtractElement(const User &U,
2981 MachineIRBuilder &MIRBuilder) {
2982 // If it is a <1 x Ty> vector, use the scalar as it is
2983 // not a legal vector type in LLT.
2984 if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
2985 return translateCopy(U, *U.getOperand(0), MIRBuilder);
2986
2987 Register Res = getOrCreateVReg(U);
2988 Register Val = getOrCreateVReg(*U.getOperand(0));
2989 const auto &TLI = *MF->getSubtarget().getTargetLowering();
2990 unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
2991 Register Idx;
2992 if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
2993 if (CI->getBitWidth() != PreferredVecIdxWidth) {
2994 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
2995 auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
2996 Idx = getOrCreateVReg(*NewIdxCI);
2997 }
2998 }
2999 if (!Idx)
3000 Idx = getOrCreateVReg(*U.getOperand(1));
3001 if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
3002 const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
3003 Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
3004 }
3005 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
3006 return true;
3007}
3008
3009bool IRTranslator::translateShuffleVector(const User &U,
3010 MachineIRBuilder &MIRBuilder) {
3012 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
3013 Mask = SVI->getShuffleMask();
3014 else
3015 Mask = cast<ConstantExpr>(U).getShuffleMask();
3016 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
3017 MIRBuilder
3018 .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
3019 {getOrCreateVReg(*U.getOperand(0)),
3020 getOrCreateVReg(*U.getOperand(1))})
3021 .addShuffleMask(MaskAlloc);
3022 return true;
3023}
3024
3025bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
3026 const PHINode &PI = cast<PHINode>(U);
3027
3029 for (auto Reg : getOrCreateVRegs(PI)) {
3030 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
3031 Insts.push_back(MIB.getInstr());
3032 }
3033
3034 PendingPHIs.emplace_back(&PI, std::move(Insts));
3035 return true;
3036}
3037
3038bool IRTranslator::translateAtomicCmpXchg(const User &U,
3039 MachineIRBuilder &MIRBuilder) {
3040 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
3041
3042 auto &TLI = *MF->getSubtarget().getTargetLowering();
3043 auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
3044
3045 auto Res = getOrCreateVRegs(I);
3046 Register OldValRes = Res[0];
3047 Register SuccessRes = Res[1];
3048 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3049 Register Cmp = getOrCreateVReg(*I.getCompareOperand());
3050 Register NewVal = getOrCreateVReg(*I.getNewValOperand());
3051
3053 OldValRes, SuccessRes, Addr, Cmp, NewVal,
3055 MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
3056 getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3057 I.getSuccessOrdering(), I.getFailureOrdering()));
3058 return true;
3059}
3060
3061bool IRTranslator::translateAtomicRMW(const User &U,
3062 MachineIRBuilder &MIRBuilder) {
3063 const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
3064 auto &TLI = *MF->getSubtarget().getTargetLowering();
3065 auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
3066
3067 Register Res = getOrCreateVReg(I);
3068 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3069 Register Val = getOrCreateVReg(*I.getValOperand());
3070
3071 unsigned Opcode = 0;
3072 switch (I.getOperation()) {
3073 default:
3074 return false;
3076 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3077 break;
3078 case AtomicRMWInst::Add:
3079 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3080 break;
3081 case AtomicRMWInst::Sub:
3082 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3083 break;
3084 case AtomicRMWInst::And:
3085 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3086 break;
3088 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3089 break;
3090 case AtomicRMWInst::Or:
3091 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3092 break;
3093 case AtomicRMWInst::Xor:
3094 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3095 break;
3096 case AtomicRMWInst::Max:
3097 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3098 break;
3099 case AtomicRMWInst::Min:
3100 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3101 break;
3103 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3104 break;
3106 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3107 break;
3109 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3110 break;
3112 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3113 break;
3115 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3116 break;
3118 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3119 break;
3121 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3122 break;
3124 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3125 break;
3126 }
3127
3128 MIRBuilder.buildAtomicRMW(
3129 Opcode, Res, Addr, Val,
3130 *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3131 Flags, MRI->getType(Val), getMemOpAlign(I),
3132 I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3133 I.getOrdering()));
3134 return true;
3135}
3136
3137bool IRTranslator::translateFence(const User &U,
3138 MachineIRBuilder &MIRBuilder) {
3139 const FenceInst &Fence = cast<FenceInst>(U);
3140 MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
3141 Fence.getSyncScopeID());
3142 return true;
3143}
3144
3145bool IRTranslator::translateFreeze(const User &U,
3146 MachineIRBuilder &MIRBuilder) {
3147 const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);
3148 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));
3149
3150 assert(DstRegs.size() == SrcRegs.size() &&
3151 "Freeze with different source and destination type?");
3152
3153 for (unsigned I = 0; I < DstRegs.size(); ++I) {
3154 MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);
3155 }
3156
3157 return true;
3158}
3159
3160void IRTranslator::finishPendingPhis() {
3161#ifndef NDEBUG
3162 DILocationVerifier Verifier;
3163 GISelObserverWrapper WrapperObserver(&Verifier);
3164 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3165#endif // ifndef NDEBUG
3166 for (auto &Phi : PendingPHIs) {
3167 const PHINode *PI = Phi.first;
3168 if (PI->getType()->isEmptyTy())
3169 continue;
3170 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3171 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3172 EntryBuilder->setDebugLoc(PI->getDebugLoc());
3173#ifndef NDEBUG
3174 Verifier.setCurrentInst(PI);
3175#endif // ifndef NDEBUG
3176
3178 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3179 auto IRPred = PI->getIncomingBlock(i);
3180 ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
3181 for (auto *Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
3182 if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
3183 continue;
3184 SeenPreds.insert(Pred);
3185 for (unsigned j = 0; j < ValRegs.size(); ++j) {
3186 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3187 MIB.addUse(ValRegs[j]);
3188 MIB.addMBB(Pred);
3189 }
3190 }
3191 }
3192 }
3193}
3194
3195void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,
3196 const DILocalVariable *Variable,
3197 const DIExpression *Expression,
3198 const DebugLoc &DL,
3199 MachineIRBuilder &MIRBuilder) {
3200 assert(Variable->isValidLocationForIntrinsic(DL) &&
3201 "Expected inlined-at fields to agree");
3202 // Act as if we're handling a debug intrinsic.
3203 MIRBuilder.setDebugLoc(DL);
3204
3205 if (!V || HasArgList) {
3206 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
3207 // terminate any prior location.
3208 MIRBuilder.buildIndirectDbgValue(0, Variable, Expression);
3209 return;
3210 }
3211
3212 if (const auto *CI = dyn_cast<Constant>(V)) {
3213 MIRBuilder.buildConstDbgValue(*CI, Variable, Expression);
3214 return;
3215 }
3216
3217 if (auto *AI = dyn_cast<AllocaInst>(V);
3218 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
3219 // If the value is an alloca and the expression starts with a
3220 // dereference, track a stack slot instead of a register, as registers
3221 // may be clobbered.
3222 auto ExprOperands = Expression->getElements();
3223 auto *ExprDerefRemoved =
3224 DIExpression::get(AI->getContext(), ExprOperands.drop_front());
3225 MIRBuilder.buildFIDbgValue(getOrCreateFrameIndex(*AI), Variable,
3226 ExprDerefRemoved);
3227 return;
3228 }
3229 if (translateIfEntryValueArgument(false, V, Variable, Expression, DL,
3230 MIRBuilder))
3231 return;
3232 for (Register Reg : getOrCreateVRegs(*V)) {
3233 // FIXME: This does not handle register-indirect values at offset 0. The
3234 // direct/indirect thing shouldn't really be handled by something as
3235 // implicit as reg+noreg vs reg+imm in the first place, but it seems
3236 // pretty baked in right now.
3237 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expression);
3238 }
3239 return;
3240}
3241
3242void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,
3243 const DILocalVariable *Variable,
3244 const DIExpression *Expression,
3245 const DebugLoc &DL,
3246 MachineIRBuilder &MIRBuilder) {
3247 if (!Address || isa<UndefValue>(Address)) {
3248 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
3249 return;
3250 }
3251
3252 assert(Variable->isValidLocationForIntrinsic(DL) &&
3253 "Expected inlined-at fields to agree");
3254 auto AI = dyn_cast<AllocaInst>(Address);
3255 if (AI && AI->isStaticAlloca()) {
3256 // Static allocas are tracked at the MF level, no need for DBG_VALUE
3257 // instructions (in fact, they get ignored if they *do* exist).
3258 MF->setVariableDbgInfo(Variable, Expression,
3259 getOrCreateFrameIndex(*AI), DL);
3260 return;
3261 }
3262
3263 if (translateIfEntryValueArgument(true, Address, Variable,
3264 Expression, DL,
3265 MIRBuilder))
3266 return;
3267
3268 // A dbg.declare describes the address of a source variable, so lower it
3269 // into an indirect DBG_VALUE.
3270 MIRBuilder.setDebugLoc(DL);
3271 MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
3272 Variable, Expression);
3273 return;
3274}
3275
3276void IRTranslator::translateDbgInfo(const Instruction &Inst,
3277 MachineIRBuilder &MIRBuilder) {
3278 for (DbgRecord &DR : Inst.getDbgValueRange()) {
3279 if (DPLabel *DPL = dyn_cast<DPLabel>(&DR)) {
3280 MIRBuilder.setDebugLoc(DPL->getDebugLoc());
3281 assert(DPL->getLabel() && "Missing label");
3282 assert(DPL->getLabel()->isValidLocationForIntrinsic(
3283 MIRBuilder.getDebugLoc()) &&
3284 "Expected inlined-at fields to agree");
3285 MIRBuilder.buildDbgLabel(DPL->getLabel());
3286 continue;
3287 }
3288 DPValue &DPV = cast<DPValue>(DR);
3289 const DILocalVariable *Variable = DPV.getVariable();
3290 const DIExpression *Expression = DPV.getExpression();
3291 Value *V = DPV.getVariableLocationOp(0);
3292 if (DPV.isDbgDeclare())
3293 translateDbgDeclareRecord(V, DPV.hasArgList(), Variable,
3294 Expression, DPV.getDebugLoc(), MIRBuilder);
3295 else
3296 translateDbgValueRecord(V, DPV.hasArgList(), Variable,
3297 Expression, DPV.getDebugLoc(), MIRBuilder);
3298 }
3299}
3300
3301bool IRTranslator::translate(const Instruction &Inst) {
3302 CurBuilder->setDebugLoc(Inst.getDebugLoc());
3303 CurBuilder->setPCSections(Inst.getMetadata(LLVMContext::MD_pcsections));
3304
3305 auto &TLI = *MF->getSubtarget().getTargetLowering();
3306 if (TLI.fallBackToDAGISel(Inst))
3307 return false;
3308
3309 switch (Inst.getOpcode()) {
3310#define HANDLE_INST(NUM, OPCODE, CLASS) \
3311 case Instruction::OPCODE: \
3312 return translate##OPCODE(Inst, *CurBuilder.get());
3313#include "llvm/IR/Instruction.def"
3314 default:
3315 return false;
3316 }
3317}
3318
3319bool IRTranslator::translate(const Constant &C, Register Reg) {
3320 // We only emit constants into the entry block from here. To prevent jumpy
3321 // debug behaviour remove debug line.
3322 if (auto CurrInstDL = CurBuilder->getDL())
3323 EntryBuilder->setDebugLoc(DebugLoc());
3324
3325 if (auto CI = dyn_cast<ConstantInt>(&C))
3326 EntryBuilder->buildConstant(Reg, *CI);
3327 else if (auto CF = dyn_cast<ConstantFP>(&C))
3328 EntryBuilder->buildFConstant(Reg, *CF);
3329 else if (isa<UndefValue>(C))
3330 EntryBuilder->buildUndef(Reg);
3331 else if (isa<ConstantPointerNull>(C))
3332 EntryBuilder->buildConstant(Reg, 0);
3333 else if (auto GV = dyn_cast<GlobalValue>(&C))
3334 EntryBuilder->buildGlobalValue(Reg, GV);
3335 else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
3336 if (!isa<FixedVectorType>(CAZ->getType()))
3337 return false;
3338 // Return the scalar if it is a <1 x Ty> vector.
3339 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3340 if (NumElts == 1)
3341 return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder);
3343 for (unsigned I = 0; I < NumElts; ++I) {
3344 Constant &Elt = *CAZ->getElementValue(I);
3345 Ops.push_back(getOrCreateVReg(Elt));
3346 }
3347 EntryBuilder->buildBuildVector(Reg, Ops);
3348 } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
3349 // Return the scalar if it is a <1 x Ty> vector.
3350 if (CV->getNumElements() == 1)
3351 return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder);
3353 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3354 Constant &Elt = *CV->getElementAsConstant(i);
3355 Ops.push_back(getOrCreateVReg(Elt));
3356 }
3357 EntryBuilder->buildBuildVector(Reg, Ops);
3358 } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
3359 switch(CE->getOpcode()) {
3360#define HANDLE_INST(NUM, OPCODE, CLASS) \
3361 case Instruction::OPCODE: \
3362 return translate##OPCODE(*CE, *EntryBuilder.get());
3363#include "llvm/IR/Instruction.def"
3364 default:
3365 return false;
3366 }
3367 } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
3368 if (CV->getNumOperands() == 1)
3369 return translateCopy(C, *CV->getOperand(0), *EntryBuilder);
3371 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3372 Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
3373 }
3374 EntryBuilder->buildBuildVector(Reg, Ops);
3375 } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
3376 EntryBuilder->buildBlockAddress(Reg, BA);
3377 } else
3378 return false;
3379
3380 return true;
3381}
3382
3383bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3385 for (auto &BTB : SL->BitTestCases) {
3386 // Emit header first, if it wasn't already emitted.
3387 if (!BTB.Emitted)
3388 emitBitTestHeader(BTB, BTB.Parent);
3389
3390 BranchProbability UnhandledProb = BTB.Prob;
3391 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3392 UnhandledProb -= BTB.Cases[j].ExtraProb;
3393 // Set the current basic block to the mbb we wish to insert the code into
3394 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3395 // If all cases cover a contiguous range, it is not necessary to jump to
3396 // the default block after the last bit test fails. This is because the
3397 // range check during bit test header creation has guaranteed that every
3398 // case here doesn't go outside the range. In this case, there is no need
3399 // to perform the last bit test, as it will always be true. Instead, make
3400 // the second-to-last bit-test fall through to the target of the last bit
3401 // test, and delete the last bit test.
3402
3403 MachineBasicBlock *NextMBB;
3404 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3405 // Second-to-last bit-test with contiguous range: fall through to the
3406 // target of the final bit test.
3407 NextMBB = BTB.Cases[j + 1].TargetBB;
3408 } else if (j + 1 == ej) {
3409 // For the last bit test, fall through to Default.
3410 NextMBB = BTB.Default;
3411 } else {
3412 // Otherwise, fall through to the next bit test.
3413 NextMBB = BTB.Cases[j + 1].ThisBB;
3414 }
3415
3416 emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
3417
3418 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3419 // We need to record the replacement phi edge here that normally
3420 // happens in emitBitTestCase before we delete the case, otherwise the
3421 // phi edge will be lost.
3422 addMachineCFGPred({BTB.Parent->getBasicBlock(),
3423 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3424 MBB);
3425 // Since we're not going to use the final bit test, remove it.
3426 BTB.Cases.pop_back();
3427 break;
3428 }
3429 }
3430 // This is "default" BB. We have two jumps to it. From "header" BB and from
3431 // last "case" BB, unless the latter was skipped.
3432 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3433 BTB.Default->getBasicBlock()};
3434 addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
3435 if (!BTB.ContiguousRange) {
3436 addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
3437 }
3438 }
3439 SL->BitTestCases.clear();
3440
3441 for (auto &JTCase : SL->JTCases) {
3442 // Emit header first, if it wasn't already emitted.
3443 if (!JTCase.first.Emitted)
3444 emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
3445
3446 emitJumpTable(JTCase.second, JTCase.second.MBB);
3447 }
3448 SL->JTCases.clear();
3449
3450 for (auto &SwCase : SL->SwitchCases)
3451 emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
3452 SL->SwitchCases.clear();
3453
3454 // Check if we need to generate stack-protector guard checks.
3455 StackProtector &SP = getAnalysis<StackProtector>();
3456 if (SP.shouldEmitSDCheck(BB)) {
3457 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3458 bool FunctionBasedInstrumentation =
3460 SPDescriptor.initialize(&BB, &MBB, FunctionBasedInstrumentation);
3461 }
3462 // Handle stack protector.
3463 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3464 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3465 return false;
3466 } else if (SPDescriptor.shouldEmitStackProtector()) {
3467 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3468 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3469
3470 // Find the split point to split the parent mbb. At the same time copy all
3471 // physical registers used in the tail of parent mbb into virtual registers
3472 // before the split point and back into physical registers after the split
3473 // point. This prevents us needing to deal with Live-ins and many other
3474 // register allocation issues caused by us splitting the parent mbb. The
3475 // register allocator will clean up said virtual copies later on.
3477 ParentMBB, *MF->getSubtarget().getInstrInfo());
3478
3479 // Splice the terminator of ParentMBB into SuccessMBB.
3480 SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
3481 ParentMBB->end());
3482
3483 // Add compare/jump on neq/jump to the parent BB.
3484 if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))
3485 return false;
3486
3487 // CodeGen Failure MBB if we have not codegened it yet.
3488 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3489 if (FailureMBB->empty()) {
3490 if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))
3491 return false;
3492 }
3493
3494 // Clear the Per-BB State.
3495 SPDescriptor.resetPerBBState();
3496 }
3497 return true;
3498}
3499
3500bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3501 MachineBasicBlock *ParentBB) {
3502 CurBuilder->setInsertPt(*ParentBB, ParentBB->end());
3503 // First create the loads to the guard/stack slot for the comparison.
3504 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3506 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
3507 LLT PtrMemTy = getLLTForMVT(TLI.getPointerMemTy(*DL));
3508
3509 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3510 int FI = MFI.getStackProtectorIndex();
3511
3512 Register Guard;
3513 Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);
3514 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3515 Align Align = DL->getPrefTypeAlign(PointerType::getUnqual(M.getContext()));
3516
3517 // Generate code to load the content of the guard slot.
3518 Register GuardVal =
3519 CurBuilder
3520 ->buildLoad(PtrMemTy, StackSlotPtr,
3523 .getReg(0);
3524
3525 if (TLI.useStackGuardXorFP()) {
3526 LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3527 return false;
3528 }
3529
3530 // Retrieve guard check function, nullptr if instrumentation is inlined.
3531 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3532 // This path is currently untestable on GlobalISel, since the only platform
3533 // that needs this seems to be Windows, and we fall back on that currently.
3534 // The code still lives here in case that changes.
3535 // Silence warning about unused variable until the code below that uses
3536 // 'GuardCheckFn' is enabled.
3537 (void)GuardCheckFn;
3538 return false;
3539#if 0
3540 // The target provides a guard check function to validate the guard value.
3541 // Generate a call to that function with the content of the guard slot as
3542 // argument.
3543 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3544 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3546 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3547 Flags.setInReg();
3548 CallLowering::ArgInfo GuardArgInfo(
3549 {GuardVal, FnTy->getParamType(0), {Flags}});
3550
3552 Info.OrigArgs.push_back(GuardArgInfo);
3553 Info.CallConv = GuardCheckFn->getCallingConv();
3554 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
3555 Info.OrigRet = {Register(), FnTy->getReturnType()};
3556 if (!CLI->lowerCall(MIRBuilder, Info)) {
3557 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
3558 return false;
3559 }
3560 return true;
3561#endif
3562 }
3563
3564 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3565 // Otherwise, emit a volatile load to retrieve the stack guard value.
3566 if (TLI.useLoadStackGuardNode()) {
3567 Guard =
3569 getStackGuard(Guard, *CurBuilder);
3570 } else {
3571 // TODO: test using android subtarget when we support @llvm.thread.pointer.
3572 const Value *IRGuard = TLI.getSDagStackGuard(M);
3573 Register GuardPtr = getOrCreateVReg(*IRGuard);
3574
3575 Guard = CurBuilder
3576 ->buildLoad(PtrMemTy, GuardPtr,
3580 .getReg(0);
3581 }
3582
3583 // Perform the comparison.
3584 auto Cmp =
3585 CurBuilder->buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Guard, GuardVal);
3586 // If the guard/stackslot do not equal, branch to failure MBB.
3587 CurBuilder->buildBrCond(Cmp, *SPD.getFailureMBB());
3588 // Otherwise branch to success MBB.
3589 CurBuilder->buildBr(*SPD.getSuccessMBB());
3590 return true;
3591}
3592
3593bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
3594 MachineBasicBlock *FailureBB) {
3595 CurBuilder->setInsertPt(*FailureBB, FailureBB->end());
3596 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3597
3598 const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
3599 const char *Name = TLI.getLibcallName(Libcall);
3600
3602 Info.CallConv = TLI.getLibcallCallingConv(Libcall);
3604 Info.OrigRet = {Register(), Type::getVoidTy(MF->getFunction().getContext()),
3605 0};
3606 if (!CLI->lowerCall(*CurBuilder, Info)) {
3607 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
3608 return false;
3609 }
3610
3611 // On PS4/PS5, the "return address" must still be within the calling
3612 // function, even if it's at the very end, so emit an explicit TRAP here.
3613 // WebAssembly needs an unreachable instruction after a non-returning call,
3614 // because the function return type can be different from __stack_chk_fail's
3615 // return type (void).
3616 const TargetMachine &TM = MF->getTarget();
3617 if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) {
3618 LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
3619 return false;
3620 }
3621 return true;
3622}
3623
3624void IRTranslator::finalizeFunction() {
3625 // Release the memory used by the different maps we
3626 // needed during the translation.
3627 PendingPHIs.clear();
3628 VMap.reset();
3629 FrameIndices.clear();
3630 MachinePreds.clear();
3631 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3632 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3633 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
3634 EntryBuilder.reset();
3635 CurBuilder.reset();
3636 FuncInfo.clear();
3637 SPDescriptor.resetPerFunctionState();
3638}
3639
3640/// Returns true if a BasicBlock \p BB within a variadic function contains a
3641/// variadic musttail call.
3642static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3643 if (!IsVarArg)
3644 return false;
3645
3646 // Walk the block backwards, because tail calls usually only appear at the end
3647 // of a block.
3648 return llvm::any_of(llvm::reverse(BB), [](const Instruction &I) {
3649 const auto *CI = dyn_cast<CallInst>(&I);
3650 return CI && CI->isMustTailCall();
3651 });
3652}
3653
3655 MF = &CurMF;
3656 const Function &F = MF->getFunction();
3658 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3659 // Set the CSEConfig and run the analysis.
3660 GISelCSEInfo *CSEInfo = nullptr;
3661 TPC = &getAnalysis<TargetPassConfig>();
3662 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
3664 : TPC->isGISelCSEEnabled();
3665
3666 if (EnableCSE) {
3667 EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3668 CSEInfo = &Wrapper.get(TPC->getCSEConfig());
3669 EntryBuilder->setCSEInfo(CSEInfo);
3670 CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3671 CurBuilder->setCSEInfo(CSEInfo);
3672 } else {
3673 EntryBuilder = std::make_unique<MachineIRBuilder>();
3674 CurBuilder = std::make_unique<MachineIRBuilder>();
3675 }
3676 CLI = MF->getSubtarget().getCallLowering();
3677 CurBuilder->setMF(*MF);
3678 EntryBuilder->setMF(*MF);
3679 MRI = &MF->getRegInfo();
3680 DL = &F.getParent()->getDataLayout();
3681 ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
3682 const TargetMachine &TM = MF->getTarget();
3683 TM.resetTargetOptions(F);
3684 EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
3685 FuncInfo.MF = MF;
3686 if (EnableOpts) {
3687 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3688 FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3689 } else {
3690 AA = nullptr;
3691 FuncInfo.BPI = nullptr;
3692 }
3693
3694 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
3695 MF->getFunction());
3696 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
3697 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
3698
3699 const auto &TLI = *MF->getSubtarget().getTargetLowering();
3700
3701 SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
3702 SL->init(TLI, TM, *DL);
3703
3704
3705
3706 assert(PendingPHIs.empty() && "stale PHIs");
3707
3708 // Targets which want to use big endian can enable it using
3709 // enableBigEndian()
3710 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
3711 // Currently we don't properly handle big endian code.
3712 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3713 F.getSubprogram(), &F.getEntryBlock());
3714 R << "unable to translate in big endian mode";
3715 reportTranslationError(*MF, *TPC, *ORE, R);
3716 }
3717
3718 // Release the per-function state when we return, whether we succeeded or not.
3719 auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
3720
3721 // Setup a separate basic-block for the arguments and constants
3723 MF->push_back(EntryBB);
3724 EntryBuilder->setMBB(*EntryBB);
3725
3726 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3727 SwiftError.setFunction(CurMF);
3728 SwiftError.createEntriesInEntryBlock(DbgLoc);
3729
3730 bool IsVarArg = F.isVarArg();
3731 bool HasMustTailInVarArgFn = false;
3732
3733 // Create all blocks, in IR order, to preserve the layout.
3734 for (const BasicBlock &BB: F) {
3735 auto *&MBB = BBToMBB[&BB];
3736
3737 MBB = MF->CreateMachineBasicBlock(&BB);
3738 MF->push_back(MBB);
3739
3740 if (BB.hasAddressTaken())
3741 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
3742
3743 if (!HasMustTailInVarArgFn)
3744 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
3745 }
3746
3747 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
3748
3749 // Make our arguments/constants entry block fallthrough to the IR entry block.
3750 EntryBB->addSuccessor(&getMBB(F.front()));
3751
3752 if (CLI->fallBackToDAGISel(*MF)) {
3753 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3754 F.getSubprogram(), &F.getEntryBlock());
3755 R << "unable to lower function: " << ore::NV("Prototype", F.getType());
3756 reportTranslationError(*MF, *TPC, *ORE, R);
3757 return false;
3758 }
3759
3760 // Lower the actual args into this basic block.
3761 SmallVector<ArrayRef<Register>, 8> VRegArgs;
3762 for (const Argument &Arg: F.args()) {
3763 if (DL->getTypeStoreSize(Arg.getType()).isZero())
3764 continue; // Don't handle zero sized types.
3765 ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
3766 VRegArgs.push_back(VRegs);
3767
3768 if (Arg.hasSwiftErrorAttr()) {
3769 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
3770 SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
3771 }
3772 }
3773
3774 if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) {
3775 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3776 F.getSubprogram(), &F.getEntryBlock());
3777 R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
3778 reportTranslationError(*MF, *TPC, *ORE, R);
3779 return false;
3780 }
3781
3782 // Need to visit defs before uses when translating instructions.
3783 GISelObserverWrapper WrapperObserver;
3784 if (EnableCSE && CSEInfo)
3785 WrapperObserver.addObserver(CSEInfo);
3786 {
3788#ifndef NDEBUG
3789 DILocationVerifier Verifier;
3790 WrapperObserver.addObserver(&Verifier);
3791#endif // ifndef NDEBUG
3792 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3793 RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
3794 for (const BasicBlock *BB : RPOT) {
3795 MachineBasicBlock &MBB = getMBB(*BB);
3796 // Set the insertion point of all the following translations to
3797 // the end of this basic block.
3798 CurBuilder->setMBB(MBB);
3799 HasTailCall = false;
3800 for (const Instruction &Inst : *BB) {
3801 // If we translated a tail call in the last step, then we know
3802 // everything after the call is either a return, or something that is
3803 // handled by the call itself. (E.g. a lifetime marker or assume
3804 // intrinsic.) In this case, we should stop translating the block and
3805 // move on.
3806 if (HasTailCall)
3807 break;
3808#ifndef NDEBUG
3809 Verifier.setCurrentInst(&Inst);
3810#endif // ifndef NDEBUG
3811
3812 // Translate any debug-info attached to the instruction.
3813 translateDbgInfo(Inst, *CurBuilder.get());
3814
3815 if (translate(Inst))
3816 continue;
3817
3818 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3819 Inst.getDebugLoc(), BB);
3820 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
3821
3822 if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
3823 std::string InstStrStorage;
3824 raw_string_ostream InstStr(InstStrStorage);
3825 InstStr << Inst;
3826
3827 R << ": '" << InstStr.str() << "'";
3828 }
3829
3830 reportTranslationError(*MF, *TPC, *ORE, R);
3831 return false;
3832 }
3833
3834 if (!finalizeBasicBlock(*BB, MBB)) {
3835 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3836 BB->getTerminator()->getDebugLoc(), BB);
3837 R << "unable to translate basic block";
3838 reportTranslationError(*MF, *TPC, *ORE, R);
3839 return false;
3840 }
3841 }
3842#ifndef NDEBUG
3843 WrapperObserver.removeObserver(&Verifier);
3844#endif
3845 }
3846
3847 finishPendingPhis();
3848
3849 SwiftError.propagateVRegs();
3850
3851 // Merge the argument lowering and constants block with its single
3852 // successor, the LLVM-IR entry block. We want the basic block to
3853 // be maximal.
3854 assert(EntryBB->succ_size() == 1 &&
3855 "Custom BB used for lowering should have only one successor");
3856 // Get the successor of the current entry block.
3857 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
3858 assert(NewEntryBB.pred_size() == 1 &&
3859 "LLVM-IR entry block has a predecessor!?");
3860 // Move all the instruction from the current entry block to the
3861 // new entry block.
3862 NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
3863 EntryBB->end());
3864
3865 // Update the live-in information for the new entry block.
3866 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
3867 NewEntryBB.addLiveIn(LiveIn);
3868 NewEntryBB.sortUniqueLiveIns();
3869
3870 // Get rid of the now empty basic block.
3871 EntryBB->removeSuccessor(&NewEntryBB);
3872 MF->remove(EntryBB);
3873 MF->deleteMachineBasicBlock(EntryBB);
3874
3875 assert(&MF->front() == &NewEntryBB &&
3876 "New entry wasn't next in the list of basic block!");
3877
3878 // Initialize stack protector information.
3879 StackProtector &SP = getAnalysis<StackProtector>();
3881
3882 return false;
3883}
unsigned SubReg
#define Success
aarch64 promote const
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Provides analysis for continuously CSEing during GISel passes.
This file implements a version of MachineIRBuilder which CSEs insts within a MachineBasicBlock.
This file describes how to lower LLVM calls to machine code calls.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:301
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
std::string Name
uint64_t Size
This contains common code to allow clients to notify changes to machine instr.
const HexagonInstrInfo * TII
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB)
Returns true if a BasicBlock BB within a variadic function contains a variadic musttail call.
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
static unsigned getConstrainedOpcode(Intrinsic::ID ID)
IRTranslator LLVM IR MI
#define DEBUG_TYPE
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
static bool isValInBlock(const Value *V, const BasicBlock *BB)
static bool isSwiftError(const Value *V)
This file declares the IRTranslator pass.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file describes how to lower LLVM inline asm to machine code INLINEASM.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:81
Implement a low-level type suitable for MachineInstr level instruction selection.
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
This file contains the declarations for metadata subclasses.
uint64_t High
IntegerType * Int32Ty
LLVMContext & Context
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
verify safepoint Safepoint IR Verifier
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
Value * RHS
Value * LHS
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:76
an instruction to allocate memory on the stack
Definition: Instructions.h:59
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:157
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:132
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:107
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:125
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:103
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
An immutable pass that tracks lazily created AssumptionCache objects.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
@ Add
*p = old + v
Definition: Instructions.h:764
@ FAdd
*p = old + v
Definition: Instructions.h:785
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:778
@ Or
*p = old | v
Definition: Instructions.h:772
@ Sub
*p = old - v
Definition: Instructions.h:766
@ And
*p = old & v
Definition: Instructions.h:768
@ Xor
*p = old ^ v
Definition: Instructions.h:774
@ FSub
*p = old - v
Definition: Instructions.h:788
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:800
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:776
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:782
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:796
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:780
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:792
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:804
@ Nand
*p = ~(old & v)
Definition: Instructions.h:770
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
Definition: Attributes.h:826
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:318
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
Definition: BasicBlock.h:639
InstListType::const_iterator const_iterator
Definition: BasicBlock.h:165
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:342
const Instruction & front() const
Definition: BasicBlock.h:452
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:359
const Instruction & back() const
Definition: BasicBlock.h:454
Legacy analysis pass which computes BlockFrequencyInfo.
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Legacy analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1457
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1763
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1696
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1616
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2314
Value * getCalledOperand() const
Definition: InstrTypes.h:1689
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1641
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1622
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1632
unsigned arg_size() const
Definition: InstrTypes.h:1639
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1773
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
bool isMustTailCall() const
bool checkReturnTypeForCallConv(MachineFunction &MF) const
Toplevel function to check the return type based on the target calling convention.
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
Definition: CallLowering.h:543
virtual bool enableBigEndian() const
For targets which want to use big-endian can enable it with enableBigEndian() hook.
Definition: CallLowering.h:591
virtual bool supportSwiftError() const
Definition: CallLowering.h:446
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
Definition: CallLowering.h:511
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const
This hook must be implemented to lower the given call instruction, including argument and return valu...
Definition: CallLowering.h:555
virtual bool fallBackToDAGISel(const MachineFunction &MF) const
Definition: CallLowering.h:529
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:957
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:967
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:984
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:996
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:997
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:990
@ ICMP_EQ
equal
Definition: InstrTypes.h:988
@ ICMP_NE
not equal
Definition: InstrTypes.h:989
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:993
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:969
bool isFPPredicate() const
Definition: InstrTypes.h:1085
bool isIntPredicate() const
Definition: InstrTypes.h:1086
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:204
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:153
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:144
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:417
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Records a position in IR for a source label (DILabel).
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:410
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:720
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:905
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:504
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:672
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
Align getPointerABIAlignment(unsigned AS) const
Layout pointer alignment.
Definition: DataLayout.cpp:742
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
Value * getValue(unsigned OpIdx=0) const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
A debug info location.
Definition: DebugLoc.h:33
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
An instruction for ordering other memory operations.
Definition: Instructions.h:460
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:498
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:487
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:692
BranchProbabilityInfo * BPI
void clear()
clear - Clear out all the function-specific state.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:178
const BasicBlock & getEntryBlock() const
Definition: Function.h:779
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1828
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:674
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1870
const Function & getFunction() const
Definition: Function.h:160
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:235
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:342
The actual analysis pass wrapper.
Definition: CSEInfo.h:222
Simple wrapper that does the following.
Definition: CSEInfo.h:204
The CSE Analysis object.
Definition: CSEInfo.h:69
Abstract class that contains various methods for clients to notify about changes.
Simple wrapper observer that takes several observers, and calls each one for each event.
void removeObserver(GISelChangeObserver *O)
void addObserver(GISelChangeObserver *O)
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:566
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:528
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:278
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
bool isTailCall(const MachineInstr &MI) const override
This instruction compares its operands according to the predicate given to the constructor.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
IRTranslator(CodeGenOptLevel OptLevel=CodeGenOptLevel::None)
static char ID
Definition: IRTranslator.h:68
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Indirect Branch Instruction.
bool lowerInlineAsm(MachineIRBuilder &MIRBuilder, const CallBase &CB, std::function< ArrayRef< Register >(const Value &Val)> GetOrCreateVRegs) const
Lower the given inline asm call instruction GetOrCreateVRegs is a callback to materialize a register ...
This instruction inserts a struct field of array element value into an aggregate value.
iterator_range< simple_ilist< DbgRecord >::iterator > getDbgValueRange() const
Return a range over the DPValues attached to this instruction.
Definition: Instruction.h:82
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:452
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:339
const BasicBlock * getParent() const
Definition: Instruction.h:150
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:357
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1704
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:250
bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
Invoke instruction.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
Definition: LowLevelType.h:206
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr bool isVector() const
Definition: LowLevelType.h:147
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelType.h:49
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:185
constexpr bool isPointer() const
Definition: LowLevelType.h:141
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelType.h:92
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:184
Value * getPointerOperand()
Definition: Instructions.h:280
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:245
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:255
static LocationSize precise(uint64_t Value)
Context object for machine code objects.
Definition: MCContext.h:76
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
unsigned pred_size() const
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
void setAddressTakenIRBlock(BasicBlock *BB)
Set this block to reflect that it corresponds to an IR-level basic block with a BlockAddress.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
void setHasMustTailInVarArgFunc(bool B)
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
ArrayRef< int > allocateShuffleMask(ArrayRef< int > Mask)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
void push_back(MachineBasicBlock *MBB)
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
void deleteMachineBasicBlock(MachineBasicBlock *MBB)
DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
void remove(iterator MBBI)
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
Helper class to build MachineInstr.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPEXT Op.
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, const SrcOp &Src2, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FMA Op0, Op1, Op2.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr,...
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
MachineInstrBuilder buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, Align Alignment)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
MachineInstrBuilder buildFIDbgValue(int FI, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in th...
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
const DebugLoc & getDebugLoc()
Get the current instruction's debug location.
MachineInstrBuilder buildFFrexp(const DstOp &Fract, const DstOp &Exp, const SrcOp &Src, std::optional< unsigned > Flags=std::nullopt)
Build and insert Fract, Exp = G_FFREXP Src.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildPrefetch(const SrcOp &Addr, unsigned RW, unsigned Locality, unsigned CacheType, MachineMemOperand &MMO)
Build and insert G_PREFETCH Addr, RW, Locality, CacheType.
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
Definition: MachineInstr.h:69
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
static uint32_t copyFlagsFromInstruction(const Instruction &I)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:554
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MCContext & getContext() const
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
Representation for a specific memory location.
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
The optimization diagnostic interface.
Diagnostic information for missed-optimization remarks.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
A simple RAII based Delegate installer.
A simple RAII based Observer installer.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Definition: Register.h:110
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents the LLVM 'select' instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
void initialize(const BasicBlock *BB, MachineBasicBlock *MBB, bool FunctionBasedInstrumentation)
Initialize the stack protector descriptor structure for a new basic block.
MachineBasicBlock * getSuccessMBB()
void resetPerBBState()
Reset state that changes when we handle different basic blocks.
void resetPerFunctionState()
Reset state that only changes when we switch functions.
MachineBasicBlock * getFailureMBB()