LLVM 19.0.0git
IRTranslator.cpp
Go to the documentation of this file.
1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallSet.h"
21#include "llvm/Analysis/Loads.h"
52#include "llvm/IR/BasicBlock.h"
53#include "llvm/IR/CFG.h"
54#include "llvm/IR/Constant.h"
55#include "llvm/IR/Constants.h"
56#include "llvm/IR/DataLayout.h"
59#include "llvm/IR/Function.h"
61#include "llvm/IR/InlineAsm.h"
62#include "llvm/IR/InstrTypes.h"
65#include "llvm/IR/Intrinsics.h"
66#include "llvm/IR/IntrinsicsAMDGPU.h"
67#include "llvm/IR/LLVMContext.h"
68#include "llvm/IR/Metadata.h"
70#include "llvm/IR/Statepoint.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
75#include "llvm/MC/MCContext.h"
76#include "llvm/Pass.h"
79#include "llvm/Support/Debug.h"
87#include <algorithm>
88#include <cassert>
89#include <cstdint>
90#include <iterator>
91#include <optional>
92#include <string>
93#include <utility>
94#include <vector>
95
96#define DEBUG_TYPE "irtranslator"
97
98using namespace llvm;
99
100static cl::opt<bool>
101 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
102 cl::desc("Should enable CSE in irtranslator"),
103 cl::Optional, cl::init(false));
104char IRTranslator::ID = 0;
105
106INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
107 false, false)
115
120 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
121
122 // Print the function name explicitly if we don't have a debug location (which
123 // makes the diagnostic less useful) or if we're going to emit a raw error.
124 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
125 R << (" (in function: " + MF.getName() + ")").str();
126
127 if (TPC.isGlobalISelAbortEnabled())
128 report_fatal_error(Twine(R.getMsg()));
129 else
130 ORE.emit(R);
131}
132
134 : MachineFunctionPass(ID), OptLevel(optlevel) {}
135
136#ifndef NDEBUG
137namespace {
138/// Verify that every instruction created has the same DILocation as the
139/// instruction being translated.
140class DILocationVerifier : public GISelChangeObserver {
141 const Instruction *CurrInst = nullptr;
142
143public:
144 DILocationVerifier() = default;
145 ~DILocationVerifier() = default;
146
147 const Instruction *getCurrentInst() const { return CurrInst; }
148 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
149
150 void erasingInstr(MachineInstr &MI) override {}
151 void changingInstr(MachineInstr &MI) override {}
152 void changedInstr(MachineInstr &MI) override {}
153
154 void createdInstr(MachineInstr &MI) override {
155 assert(getCurrentInst() && "Inserted instruction without a current MI");
156
157 // Only print the check message if we're actually checking it.
158#ifndef NDEBUG
159 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
160 << " was copied to " << MI);
161#endif
162 // We allow insts in the entry block to have no debug loc because
163 // they could have originated from constants, and we don't want a jumpy
164 // debug experience.
165 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
166 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
167 (MI.isDebugInstr())) &&
168 "Line info was not transferred to all instructions");
169 }
170};
171} // namespace
172#endif // ifndef NDEBUG
173
174
180 if (OptLevel != CodeGenOptLevel::None) {
183 }
188}
189
191IRTranslator::allocateVRegs(const Value &Val) {
192 auto VRegsIt = VMap.findVRegs(Val);
193 if (VRegsIt != VMap.vregs_end())
194 return *VRegsIt->second;
195 auto *Regs = VMap.getVRegs(Val);
196 auto *Offsets = VMap.getOffsets(Val);
197 SmallVector<LLT, 4> SplitTys;
198 computeValueLLTs(*DL, *Val.getType(), SplitTys,
199 Offsets->empty() ? Offsets : nullptr);
200 for (unsigned i = 0; i < SplitTys.size(); ++i)
201 Regs->push_back(0);
202 return *Regs;
203}
204
205ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
206 auto VRegsIt = VMap.findVRegs(Val);
207 if (VRegsIt != VMap.vregs_end())
208 return *VRegsIt->second;
209
210 if (Val.getType()->isVoidTy())
211 return *VMap.getVRegs(Val);
212
213 // Create entry for this type.
214 auto *VRegs = VMap.getVRegs(Val);
215 auto *Offsets = VMap.getOffsets(Val);
216
217 if (!Val.getType()->isTokenTy())
218 assert(Val.getType()->isSized() &&
219 "Don't know how to create an empty vreg");
220
221 SmallVector<LLT, 4> SplitTys;
222 computeValueLLTs(*DL, *Val.getType(), SplitTys,
223 Offsets->empty() ? Offsets : nullptr);
224
225 if (!isa<Constant>(Val)) {
226 for (auto Ty : SplitTys)
227 VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
228 return *VRegs;
229 }
230
231 if (Val.getType()->isAggregateType()) {
232 // UndefValue, ConstantAggregateZero
233 auto &C = cast<Constant>(Val);
234 unsigned Idx = 0;
235 while (auto Elt = C.getAggregateElement(Idx++)) {
236 auto EltRegs = getOrCreateVRegs(*Elt);
237 llvm::copy(EltRegs, std::back_inserter(*VRegs));
238 }
239 } else {
240 assert(SplitTys.size() == 1 && "unexpectedly split LLT");
241 VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
242 bool Success = translate(cast<Constant>(Val), VRegs->front());
243 if (!Success) {
244 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
246 &MF->getFunction().getEntryBlock());
247 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
248 reportTranslationError(*MF, *TPC, *ORE, R);
249 return *VRegs;
250 }
251 }
252
253 return *VRegs;
254}
255
256int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
257 auto MapEntry = FrameIndices.find(&AI);
258 if (MapEntry != FrameIndices.end())
259 return MapEntry->second;
260
261 uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
262 uint64_t Size =
263 ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
264
265 // Always allocate at least one byte.
266 Size = std::max<uint64_t>(Size, 1u);
267
268 int &FI = FrameIndices[&AI];
269 FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
270 return FI;
271}
272
273Align IRTranslator::getMemOpAlign(const Instruction &I) {
274 if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
275 return SI->getAlign();
276 if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
277 return LI->getAlign();
278 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
279 return AI->getAlign();
280 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
281 return AI->getAlign();
282
283 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
284 R << "unable to translate memop: " << ore::NV("Opcode", &I);
285 reportTranslationError(*MF, *TPC, *ORE, R);
286 return Align(1);
287}
288
289MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
290 MachineBasicBlock *&MBB = BBToMBB[&BB];
291 assert(MBB && "BasicBlock was not encountered before");
292 return *MBB;
293}
294
295void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
296 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
297 MachinePreds[Edge].push_back(NewPred);
298}
299
300bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
301 MachineIRBuilder &MIRBuilder) {
302 // Get or create a virtual register for each value.
303 // Unless the value is a Constant => loadimm cst?
304 // or inline constant each time?
305 // Creation of a virtual register needs to have a size.
306 Register Op0 = getOrCreateVReg(*U.getOperand(0));
307 Register Op1 = getOrCreateVReg(*U.getOperand(1));
308 Register Res = getOrCreateVReg(U);
309 uint32_t Flags = 0;
310 if (isa<Instruction>(U)) {
311 const Instruction &I = cast<Instruction>(U);
313 }
314
315 MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
316 return true;
317}
318
319bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
320 MachineIRBuilder &MIRBuilder) {
321 Register Op0 = getOrCreateVReg(*U.getOperand(0));
322 Register Res = getOrCreateVReg(U);
323 uint32_t Flags = 0;
324 if (isa<Instruction>(U)) {
325 const Instruction &I = cast<Instruction>(U);
327 }
328 MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);
329 return true;
330}
331
332bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
333 return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
334}
335
336bool IRTranslator::translateCompare(const User &U,
337 MachineIRBuilder &MIRBuilder) {
338 auto *CI = dyn_cast<CmpInst>(&U);
339 Register Op0 = getOrCreateVReg(*U.getOperand(0));
340 Register Op1 = getOrCreateVReg(*U.getOperand(1));
341 Register Res = getOrCreateVReg(U);
342 CmpInst::Predicate Pred =
343 CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
344 cast<ConstantExpr>(U).getPredicate());
345 if (CmpInst::isIntPredicate(Pred))
346 MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
347 else if (Pred == CmpInst::FCMP_FALSE)
348 MIRBuilder.buildCopy(
349 Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
350 else if (Pred == CmpInst::FCMP_TRUE)
351 MIRBuilder.buildCopy(
352 Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
353 else {
354 uint32_t Flags = 0;
355 if (CI)
357 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
358 }
359
360 return true;
361}
362
363bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
364 const ReturnInst &RI = cast<ReturnInst>(U);
365 const Value *Ret = RI.getReturnValue();
366 if (Ret && DL->getTypeStoreSize(Ret->getType()).isZero())
367 Ret = nullptr;
368
369 ArrayRef<Register> VRegs;
370 if (Ret)
371 VRegs = getOrCreateVRegs(*Ret);
372
373 Register SwiftErrorVReg = 0;
374 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
375 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
376 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
377 }
378
379 // The target may mess up with the insertion point, but
380 // this is not important as a return is the last instruction
381 // of the block anyway.
382 return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
383}
384
385void IRTranslator::emitBranchForMergedCondition(
387 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
388 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
389 // If the leaf of the tree is a comparison, merge the condition into
390 // the caseblock.
391 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
392 CmpInst::Predicate Condition;
393 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
394 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
395 } else {
396 const FCmpInst *FC = cast<FCmpInst>(Cond);
397 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
398 }
399
400 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),
401 BOp->getOperand(1), nullptr, TBB, FBB, CurBB,
402 CurBuilder->getDebugLoc(), TProb, FProb);
403 SL->SwitchCases.push_back(CB);
404 return;
405 }
406
407 // Create a CaseBlock record representing this branch.
410 Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),
411 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
412 SL->SwitchCases.push_back(CB);
413}
414
415static bool isValInBlock(const Value *V, const BasicBlock *BB) {
416 if (const Instruction *I = dyn_cast<Instruction>(V))
417 return I->getParent() == BB;
418 return true;
419}
420
421void IRTranslator::findMergedConditions(
423 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
425 BranchProbability FProb, bool InvertCond) {
426 using namespace PatternMatch;
427 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
428 "Expected Opc to be AND/OR");
429 // Skip over not part of the tree and remember to invert op and operands at
430 // next level.
431 Value *NotCond;
432 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
433 isValInBlock(NotCond, CurBB->getBasicBlock())) {
434 findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
435 !InvertCond);
436 return;
437 }
438
439 const Instruction *BOp = dyn_cast<Instruction>(Cond);
440 const Value *BOpOp0, *BOpOp1;
441 // Compute the effective opcode for Cond, taking into account whether it needs
442 // to be inverted, e.g.
443 // and (not (or A, B)), C
444 // gets lowered as
445 // and (and (not A, not B), C)
447 if (BOp) {
448 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
449 ? Instruction::And
450 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
451 ? Instruction::Or
453 if (InvertCond) {
454 if (BOpc == Instruction::And)
455 BOpc = Instruction::Or;
456 else if (BOpc == Instruction::Or)
457 BOpc = Instruction::And;
458 }
459 }
460
461 // If this node is not part of the or/and tree, emit it as a branch.
462 // Note that all nodes in the tree should have same opcode.
463 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
464 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
465 !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||
466 !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {
467 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
468 InvertCond);
469 return;
470 }
471
472 // Create TmpBB after CurBB.
473 MachineFunction::iterator BBI(CurBB);
474 MachineBasicBlock *TmpBB =
476 CurBB->getParent()->insert(++BBI, TmpBB);
477
478 if (Opc == Instruction::Or) {
479 // Codegen X | Y as:
480 // BB1:
481 // jmp_if_X TBB
482 // jmp TmpBB
483 // TmpBB:
484 // jmp_if_Y TBB
485 // jmp FBB
486 //
487
488 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
489 // The requirement is that
490 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
491 // = TrueProb for original BB.
492 // Assuming the original probabilities are A and B, one choice is to set
493 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
494 // A/(1+B) and 2B/(1+B). This choice assumes that
495 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
496 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
497 // TmpBB, but the math is more complicated.
498
499 auto NewTrueProb = TProb / 2;
500 auto NewFalseProb = TProb / 2 + FProb;
501 // Emit the LHS condition.
502 findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
503 NewFalseProb, InvertCond);
504
505 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
506 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
507 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
508 // Emit the RHS condition into TmpBB.
509 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
510 Probs[1], InvertCond);
511 } else {
512 assert(Opc == Instruction::And && "Unknown merge op!");
513 // Codegen X & Y as:
514 // BB1:
515 // jmp_if_X TmpBB
516 // jmp FBB
517 // TmpBB:
518 // jmp_if_Y TBB
519 // jmp FBB
520 //
521 // This requires creation of TmpBB after CurBB.
522
523 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
524 // The requirement is that
525 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
526 // = FalseProb for original BB.
527 // Assuming the original probabilities are A and B, one choice is to set
528 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
529 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
530 // TrueProb for BB1 * FalseProb for TmpBB.
531
532 auto NewTrueProb = TProb + FProb / 2;
533 auto NewFalseProb = FProb / 2;
534 // Emit the LHS condition.
535 findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
536 NewFalseProb, InvertCond);
537
538 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
539 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
540 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
541 // Emit the RHS condition into TmpBB.
542 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
543 Probs[1], InvertCond);
544 }
545}
546
547bool IRTranslator::shouldEmitAsBranches(
548 const std::vector<SwitchCG::CaseBlock> &Cases) {
549 // For multiple cases, it's better to emit as branches.
550 if (Cases.size() != 2)
551 return true;
552
553 // If this is two comparisons of the same values or'd or and'd together, they
554 // will get folded into a single comparison, so don't emit two blocks.
555 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
556 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
557 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
558 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
559 return false;
560 }
561
562 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
563 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
564 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
565 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
566 isa<Constant>(Cases[0].CmpRHS) &&
567 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
568 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
569 Cases[0].TrueBB == Cases[1].ThisBB)
570 return false;
571 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
572 Cases[0].FalseBB == Cases[1].ThisBB)
573 return false;
574 }
575
576 return true;
577}
578
579bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
580 const BranchInst &BrInst = cast<BranchInst>(U);
581 auto &CurMBB = MIRBuilder.getMBB();
582 auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));
583
584 if (BrInst.isUnconditional()) {
585 // If the unconditional target is the layout successor, fallthrough.
586 if (OptLevel == CodeGenOptLevel::None ||
587 !CurMBB.isLayoutSuccessor(Succ0MBB))
588 MIRBuilder.buildBr(*Succ0MBB);
589
590 // Link successors.
591 for (const BasicBlock *Succ : successors(&BrInst))
592 CurMBB.addSuccessor(&getMBB(*Succ));
593 return true;
594 }
595
596 // If this condition is one of the special cases we handle, do special stuff
597 // now.
598 const Value *CondVal = BrInst.getCondition();
599 MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));
600
601 // If this is a series of conditions that are or'd or and'd together, emit
602 // this as a sequence of branches instead of setcc's with and/or operations.
603 // As long as jumps are not expensive (exceptions for multi-use logic ops,
604 // unpredictable branches, and vector extracts because those jumps are likely
605 // expensive for any target), this should improve performance.
606 // For example, instead of something like:
607 // cmp A, B
608 // C = seteq
609 // cmp D, E
610 // F = setle
611 // or C, F
612 // jnz foo
613 // Emit:
614 // cmp A, B
615 // je foo
616 // cmp D, E
617 // jle foo
618 using namespace PatternMatch;
619 const Instruction *CondI = dyn_cast<Instruction>(CondVal);
620 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
621 !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {
623 Value *Vec;
624 const Value *BOp0, *BOp1;
625 if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
626 Opcode = Instruction::And;
627 else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
628 Opcode = Instruction::Or;
629
630 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
631 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
632 findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
633 getEdgeProbability(&CurMBB, Succ0MBB),
634 getEdgeProbability(&CurMBB, Succ1MBB),
635 /*InvertCond=*/false);
636 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
637
638 // Allow some cases to be rejected.
639 if (shouldEmitAsBranches(SL->SwitchCases)) {
640 // Emit the branch for this block.
641 emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
642 SL->SwitchCases.erase(SL->SwitchCases.begin());
643 return true;
644 }
645
646 // Okay, we decided not to do this, remove any inserted MBB's and clear
647 // SwitchCases.
648 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
649 MF->erase(SL->SwitchCases[I].ThisBB);
650
651 SL->SwitchCases.clear();
652 }
653 }
654
655 // Create a CaseBlock record representing this branch.
656 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
658 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
659 CurBuilder->getDebugLoc());
660
661 // Use emitSwitchCase to actually insert the fast branch sequence for this
662 // cond branch.
663 emitSwitchCase(CB, &CurMBB, *CurBuilder);
664 return true;
665}
666
667void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
669 BranchProbability Prob) {
670 if (!FuncInfo.BPI) {
671 Src->addSuccessorWithoutProb(Dst);
672 return;
673 }
674 if (Prob.isUnknown())
675 Prob = getEdgeProbability(Src, Dst);
676 Src->addSuccessor(Dst, Prob);
677}
678
680IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
681 const MachineBasicBlock *Dst) const {
682 const BasicBlock *SrcBB = Src->getBasicBlock();
683 const BasicBlock *DstBB = Dst->getBasicBlock();
684 if (!FuncInfo.BPI) {
685 // If BPI is not available, set the default probability as 1 / N, where N is
686 // the number of successors.
687 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
688 return BranchProbability(1, SuccSize);
689 }
690 return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
691}
692
693bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
694 using namespace SwitchCG;
695 // Extract cases from the switch.
696 const SwitchInst &SI = cast<SwitchInst>(U);
697 BranchProbabilityInfo *BPI = FuncInfo.BPI;
698 CaseClusterVector Clusters;
699 Clusters.reserve(SI.getNumCases());
700 for (const auto &I : SI.cases()) {
701 MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
702 assert(Succ && "Could not find successor mbb in mapping");
703 const ConstantInt *CaseVal = I.getCaseValue();
704 BranchProbability Prob =
705 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
706 : BranchProbability(1, SI.getNumCases() + 1);
707 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
708 }
709
710 MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
711
712 // Cluster adjacent cases with the same destination. We do this at all
713 // optimization levels because it's cheap to do and will make codegen faster
714 // if there are many clusters.
715 sortAndRangeify(Clusters);
716
717 MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
718
719 // If there is only the default destination, jump there directly.
720 if (Clusters.empty()) {
721 SwitchMBB->addSuccessor(DefaultMBB);
722 if (DefaultMBB != SwitchMBB->getNextNode())
723 MIB.buildBr(*DefaultMBB);
724 return true;
725 }
726
727 SL->findJumpTables(Clusters, &SI, std::nullopt, DefaultMBB, nullptr, nullptr);
728 SL->findBitTestClusters(Clusters, &SI);
729
730 LLVM_DEBUG({
731 dbgs() << "Case clusters: ";
732 for (const CaseCluster &C : Clusters) {
733 if (C.Kind == CC_JumpTable)
734 dbgs() << "JT:";
735 if (C.Kind == CC_BitTests)
736 dbgs() << "BT:";
737
738 C.Low->getValue().print(dbgs(), true);
739 if (C.Low != C.High) {
740 dbgs() << '-';
741 C.High->getValue().print(dbgs(), true);
742 }
743 dbgs() << ' ';
744 }
745 dbgs() << '\n';
746 });
747
748 assert(!Clusters.empty());
749 SwitchWorkList WorkList;
750 CaseClusterIt First = Clusters.begin();
751 CaseClusterIt Last = Clusters.end() - 1;
752 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
753 WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
754
755 while (!WorkList.empty()) {
756 SwitchWorkListItem W = WorkList.pop_back_val();
757
758 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
759 // For optimized builds, lower large range as a balanced binary tree.
760 if (NumClusters > 3 &&
762 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
763 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB, MIB);
764 continue;
765 }
766
767 if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
768 return false;
769 }
770 return true;
771}
772
773void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
775 Value *Cond, MachineBasicBlock *SwitchMBB,
776 MachineIRBuilder &MIB) {
777 using namespace SwitchCG;
778 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
779 "Clusters not sorted?");
780 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
781
782 auto [LastLeft, FirstRight, LeftProb, RightProb] =
783 SL->computeSplitWorkItemInfo(W);
784
785 // Use the first element on the right as pivot since we will make less-than
786 // comparisons against it.
787 CaseClusterIt PivotCluster = FirstRight;
788 assert(PivotCluster > W.FirstCluster);
789 assert(PivotCluster <= W.LastCluster);
790
791 CaseClusterIt FirstLeft = W.FirstCluster;
792 CaseClusterIt LastRight = W.LastCluster;
793
794 const ConstantInt *Pivot = PivotCluster->Low;
795
796 // New blocks will be inserted immediately after the current one.
798 ++BBI;
799
800 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
801 // we can branch to its destination directly if it's squeezed exactly in
802 // between the known lower bound and Pivot - 1.
803 MachineBasicBlock *LeftMBB;
804 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
805 FirstLeft->Low == W.GE &&
806 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
807 LeftMBB = FirstLeft->MBB;
808 } else {
809 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
810 FuncInfo.MF->insert(BBI, LeftMBB);
811 WorkList.push_back(
812 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
813 }
814
815 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
816 // single cluster, RHS.Low == Pivot, and we can branch to its destination
817 // directly if RHS.High equals the current upper bound.
818 MachineBasicBlock *RightMBB;
819 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
820 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
821 RightMBB = FirstRight->MBB;
822 } else {
823 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
824 FuncInfo.MF->insert(BBI, RightMBB);
825 WorkList.push_back(
826 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
827 }
828
829 // Create the CaseBlock record that will be used to lower the branch.
830 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
831 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
832 RightProb);
833
834 if (W.MBB == SwitchMBB)
835 emitSwitchCase(CB, SwitchMBB, MIB);
836 else
837 SL->SwitchCases.push_back(CB);
838}
839
840void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
842 // Emit the code for the jump table
843 assert(JT.Reg != -1U && "Should lower JT Header first!");
845 MIB.setMBB(*MBB);
846 MIB.setDebugLoc(CurBuilder->getDebugLoc());
847
849 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
850
851 auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
852 MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
853}
854
855bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
857 MachineBasicBlock *HeaderBB) {
858 MachineIRBuilder MIB(*HeaderBB->getParent());
859 MIB.setMBB(*HeaderBB);
860 MIB.setDebugLoc(CurBuilder->getDebugLoc());
861
862 const Value &SValue = *JTH.SValue;
863 // Subtract the lowest switch case value from the value being switched on.
864 const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
865 Register SwitchOpReg = getOrCreateVReg(SValue);
866 auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
867 auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
868
869 // This value may be smaller or larger than the target's pointer type, and
870 // therefore require extension or truncating.
871 auto *PtrIRTy = PointerType::getUnqual(SValue.getContext());
872 const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
873 Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
874
875 JT.Reg = Sub.getReg(0);
876
877 if (JTH.FallthroughUnreachable) {
878 if (JT.MBB != HeaderBB->getNextNode())
879 MIB.buildBr(*JT.MBB);
880 return true;
881 }
882
883 // Emit the range check for the jump table, and branch to the default block
884 // for the switch statement if the value being switched on exceeds the
885 // largest case in the switch.
886 auto Cst = getOrCreateVReg(
887 *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
888 Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
889 auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
890
891 auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
892
893 // Avoid emitting unnecessary branches to the next block.
894 if (JT.MBB != HeaderBB->getNextNode())
895 BrCond = MIB.buildBr(*JT.MBB);
896 return true;
897}
898
899void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
900 MachineBasicBlock *SwitchBB,
901 MachineIRBuilder &MIB) {
902 Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
904 DebugLoc OldDbgLoc = MIB.getDebugLoc();
905 MIB.setDebugLoc(CB.DbgLoc);
906 MIB.setMBB(*CB.ThisBB);
907
908 if (CB.PredInfo.NoCmp) {
909 // Branch or fall through to TrueBB.
910 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
911 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
912 CB.ThisBB);
914 if (CB.TrueBB != CB.ThisBB->getNextNode())
915 MIB.buildBr(*CB.TrueBB);
916 MIB.setDebugLoc(OldDbgLoc);
917 return;
918 }
919
920 const LLT i1Ty = LLT::scalar(1);
921 // Build the compare.
922 if (!CB.CmpMHS) {
923 const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);
924 // For conditional branch lowering, we might try to do something silly like
925 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
926 // just re-use the existing condition vreg.
927 if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
929 Cond = CondLHS;
930 } else {
931 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
933 Cond =
934 MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
935 else
936 Cond =
937 MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
938 }
939 } else {
941 "Can only handle SLE ranges");
942
943 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
944 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
945
946 Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
947 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
948 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
949 Cond =
950 MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
951 } else {
952 const LLT CmpTy = MRI->getType(CmpOpReg);
953 auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
954 auto Diff = MIB.buildConstant(CmpTy, High - Low);
955 Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
956 }
957 }
958
959 // Update successor info
960 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
961
962 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
963 CB.ThisBB);
964
965 // TrueBB and FalseBB are always different unless the incoming IR is
966 // degenerate. This only happens when running llc on weird IR.
967 if (CB.TrueBB != CB.FalseBB)
968 addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
970
971 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
972 CB.ThisBB);
973
974 MIB.buildBrCond(Cond, *CB.TrueBB);
975 MIB.buildBr(*CB.FalseBB);
976 MIB.setDebugLoc(OldDbgLoc);
977}
978
979bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
980 MachineBasicBlock *SwitchMBB,
981 MachineBasicBlock *CurMBB,
982 MachineBasicBlock *DefaultMBB,
983 MachineIRBuilder &MIB,
985 BranchProbability UnhandledProbs,
987 MachineBasicBlock *Fallthrough,
988 bool FallthroughUnreachable) {
989 using namespace SwitchCG;
990 MachineFunction *CurMF = SwitchMBB->getParent();
991 // FIXME: Optimize away range check based on pivot comparisons.
992 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
993 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
994 BranchProbability DefaultProb = W.DefaultProb;
995
996 // The jump block hasn't been inserted yet; insert it here.
997 MachineBasicBlock *JumpMBB = JT->MBB;
998 CurMF->insert(BBI, JumpMBB);
999
1000 // Since the jump table block is separate from the switch block, we need
1001 // to keep track of it as a machine predecessor to the default block,
1002 // otherwise we lose the phi edges.
1003 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1004 CurMBB);
1005 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1006 JumpMBB);
1007
1008 auto JumpProb = I->Prob;
1009 auto FallthroughProb = UnhandledProbs;
1010
1011 // If the default statement is a target of the jump table, we evenly
1012 // distribute the default probability to successors of CurMBB. Also
1013 // update the probability on the edge from JumpMBB to Fallthrough.
1014 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1015 SE = JumpMBB->succ_end();
1016 SI != SE; ++SI) {
1017 if (*SI == DefaultMBB) {
1018 JumpProb += DefaultProb / 2;
1019 FallthroughProb -= DefaultProb / 2;
1020 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
1021 JumpMBB->normalizeSuccProbs();
1022 } else {
1023 // Also record edges from the jump table block to it's successors.
1024 addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1025 JumpMBB);
1026 }
1027 }
1028
1029 if (FallthroughUnreachable)
1030 JTH->FallthroughUnreachable = true;
1031
1032 if (!JTH->FallthroughUnreachable)
1033 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
1034 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
1035 CurMBB->normalizeSuccProbs();
1036
1037 // The jump table header will be inserted in our current block, do the
1038 // range check, and fall through to our fallthrough block.
1039 JTH->HeaderBB = CurMBB;
1040 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1041
1042 // If we're in the right place, emit the jump table header right now.
1043 if (CurMBB == SwitchMBB) {
1044 if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
1045 return false;
1046 JTH->Emitted = true;
1047 }
1048 return true;
1049}
1050bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
1051 Value *Cond,
1052 MachineBasicBlock *Fallthrough,
1053 bool FallthroughUnreachable,
1054 BranchProbability UnhandledProbs,
1055 MachineBasicBlock *CurMBB,
1056 MachineIRBuilder &MIB,
1057 MachineBasicBlock *SwitchMBB) {
1058 using namespace SwitchCG;
1059 const Value *RHS, *LHS, *MHS;
1060 CmpInst::Predicate Pred;
1061 if (I->Low == I->High) {
1062 // Check Cond == I->Low.
1063 Pred = CmpInst::ICMP_EQ;
1064 LHS = Cond;
1065 RHS = I->Low;
1066 MHS = nullptr;
1067 } else {
1068 // Check I->Low <= Cond <= I->High.
1069 Pred = CmpInst::ICMP_SLE;
1070 LHS = I->Low;
1071 MHS = Cond;
1072 RHS = I->High;
1073 }
1074
1075 // If Fallthrough is unreachable, fold away the comparison.
1076 // The false probability is the sum of all unhandled cases.
1077 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1078 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1079
1080 emitSwitchCase(CB, SwitchMBB, MIB);
1081 return true;
1082}
1083
1084void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1085 MachineBasicBlock *SwitchBB) {
1086 MachineIRBuilder &MIB = *CurBuilder;
1087 MIB.setMBB(*SwitchBB);
1088
1089 // Subtract the minimum value.
1090 Register SwitchOpReg = getOrCreateVReg(*B.SValue);
1091
1092 LLT SwitchOpTy = MRI->getType(SwitchOpReg);
1093 Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);
1094 auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1095
1097 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1098
1099 LLT MaskTy = SwitchOpTy;
1100 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1101 !llvm::has_single_bit<uint32_t>(MaskTy.getSizeInBits()))
1102 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1103 else {
1104 // Ensure that the type will fit the mask value.
1105 for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1106 if (!isUIntN(SwitchOpTy.getSizeInBits(), B.Cases[I].Mask)) {
1107 // Switch table case range are encoded into series of masks.
1108 // Just use pointer type, it's guaranteed to fit.
1109 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1110 break;
1111 }
1112 }
1113 }
1114 Register SubReg = RangeSub.getReg(0);
1115 if (SwitchOpTy != MaskTy)
1116 SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);
1117
1118 B.RegVT = getMVTForLLT(MaskTy);
1119 B.Reg = SubReg;
1120
1121 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1122
1123 if (!B.FallthroughUnreachable)
1124 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
1125 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
1126
1127 SwitchBB->normalizeSuccProbs();
1128
1129 if (!B.FallthroughUnreachable) {
1130 // Conditional branch to the default block.
1131 auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);
1132 auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),
1133 RangeSub, RangeCst);
1134 MIB.buildBrCond(RangeCmp, *B.Default);
1135 }
1136
1137 // Avoid emitting unnecessary branches to the next block.
1138 if (MBB != SwitchBB->getNextNode())
1139 MIB.buildBr(*MBB);
1140}
1141
1142void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1143 MachineBasicBlock *NextMBB,
1144 BranchProbability BranchProbToNext,
1146 MachineBasicBlock *SwitchBB) {
1147 MachineIRBuilder &MIB = *CurBuilder;
1148 MIB.setMBB(*SwitchBB);
1149
1150 LLT SwitchTy = getLLTForMVT(BB.RegVT);
1151 Register Cmp;
1152 unsigned PopCount = llvm::popcount(B.Mask);
1153 if (PopCount == 1) {
1154 // Testing for a single bit; just compare the shift count with what it
1155 // would need to be to shift a 1 bit in that position.
1156 auto MaskTrailingZeros =
1157 MIB.buildConstant(SwitchTy, llvm::countr_zero(B.Mask));
1158 Cmp =
1159 MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
1160 .getReg(0);
1161 } else if (PopCount == BB.Range) {
1162 // There is only one zero bit in the range, test for it directly.
1163 auto MaskTrailingOnes =
1164 MIB.buildConstant(SwitchTy, llvm::countr_one(B.Mask));
1165 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
1166 .getReg(0);
1167 } else {
1168 // Make desired shift.
1169 auto CstOne = MIB.buildConstant(SwitchTy, 1);
1170 auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);
1171
1172 // Emit bit tests and jumps.
1173 auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);
1174 auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);
1175 auto CstZero = MIB.buildConstant(SwitchTy, 0);
1176 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)
1177 .getReg(0);
1178 }
1179
1180 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1181 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
1182 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1183 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1184 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1185 // one as they are relative probabilities (and thus work more like weights),
1186 // and hence we need to normalize them to let the sum of them become one.
1187 SwitchBB->normalizeSuccProbs();
1188
1189 // Record the fact that the IR edge from the header to the bit test target
1190 // will go through our new block. Neeeded for PHIs to have nodes added.
1191 addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1192 SwitchBB);
1193
1194 MIB.buildBrCond(Cmp, *B.TargetBB);
1195
1196 // Avoid emitting unnecessary branches to the next block.
1197 if (NextMBB != SwitchBB->getNextNode())
1198 MIB.buildBr(*NextMBB);
1199}
1200
1201bool IRTranslator::lowerBitTestWorkItem(
1203 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1205 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1207 bool FallthroughUnreachable) {
1208 using namespace SwitchCG;
1209 MachineFunction *CurMF = SwitchMBB->getParent();
1210 // FIXME: Optimize away range check based on pivot comparisons.
1211 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1212 // The bit test blocks haven't been inserted yet; insert them here.
1213 for (BitTestCase &BTC : BTB->Cases)
1214 CurMF->insert(BBI, BTC.ThisBB);
1215
1216 // Fill in fields of the BitTestBlock.
1217 BTB->Parent = CurMBB;
1218 BTB->Default = Fallthrough;
1219
1220 BTB->DefaultProb = UnhandledProbs;
1221 // If the cases in bit test don't form a contiguous range, we evenly
1222 // distribute the probability on the edge to Fallthrough to two
1223 // successors of CurMBB.
1224 if (!BTB->ContiguousRange) {
1225 BTB->Prob += DefaultProb / 2;
1226 BTB->DefaultProb -= DefaultProb / 2;
1227 }
1228
1229 if (FallthroughUnreachable)
1230 BTB->FallthroughUnreachable = true;
1231
1232 // If we're in the right place, emit the bit test header right now.
1233 if (CurMBB == SwitchMBB) {
1234 emitBitTestHeader(*BTB, SwitchMBB);
1235 BTB->Emitted = true;
1236 }
1237 return true;
1238}
1239
1240bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1241 Value *Cond,
1242 MachineBasicBlock *SwitchMBB,
1243 MachineBasicBlock *DefaultMBB,
1244 MachineIRBuilder &MIB) {
1245 using namespace SwitchCG;
1246 MachineFunction *CurMF = FuncInfo.MF;
1247 MachineBasicBlock *NextMBB = nullptr;
1249 if (++BBI != FuncInfo.MF->end())
1250 NextMBB = &*BBI;
1251
1252 if (EnableOpts) {
1253 // Here, we order cases by probability so the most likely case will be
1254 // checked first. However, two clusters can have the same probability in
1255 // which case their relative ordering is non-deterministic. So we use Low
1256 // as a tie-breaker as clusters are guaranteed to never overlap.
1257 llvm::sort(W.FirstCluster, W.LastCluster + 1,
1258 [](const CaseCluster &a, const CaseCluster &b) {
1259 return a.Prob != b.Prob
1260 ? a.Prob > b.Prob
1261 : a.Low->getValue().slt(b.Low->getValue());
1262 });
1263
1264 // Rearrange the case blocks so that the last one falls through if possible
1265 // without changing the order of probabilities.
1266 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1267 --I;
1268 if (I->Prob > W.LastCluster->Prob)
1269 break;
1270 if (I->Kind == CC_Range && I->MBB == NextMBB) {
1271 std::swap(*I, *W.LastCluster);
1272 break;
1273 }
1274 }
1275 }
1276
1277 // Compute total probability.
1278 BranchProbability DefaultProb = W.DefaultProb;
1279 BranchProbability UnhandledProbs = DefaultProb;
1280 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1281 UnhandledProbs += I->Prob;
1282
1283 MachineBasicBlock *CurMBB = W.MBB;
1284 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1285 bool FallthroughUnreachable = false;
1286 MachineBasicBlock *Fallthrough;
1287 if (I == W.LastCluster) {
1288 // For the last cluster, fall through to the default destination.
1289 Fallthrough = DefaultMBB;
1290 FallthroughUnreachable = isa<UnreachableInst>(
1291 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1292 } else {
1293 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
1294 CurMF->insert(BBI, Fallthrough);
1295 }
1296 UnhandledProbs -= I->Prob;
1297
1298 switch (I->Kind) {
1299 case CC_BitTests: {
1300 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1301 DefaultProb, UnhandledProbs, I, Fallthrough,
1302 FallthroughUnreachable)) {
1303 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1304 return false;
1305 }
1306 break;
1307 }
1308
1309 case CC_JumpTable: {
1310 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1311 UnhandledProbs, I, Fallthrough,
1312 FallthroughUnreachable)) {
1313 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1314 return false;
1315 }
1316 break;
1317 }
1318 case CC_Range: {
1319 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1320 FallthroughUnreachable, UnhandledProbs,
1321 CurMBB, MIB, SwitchMBB)) {
1322 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1323 return false;
1324 }
1325 break;
1326 }
1327 }
1328 CurMBB = Fallthrough;
1329 }
1330
1331 return true;
1332}
1333
1334bool IRTranslator::translateIndirectBr(const User &U,
1335 MachineIRBuilder &MIRBuilder) {
1336 const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
1337
1338 const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
1339 MIRBuilder.buildBrIndirect(Tgt);
1340
1341 // Link successors.
1343 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1344 for (const BasicBlock *Succ : successors(&BrInst)) {
1345 // It's legal for indirectbr instructions to have duplicate blocks in the
1346 // destination list. We don't allow this in MIR. Skip anything that's
1347 // already a successor.
1348 if (!AddedSuccessors.insert(Succ).second)
1349 continue;
1350 CurBB.addSuccessor(&getMBB(*Succ));
1351 }
1352
1353 return true;
1354}
1355
1356static bool isSwiftError(const Value *V) {
1357 if (auto Arg = dyn_cast<Argument>(V))
1358 return Arg->hasSwiftErrorAttr();
1359 if (auto AI = dyn_cast<AllocaInst>(V))
1360 return AI->isSwiftError();
1361 return false;
1362}
1363
1364bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1365 const LoadInst &LI = cast<LoadInst>(U);
1366
1367 unsigned StoreSize = DL->getTypeStoreSize(LI.getType());
1368 if (StoreSize == 0)
1369 return true;
1370
1371 ArrayRef<Register> Regs = getOrCreateVRegs(LI);
1372 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
1373 Register Base = getOrCreateVReg(*LI.getPointerOperand());
1374 AAMDNodes AAInfo = LI.getAAMetadata();
1375
1376 const Value *Ptr = LI.getPointerOperand();
1377 Type *OffsetIRTy = DL->getIndexType(Ptr->getType());
1378 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1379
1380 if (CLI->supportSwiftError() && isSwiftError(Ptr)) {
1381 assert(Regs.size() == 1 && "swifterror should be single pointer");
1382 Register VReg =
1383 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1384 MIRBuilder.buildCopy(Regs[0], VReg);
1385 return true;
1386 }
1387
1389 TLI->getLoadMemOperandFlags(LI, *DL, AC, LibInfo);
1390 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1391 if (AA->pointsToConstantMemory(
1392 MemoryLocation(Ptr, LocationSize::precise(StoreSize), AAInfo))) {
1394 }
1395 }
1396
1397 const MDNode *Ranges =
1398 Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
1399 for (unsigned i = 0; i < Regs.size(); ++i) {
1400 Register Addr;
1401 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1402
1403 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
1404 Align BaseAlign = getMemOpAlign(LI);
1405 auto MMO = MF->getMachineMemOperand(
1406 Ptr, Flags, MRI->getType(Regs[i]),
1407 commonAlignment(BaseAlign, Offsets[i] / 8), AAInfo, Ranges,
1408 LI.getSyncScopeID(), LI.getOrdering());
1409 MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
1410 }
1411
1412 return true;
1413}
1414
1415bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1416 const StoreInst &SI = cast<StoreInst>(U);
1417 if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
1418 return true;
1419
1420 ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
1421 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
1422 Register Base = getOrCreateVReg(*SI.getPointerOperand());
1423
1424 Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());
1425 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1426
1427 if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
1428 assert(Vals.size() == 1 && "swifterror should be single pointer");
1429
1430 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1431 SI.getPointerOperand());
1432 MIRBuilder.buildCopy(VReg, Vals[0]);
1433 return true;
1434 }
1435
1437
1438 for (unsigned i = 0; i < Vals.size(); ++i) {
1439 Register Addr;
1440 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1441
1442 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1443 Align BaseAlign = getMemOpAlign(SI);
1444 auto MMO = MF->getMachineMemOperand(
1445 Ptr, Flags, MRI->getType(Vals[i]),
1446 commonAlignment(BaseAlign, Offsets[i] / 8), SI.getAAMetadata(), nullptr,
1447 SI.getSyncScopeID(), SI.getOrdering());
1448 MIRBuilder.buildStore(Vals[i], Addr, *MMO);
1449 }
1450 return true;
1451}
1452
1454 const Value *Src = U.getOperand(0);
1455 Type *Int32Ty = Type::getInt32Ty(U.getContext());
1456
1457 // getIndexedOffsetInType is designed for GEPs, so the first index is the
1458 // usual array element rather than looking into the actual aggregate.
1460 Indices.push_back(ConstantInt::get(Int32Ty, 0));
1461
1462 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
1463 for (auto Idx : EVI->indices())
1464 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1465 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1466 for (auto Idx : IVI->indices())
1467 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1468 } else {
1469 for (unsigned i = 1; i < U.getNumOperands(); ++i)
1470 Indices.push_back(U.getOperand(i));
1471 }
1472
1473 return 8 * static_cast<uint64_t>(
1474 DL.getIndexedOffsetInType(Src->getType(), Indices));
1475}
1476
1477bool IRTranslator::translateExtractValue(const User &U,
1478 MachineIRBuilder &MIRBuilder) {
1479 const Value *Src = U.getOperand(0);
1481 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1482 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
1483 unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
1484 auto &DstRegs = allocateVRegs(U);
1485
1486 for (unsigned i = 0; i < DstRegs.size(); ++i)
1487 DstRegs[i] = SrcRegs[Idx++];
1488
1489 return true;
1490}
1491
1492bool IRTranslator::translateInsertValue(const User &U,
1493 MachineIRBuilder &MIRBuilder) {
1494 const Value *Src = U.getOperand(0);
1496 auto &DstRegs = allocateVRegs(U);
1497 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
1498 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1499 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
1500 auto *InsertedIt = InsertedRegs.begin();
1501
1502 for (unsigned i = 0; i < DstRegs.size(); ++i) {
1503 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1504 DstRegs[i] = *InsertedIt++;
1505 else
1506 DstRegs[i] = SrcRegs[i];
1507 }
1508
1509 return true;
1510}
1511
1512bool IRTranslator::translateSelect(const User &U,
1513 MachineIRBuilder &MIRBuilder) {
1514 Register Tst = getOrCreateVReg(*U.getOperand(0));
1515 ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1516 ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1517 ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1518
1519 uint32_t Flags = 0;
1520 if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
1522
1523 for (unsigned i = 0; i < ResRegs.size(); ++i) {
1524 MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
1525 }
1526
1527 return true;
1528}
1529
1530bool IRTranslator::translateCopy(const User &U, const Value &V,
1531 MachineIRBuilder &MIRBuilder) {
1532 Register Src = getOrCreateVReg(V);
1533 auto &Regs = *VMap.getVRegs(U);
1534 if (Regs.empty()) {
1535 Regs.push_back(Src);
1536 VMap.getOffsets(U)->push_back(0);
1537 } else {
1538 // If we already assigned a vreg for this instruction, we can't change that.
1539 // Emit a copy to satisfy the users we already emitted.
1540 MIRBuilder.buildCopy(Regs[0], Src);
1541 }
1542 return true;
1543}
1544
1545bool IRTranslator::translateBitCast(const User &U,
1546 MachineIRBuilder &MIRBuilder) {
1547 // If we're bitcasting to the source type, we can reuse the source vreg.
1548 if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1549 getLLTForType(*U.getType(), *DL)) {
1550 // If the source is a ConstantInt then it was probably created by
1551 // ConstantHoisting and we should leave it alone.
1552 if (isa<ConstantInt>(U.getOperand(0)))
1553 return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1554 MIRBuilder);
1555 return translateCopy(U, *U.getOperand(0), MIRBuilder);
1556 }
1557
1558 return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1559}
1560
1561bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1562 MachineIRBuilder &MIRBuilder) {
1563 if (U.getType()->getScalarType()->isBFloatTy() ||
1564 U.getOperand(0)->getType()->getScalarType()->isBFloatTy())
1565 return false;
1566 Register Op = getOrCreateVReg(*U.getOperand(0));
1567 Register Res = getOrCreateVReg(U);
1568 MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1569 return true;
1570}
1571
1572bool IRTranslator::translateGetElementPtr(const User &U,
1573 MachineIRBuilder &MIRBuilder) {
1574 Value &Op0 = *U.getOperand(0);
1575 Register BaseReg = getOrCreateVReg(Op0);
1576 Type *PtrIRTy = Op0.getType();
1577 LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1578 Type *OffsetIRTy = DL->getIndexType(PtrIRTy);
1579 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1580
1581 uint32_t Flags = 0;
1582 if (isa<Instruction>(U)) {
1583 const Instruction &I = cast<Instruction>(U);
1585 }
1586
1587 // Normalize Vector GEP - all scalar operands should be converted to the
1588 // splat vector.
1589 unsigned VectorWidth = 0;
1590
1591 // True if we should use a splat vector; using VectorWidth alone is not
1592 // sufficient.
1593 bool WantSplatVector = false;
1594 if (auto *VT = dyn_cast<VectorType>(U.getType())) {
1595 VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1596 // We don't produce 1 x N vectors; those are treated as scalars.
1597 WantSplatVector = VectorWidth > 1;
1598 }
1599
1600 // We might need to splat the base pointer into a vector if the offsets
1601 // are vectors.
1602 if (WantSplatVector && !PtrTy.isVector()) {
1603 BaseReg = MIRBuilder
1604 .buildSplatBuildVector(LLT::fixed_vector(VectorWidth, PtrTy),
1605 BaseReg)
1606 .getReg(0);
1607 PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
1608 PtrTy = getLLTForType(*PtrIRTy, *DL);
1609 OffsetIRTy = DL->getIndexType(PtrIRTy);
1610 OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1611 }
1612
1613 int64_t Offset = 0;
1614 for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1615 GTI != E; ++GTI) {
1616 const Value *Idx = GTI.getOperand();
1617 if (StructType *StTy = GTI.getStructTypeOrNull()) {
1618 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1620 continue;
1621 } else {
1622 uint64_t ElementSize = GTI.getSequentialElementStride(*DL);
1623
1624 // If this is a scalar constant or a splat vector of constants,
1625 // handle it quickly.
1626 if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1627 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1628 Offset += ElementSize * *Val;
1629 continue;
1630 }
1631 }
1632
1633 if (Offset != 0) {
1634 auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1635 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1636 .getReg(0);
1637 Offset = 0;
1638 }
1639
1640 Register IdxReg = getOrCreateVReg(*Idx);
1641 LLT IdxTy = MRI->getType(IdxReg);
1642 if (IdxTy != OffsetTy) {
1643 if (!IdxTy.isVector() && WantSplatVector) {
1644 IdxReg = MIRBuilder
1646 IdxReg)
1647 .getReg(0);
1648 }
1649
1650 IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1651 }
1652
1653 // N = N + Idx * ElementSize;
1654 // Avoid doing it for ElementSize of 1.
1655 Register GepOffsetReg;
1656 if (ElementSize != 1) {
1657 auto ElementSizeMIB = MIRBuilder.buildConstant(
1658 getLLTForType(*OffsetIRTy, *DL), ElementSize);
1659 GepOffsetReg =
1660 MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
1661 } else
1662 GepOffsetReg = IdxReg;
1663
1664 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1665 }
1666 }
1667
1668 if (Offset != 0) {
1669 auto OffsetMIB =
1670 MIRBuilder.buildConstant(OffsetTy, Offset);
1671
1672 if (int64_t(Offset) >= 0 && cast<GEPOperator>(U).isInBounds())
1674
1675 MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0),
1676 Flags);
1677 return true;
1678 }
1679
1680 MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1681 return true;
1682}
1683
1684bool IRTranslator::translateMemFunc(const CallInst &CI,
1685 MachineIRBuilder &MIRBuilder,
1686 unsigned Opcode) {
1687 const Value *SrcPtr = CI.getArgOperand(1);
1688 // If the source is undef, then just emit a nop.
1689 if (isa<UndefValue>(SrcPtr))
1690 return true;
1691
1693
1694 unsigned MinPtrSize = UINT_MAX;
1695 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {
1696 Register SrcReg = getOrCreateVReg(**AI);
1697 LLT SrcTy = MRI->getType(SrcReg);
1698 if (SrcTy.isPointer())
1699 MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);
1700 SrcRegs.push_back(SrcReg);
1701 }
1702
1703 LLT SizeTy = LLT::scalar(MinPtrSize);
1704
1705 // The size operand should be the minimum of the pointer sizes.
1706 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1707 if (MRI->getType(SizeOpReg) != SizeTy)
1708 SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);
1709
1710 auto ICall = MIRBuilder.buildInstr(Opcode);
1711 for (Register SrcReg : SrcRegs)
1712 ICall.addUse(SrcReg);
1713
1714 Align DstAlign;
1715 Align SrcAlign;
1716 unsigned IsVol =
1717 cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();
1718
1719 ConstantInt *CopySize = nullptr;
1720
1721 if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1722 DstAlign = MCI->getDestAlign().valueOrOne();
1723 SrcAlign = MCI->getSourceAlign().valueOrOne();
1724 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1725 } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
1726 DstAlign = MCI->getDestAlign().valueOrOne();
1727 SrcAlign = MCI->getSourceAlign().valueOrOne();
1728 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1729 } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1730 DstAlign = MMI->getDestAlign().valueOrOne();
1731 SrcAlign = MMI->getSourceAlign().valueOrOne();
1732 CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2));
1733 } else {
1734 auto *MSI = cast<MemSetInst>(&CI);
1735 DstAlign = MSI->getDestAlign().valueOrOne();
1736 }
1737
1738 if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1739 // We need to propagate the tail call flag from the IR inst as an argument.
1740 // Otherwise, we have to pessimize and assume later that we cannot tail call
1741 // any memory intrinsics.
1742 ICall.addImm(CI.isTailCall() ? 1 : 0);
1743 }
1744
1745 // Create mem operands to store the alignment and volatile info.
1748 if (IsVol) {
1749 LoadFlags |= MachineMemOperand::MOVolatile;
1750 StoreFlags |= MachineMemOperand::MOVolatile;
1751 }
1752
1753 AAMDNodes AAInfo = CI.getAAMetadata();
1754 if (AA && CopySize &&
1756 SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) {
1757 LoadFlags |= MachineMemOperand::MOInvariant;
1758
1759 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1760 // but the previous usage implied it did. Probably should check
1761 // isDereferenceableAndAlignedPointer.
1763 }
1764
1765 ICall.addMemOperand(
1767 StoreFlags, 1, DstAlign, AAInfo));
1768 if (Opcode != TargetOpcode::G_MEMSET)
1769 ICall.addMemOperand(MF->getMachineMemOperand(
1770 MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
1771
1772 return true;
1773}
1774
1775bool IRTranslator::translateVectorInterleave2Intrinsic(
1776 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1777 assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2 &&
1778 "This function can only be called on the interleave2 intrinsic!");
1779 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
1780 Register Op0 = getOrCreateVReg(*CI.getOperand(0));
1781 Register Op1 = getOrCreateVReg(*CI.getOperand(1));
1782 Register Res = getOrCreateVReg(CI);
1783
1784 LLT OpTy = MRI->getType(Op0);
1785 MIRBuilder.buildShuffleVector(Res, Op0, Op1,
1787
1788 return true;
1789}
1790
1791bool IRTranslator::translateVectorDeinterleave2Intrinsic(
1792 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1793 assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_deinterleave2 &&
1794 "This function can only be called on the deinterleave2 intrinsic!");
1795 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
1796 // SelectionDAG).
1797 Register Op = getOrCreateVReg(*CI.getOperand(0));
1798 auto Undef = MIRBuilder.buildUndef(MRI->getType(Op));
1799 ArrayRef<Register> Res = getOrCreateVRegs(CI);
1800
1801 LLT ResTy = MRI->getType(Res[0]);
1802 MIRBuilder.buildShuffleVector(Res[0], Op, Undef,
1803 createStrideMask(0, 2, ResTy.getNumElements()));
1804 MIRBuilder.buildShuffleVector(Res[1], Op, Undef,
1805 createStrideMask(1, 2, ResTy.getNumElements()));
1806
1807 return true;
1808}
1809
1810void IRTranslator::getStackGuard(Register DstReg,
1811 MachineIRBuilder &MIRBuilder) {
1813 MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1814 auto MIB =
1815 MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1816
1818 if (!Global)
1819 return;
1820
1821 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1822 LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
1823
1824 MachinePointerInfo MPInfo(Global);
1828 MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
1829 MIB.setMemRefs({MemRef});
1830}
1831
1832bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1833 MachineIRBuilder &MIRBuilder) {
1834 ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1835 MIRBuilder.buildInstr(
1836 Op, {ResRegs[0], ResRegs[1]},
1837 {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});
1838
1839 return true;
1840}
1841
1842bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1843 MachineIRBuilder &MIRBuilder) {
1844 Register Dst = getOrCreateVReg(CI);
1845 Register Src0 = getOrCreateVReg(*CI.getOperand(0));
1846 Register Src1 = getOrCreateVReg(*CI.getOperand(1));
1847 uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
1848 MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
1849 return true;
1850}
1851
1852unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1853 switch (ID) {
1854 default:
1855 break;
1856 case Intrinsic::bswap:
1857 return TargetOpcode::G_BSWAP;
1858 case Intrinsic::bitreverse:
1859 return TargetOpcode::G_BITREVERSE;
1860 case Intrinsic::fshl:
1861 return TargetOpcode::G_FSHL;
1862 case Intrinsic::fshr:
1863 return TargetOpcode::G_FSHR;
1864 case Intrinsic::ceil:
1865 return TargetOpcode::G_FCEIL;
1866 case Intrinsic::cos:
1867 return TargetOpcode::G_FCOS;
1868 case Intrinsic::ctpop:
1869 return TargetOpcode::G_CTPOP;
1870 case Intrinsic::exp:
1871 return TargetOpcode::G_FEXP;
1872 case Intrinsic::exp2:
1873 return TargetOpcode::G_FEXP2;
1874 case Intrinsic::exp10:
1875 return TargetOpcode::G_FEXP10;
1876 case Intrinsic::fabs:
1877 return TargetOpcode::G_FABS;
1878 case Intrinsic::copysign:
1879 return TargetOpcode::G_FCOPYSIGN;
1880 case Intrinsic::minnum:
1881 return TargetOpcode::G_FMINNUM;
1882 case Intrinsic::maxnum:
1883 return TargetOpcode::G_FMAXNUM;
1884 case Intrinsic::minimum:
1885 return TargetOpcode::G_FMINIMUM;
1886 case Intrinsic::maximum:
1887 return TargetOpcode::G_FMAXIMUM;
1888 case Intrinsic::canonicalize:
1889 return TargetOpcode::G_FCANONICALIZE;
1890 case Intrinsic::floor:
1891 return TargetOpcode::G_FFLOOR;
1892 case Intrinsic::fma:
1893 return TargetOpcode::G_FMA;
1894 case Intrinsic::log:
1895 return TargetOpcode::G_FLOG;
1896 case Intrinsic::log2:
1897 return TargetOpcode::G_FLOG2;
1898 case Intrinsic::log10:
1899 return TargetOpcode::G_FLOG10;
1900 case Intrinsic::ldexp:
1901 return TargetOpcode::G_FLDEXP;
1902 case Intrinsic::nearbyint:
1903 return TargetOpcode::G_FNEARBYINT;
1904 case Intrinsic::pow:
1905 return TargetOpcode::G_FPOW;
1906 case Intrinsic::powi:
1907 return TargetOpcode::G_FPOWI;
1908 case Intrinsic::rint:
1909 return TargetOpcode::G_FRINT;
1910 case Intrinsic::round:
1911 return TargetOpcode::G_INTRINSIC_ROUND;
1912 case Intrinsic::roundeven:
1913 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1914 case Intrinsic::sin:
1915 return TargetOpcode::G_FSIN;
1916 case Intrinsic::sqrt:
1917 return TargetOpcode::G_FSQRT;
1918 case Intrinsic::trunc:
1919 return TargetOpcode::G_INTRINSIC_TRUNC;
1920 case Intrinsic::readcyclecounter:
1921 return TargetOpcode::G_READCYCLECOUNTER;
1922 case Intrinsic::readsteadycounter:
1923 return TargetOpcode::G_READSTEADYCOUNTER;
1924 case Intrinsic::ptrmask:
1925 return TargetOpcode::G_PTRMASK;
1926 case Intrinsic::lrint:
1927 return TargetOpcode::G_INTRINSIC_LRINT;
1928 // FADD/FMUL require checking the FMF, so are handled elsewhere.
1929 case Intrinsic::vector_reduce_fmin:
1930 return TargetOpcode::G_VECREDUCE_FMIN;
1931 case Intrinsic::vector_reduce_fmax:
1932 return TargetOpcode::G_VECREDUCE_FMAX;
1933 case Intrinsic::vector_reduce_fminimum:
1934 return TargetOpcode::G_VECREDUCE_FMINIMUM;
1935 case Intrinsic::vector_reduce_fmaximum:
1936 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
1937 case Intrinsic::vector_reduce_add:
1938 return TargetOpcode::G_VECREDUCE_ADD;
1939 case Intrinsic::vector_reduce_mul:
1940 return TargetOpcode::G_VECREDUCE_MUL;
1941 case Intrinsic::vector_reduce_and:
1942 return TargetOpcode::G_VECREDUCE_AND;
1943 case Intrinsic::vector_reduce_or:
1944 return TargetOpcode::G_VECREDUCE_OR;
1945 case Intrinsic::vector_reduce_xor:
1946 return TargetOpcode::G_VECREDUCE_XOR;
1947 case Intrinsic::vector_reduce_smax:
1948 return TargetOpcode::G_VECREDUCE_SMAX;
1949 case Intrinsic::vector_reduce_smin:
1950 return TargetOpcode::G_VECREDUCE_SMIN;
1951 case Intrinsic::vector_reduce_umax:
1952 return TargetOpcode::G_VECREDUCE_UMAX;
1953 case Intrinsic::vector_reduce_umin:
1954 return TargetOpcode::G_VECREDUCE_UMIN;
1955 case Intrinsic::lround:
1956 return TargetOpcode::G_LROUND;
1957 case Intrinsic::llround:
1958 return TargetOpcode::G_LLROUND;
1959 case Intrinsic::get_fpenv:
1960 return TargetOpcode::G_GET_FPENV;
1961 case Intrinsic::get_fpmode:
1962 return TargetOpcode::G_GET_FPMODE;
1963 }
1965}
1966
1967bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1969 MachineIRBuilder &MIRBuilder) {
1970
1971 unsigned Op = getSimpleIntrinsicOpcode(ID);
1972
1973 // Is this a simple intrinsic?
1975 return false;
1976
1977 // Yes. Let's translate it.
1979 for (const auto &Arg : CI.args())
1980 VRegs.push_back(getOrCreateVReg(*Arg));
1981
1982 MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1984 return true;
1985}
1986
1987// TODO: Include ConstainedOps.def when all strict instructions are defined.
1989 switch (ID) {
1990 case Intrinsic::experimental_constrained_fadd:
1991 return TargetOpcode::G_STRICT_FADD;
1992 case Intrinsic::experimental_constrained_fsub:
1993 return TargetOpcode::G_STRICT_FSUB;
1994 case Intrinsic::experimental_constrained_fmul:
1995 return TargetOpcode::G_STRICT_FMUL;
1996 case Intrinsic::experimental_constrained_fdiv:
1997 return TargetOpcode::G_STRICT_FDIV;
1998 case Intrinsic::experimental_constrained_frem:
1999 return TargetOpcode::G_STRICT_FREM;
2000 case Intrinsic::experimental_constrained_fma:
2001 return TargetOpcode::G_STRICT_FMA;
2002 case Intrinsic::experimental_constrained_sqrt:
2003 return TargetOpcode::G_STRICT_FSQRT;
2004 case Intrinsic::experimental_constrained_ldexp:
2005 return TargetOpcode::G_STRICT_FLDEXP;
2006 default:
2007 return 0;
2008 }
2009}
2010
2011bool IRTranslator::translateConstrainedFPIntrinsic(
2012 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2014
2015 unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
2016 if (!Opcode)
2017 return false;
2018
2022
2024 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(0)));
2025 if (!FPI.isUnaryOp())
2026 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(1)));
2027 if (FPI.isTernaryOp())
2028 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(2)));
2029
2030 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
2031 return true;
2032}
2033
2034std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
2035 auto VRegs = getOrCreateVRegs(Arg);
2036 if (VRegs.size() != 1)
2037 return std::nullopt;
2038
2039 // Arguments are lowered as a copy of a livein physical register.
2040 auto *VRegDef = MF->getRegInfo().getVRegDef(VRegs[0]);
2041 if (!VRegDef || !VRegDef->isCopy())
2042 return std::nullopt;
2043 return VRegDef->getOperand(1).getReg().asMCReg();
2044}
2045
2046bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,
2047 const DILocalVariable *Var,
2048 const DIExpression *Expr,
2049 const DebugLoc &DL,
2050 MachineIRBuilder &MIRBuilder) {
2051 auto *Arg = dyn_cast<Argument>(Val);
2052 if (!Arg)
2053 return false;
2054
2055 if (!Expr->isEntryValue())
2056 return false;
2057
2058 std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
2059 if (!PhysReg) {
2060 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2061 << ": expression is entry_value but "
2062 << "couldn't find a physical register\n");
2063 LLVM_DEBUG(dbgs() << *Var << "\n");
2064 return true;
2065 }
2066
2067 if (isDeclare) {
2068 // Append an op deref to account for the fact that this is a dbg_declare.
2069 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
2070 MF->setVariableDbgInfo(Var, Expr, *PhysReg, DL);
2071 } else {
2072 MIRBuilder.buildDirectDbgValue(*PhysReg, Var, Expr);
2073 }
2074
2075 return true;
2076}
2077
2079 switch (ID) {
2080 default:
2081 llvm_unreachable("Unexpected intrinsic");
2082 case Intrinsic::experimental_convergence_anchor:
2083 return TargetOpcode::CONVERGENCECTRL_ANCHOR;
2084 case Intrinsic::experimental_convergence_entry:
2085 return TargetOpcode::CONVERGENCECTRL_ENTRY;
2086 case Intrinsic::experimental_convergence_loop:
2087 return TargetOpcode::CONVERGENCECTRL_LOOP;
2088 }
2089}
2090
2091bool IRTranslator::translateConvergenceControlIntrinsic(
2092 const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
2094 Register OutputReg = getOrCreateConvergenceTokenVReg(CI);
2095 MIB.addDef(OutputReg);
2096
2097 if (ID == Intrinsic::experimental_convergence_loop) {
2099 assert(Bundle && "Expected a convergence control token.");
2100 Register InputReg =
2101 getOrCreateConvergenceTokenVReg(*Bundle->Inputs[0].get());
2102 MIB.addUse(InputReg);
2103 }
2104
2105 return true;
2106}
2107
2108bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
2109 MachineIRBuilder &MIRBuilder) {
2110 if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
2111 if (ORE->enabled()) {
2112 if (MemoryOpRemark::canHandle(MI, *LibInfo)) {
2113 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2114 R.visit(MI);
2115 }
2116 }
2117 }
2118
2119 // If this is a simple intrinsic (that is, we just need to add a def of
2120 // a vreg, and uses for each arg operand, then translate it.
2121 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
2122 return true;
2123
2124 switch (ID) {
2125 default:
2126 break;
2127 case Intrinsic::lifetime_start:
2128 case Intrinsic::lifetime_end: {
2129 // No stack colouring in O0, discard region information.
2131 return true;
2132
2133 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2134 : TargetOpcode::LIFETIME_END;
2135
2136 // Get the underlying objects for the location passed on the lifetime
2137 // marker.
2139 getUnderlyingObjects(CI.getArgOperand(1), Allocas);
2140
2141 // Iterate over each underlying object, creating lifetime markers for each
2142 // static alloca. Quit if we find a non-static alloca.
2143 for (const Value *V : Allocas) {
2144 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
2145 if (!AI)
2146 continue;
2147
2148 if (!AI->isStaticAlloca())
2149 return true;
2150
2151 MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
2152 }
2153 return true;
2154 }
2155 case Intrinsic::dbg_declare: {
2156 const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
2157 assert(DI.getVariable() && "Missing variable");
2158 translateDbgDeclareRecord(DI.getAddress(), DI.hasArgList(), DI.getVariable(),
2159 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2160 return true;
2161 }
2162 case Intrinsic::dbg_label: {
2163 const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
2164 assert(DI.getLabel() && "Missing label");
2165
2167 MIRBuilder.getDebugLoc()) &&
2168 "Expected inlined-at fields to agree");
2169
2170 MIRBuilder.buildDbgLabel(DI.getLabel());
2171 return true;
2172 }
2173 case Intrinsic::vaend:
2174 // No target I know of cares about va_end. Certainly no in-tree target
2175 // does. Simplest intrinsic ever!
2176 return true;
2177 case Intrinsic::vastart: {
2178 Value *Ptr = CI.getArgOperand(0);
2179 unsigned ListSize = TLI->getVaListSizeInBits(*DL) / 8;
2180 Align Alignment = getKnownAlignment(Ptr, *DL);
2181
2182 MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
2183 .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),
2185 ListSize, Alignment));
2186 return true;
2187 }
2188 case Intrinsic::dbg_assign:
2189 // A dbg.assign is a dbg.value with more information about stack locations,
2190 // typically produced during optimisation of variables with leaked
2191 // addresses. We can treat it like a normal dbg_value intrinsic here; to
2192 // benefit from the full analysis of stack/SSA locations, GlobalISel would
2193 // need to register for and use the AssignmentTrackingAnalysis pass.
2195 case Intrinsic::dbg_value: {
2196 // This form of DBG_VALUE is target-independent.
2197 const DbgValueInst &DI = cast<DbgValueInst>(CI);
2198 translateDbgValueRecord(DI.getValue(), DI.hasArgList(), DI.getVariable(),
2199 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2200 return true;
2201 }
2202 case Intrinsic::uadd_with_overflow:
2203 return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
2204 case Intrinsic::sadd_with_overflow:
2205 return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
2206 case Intrinsic::usub_with_overflow:
2207 return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
2208 case Intrinsic::ssub_with_overflow:
2209 return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
2210 case Intrinsic::umul_with_overflow:
2211 return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
2212 case Intrinsic::smul_with_overflow:
2213 return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
2214 case Intrinsic::uadd_sat:
2215 return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
2216 case Intrinsic::sadd_sat:
2217 return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
2218 case Intrinsic::usub_sat:
2219 return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
2220 case Intrinsic::ssub_sat:
2221 return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
2222 case Intrinsic::ushl_sat:
2223 return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
2224 case Intrinsic::sshl_sat:
2225 return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
2226 case Intrinsic::umin:
2227 return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
2228 case Intrinsic::umax:
2229 return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
2230 case Intrinsic::smin:
2231 return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
2232 case Intrinsic::smax:
2233 return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
2234 case Intrinsic::abs:
2235 // TODO: Preserve "int min is poison" arg in GMIR?
2236 return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
2237 case Intrinsic::smul_fix:
2238 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2239 case Intrinsic::umul_fix:
2240 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2241 case Intrinsic::smul_fix_sat:
2242 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2243 case Intrinsic::umul_fix_sat:
2244 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2245 case Intrinsic::sdiv_fix:
2246 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2247 case Intrinsic::udiv_fix:
2248 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2249 case Intrinsic::sdiv_fix_sat:
2250 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2251 case Intrinsic::udiv_fix_sat:
2252 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2253 case Intrinsic::fmuladd: {
2254 const TargetMachine &TM = MF->getTarget();
2255 Register Dst = getOrCreateVReg(CI);
2256 Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
2257 Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
2258 Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
2259 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2261 TLI->getValueType(*DL, CI.getType()))) {
2262 // TODO: Revisit this to see if we should move this part of the
2263 // lowering to the combiner.
2264 MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,
2266 } else {
2267 LLT Ty = getLLTForType(*CI.getType(), *DL);
2268 auto FMul = MIRBuilder.buildFMul(
2269 Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));
2270 MIRBuilder.buildFAdd(Dst, FMul, Op2,
2272 }
2273 return true;
2274 }
2275 case Intrinsic::convert_from_fp16:
2276 // FIXME: This intrinsic should probably be removed from the IR.
2277 MIRBuilder.buildFPExt(getOrCreateVReg(CI),
2278 getOrCreateVReg(*CI.getArgOperand(0)),
2280 return true;
2281 case Intrinsic::convert_to_fp16:
2282 // FIXME: This intrinsic should probably be removed from the IR.
2283 MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
2284 getOrCreateVReg(*CI.getArgOperand(0)),
2286 return true;
2287 case Intrinsic::frexp: {
2288 ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
2289 MIRBuilder.buildFFrexp(VRegs[0], VRegs[1],
2290 getOrCreateVReg(*CI.getArgOperand(0)),
2292 return true;
2293 }
2294 case Intrinsic::memcpy_inline:
2295 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
2296 case Intrinsic::memcpy:
2297 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2298 case Intrinsic::memmove:
2299 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2300 case Intrinsic::memset:
2301 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2302 case Intrinsic::eh_typeid_for: {
2304 Register Reg = getOrCreateVReg(CI);
2305 unsigned TypeID = MF->getTypeIDFor(GV);
2306 MIRBuilder.buildConstant(Reg, TypeID);
2307 return true;
2308 }
2309 case Intrinsic::objectsize:
2310 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2311
2312 case Intrinsic::is_constant:
2313 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2314
2315 case Intrinsic::stackguard:
2316 getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2317 return true;
2318 case Intrinsic::stackprotector: {
2319 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2320 Register GuardVal;
2321 if (TLI->useLoadStackGuardNode()) {
2322 GuardVal = MRI->createGenericVirtualRegister(PtrTy);
2323 getStackGuard(GuardVal, MIRBuilder);
2324 } else
2325 GuardVal = getOrCreateVReg(*CI.getArgOperand(0)); // The guard's value.
2326
2327 AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
2328 int FI = getOrCreateFrameIndex(*Slot);
2330
2331 MIRBuilder.buildStore(
2332 GuardVal, getOrCreateVReg(*Slot),
2336 PtrTy, Align(8)));
2337 return true;
2338 }
2339 case Intrinsic::stacksave: {
2340 MIRBuilder.buildInstr(TargetOpcode::G_STACKSAVE, {getOrCreateVReg(CI)}, {});
2341 return true;
2342 }
2343 case Intrinsic::stackrestore: {
2344 MIRBuilder.buildInstr(TargetOpcode::G_STACKRESTORE, {},
2345 {getOrCreateVReg(*CI.getArgOperand(0))});
2346 return true;
2347 }
2348 case Intrinsic::cttz:
2349 case Intrinsic::ctlz: {
2350 ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
2351 bool isTrailing = ID == Intrinsic::cttz;
2352 unsigned Opcode = isTrailing
2353 ? Cst->isZero() ? TargetOpcode::G_CTTZ
2354 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2355 : Cst->isZero() ? TargetOpcode::G_CTLZ
2356 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2357 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
2358 {getOrCreateVReg(*CI.getArgOperand(0))});
2359 return true;
2360 }
2361 case Intrinsic::invariant_start: {
2362 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2364 MIRBuilder.buildUndef(Undef);
2365 return true;
2366 }
2367 case Intrinsic::invariant_end:
2368 return true;
2369 case Intrinsic::expect:
2370 case Intrinsic::annotation:
2371 case Intrinsic::ptr_annotation:
2372 case Intrinsic::launder_invariant_group:
2373 case Intrinsic::strip_invariant_group: {
2374 // Drop the intrinsic, but forward the value.
2375 MIRBuilder.buildCopy(getOrCreateVReg(CI),
2376 getOrCreateVReg(*CI.getArgOperand(0)));
2377 return true;
2378 }
2379 case Intrinsic::assume:
2380 case Intrinsic::experimental_noalias_scope_decl:
2381 case Intrinsic::var_annotation:
2382 case Intrinsic::sideeffect:
2383 // Discard annotate attributes, assumptions, and artificial side-effects.
2384 return true;
2385 case Intrinsic::read_volatile_register:
2386 case Intrinsic::read_register: {
2387 Value *Arg = CI.getArgOperand(0);
2388 MIRBuilder
2389 .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2390 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
2391 return true;
2392 }
2393 case Intrinsic::write_register: {
2394 Value *Arg = CI.getArgOperand(0);
2395 MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
2396 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
2397 .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
2398 return true;
2399 }
2400 case Intrinsic::localescape: {
2401 MachineBasicBlock &EntryMBB = MF->front();
2403
2404 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2405 // is the same on all targets.
2406 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2408 if (isa<ConstantPointerNull>(Arg))
2409 continue; // Skip null pointers. They represent a hole in index space.
2410
2411 int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
2412 MCSymbol *FrameAllocSym =
2413 MF->getMMI().getContext().getOrCreateFrameAllocSymbol(EscapedName,
2414 Idx);
2415
2416 // This should be inserted at the start of the entry block.
2417 auto LocalEscape =
2418 MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)
2419 .addSym(FrameAllocSym)
2420 .addFrameIndex(FI);
2421
2422 EntryMBB.insert(EntryMBB.begin(), LocalEscape);
2423 }
2424
2425 return true;
2426 }
2427 case Intrinsic::vector_reduce_fadd:
2428 case Intrinsic::vector_reduce_fmul: {
2429 // Need to check for the reassoc flag to decide whether we want a
2430 // sequential reduction opcode or not.
2431 Register Dst = getOrCreateVReg(CI);
2432 Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));
2433 Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));
2434 unsigned Opc = 0;
2435 if (!CI.hasAllowReassoc()) {
2436 // The sequential ordering case.
2437 Opc = ID == Intrinsic::vector_reduce_fadd
2438 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2439 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2440 MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2442 return true;
2443 }
2444 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2445 // since the associativity doesn't matter.
2446 unsigned ScalarOpc;
2447 if (ID == Intrinsic::vector_reduce_fadd) {
2448 Opc = TargetOpcode::G_VECREDUCE_FADD;
2449 ScalarOpc = TargetOpcode::G_FADD;
2450 } else {
2451 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2452 ScalarOpc = TargetOpcode::G_FMUL;
2453 }
2454 LLT DstTy = MRI->getType(Dst);
2455 auto Rdx = MIRBuilder.buildInstr(
2456 Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));
2457 MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2459
2460 return true;
2461 }
2462 case Intrinsic::trap:
2463 case Intrinsic::debugtrap:
2464 case Intrinsic::ubsantrap: {
2465 StringRef TrapFuncName =
2466 CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
2467 if (TrapFuncName.empty())
2468 break; // Use the default handling.
2470 if (ID == Intrinsic::ubsantrap) {
2471 Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
2472 CI.getArgOperand(0)->getType(), 0});
2473 }
2474 Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
2475 Info.CB = &CI;
2476 Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
2477 return CLI->lowerCall(MIRBuilder, Info);
2478 }
2479 case Intrinsic::amdgcn_cs_chain:
2480 return translateCallBase(CI, MIRBuilder);
2481 case Intrinsic::fptrunc_round: {
2483
2484 // Convert the metadata argument to a constant integer
2485 Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata();
2486 std::optional<RoundingMode> RoundMode =
2487 convertStrToRoundingMode(cast<MDString>(MD)->getString());
2488
2489 // Add the Rounding mode as an integer
2490 MIRBuilder
2491 .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2492 {getOrCreateVReg(CI)},
2493 {getOrCreateVReg(*CI.getArgOperand(0))}, Flags)
2494 .addImm((int)*RoundMode);
2495
2496 return true;
2497 }
2498 case Intrinsic::is_fpclass: {
2499 Value *FpValue = CI.getOperand(0);
2500 ConstantInt *TestMaskValue = cast<ConstantInt>(CI.getOperand(1));
2501
2502 MIRBuilder
2503 .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},
2504 {getOrCreateVReg(*FpValue)})
2505 .addImm(TestMaskValue->getZExtValue());
2506
2507 return true;
2508 }
2509 case Intrinsic::set_fpenv: {
2510 Value *FPEnv = CI.getOperand(0);
2511 MIRBuilder.buildInstr(TargetOpcode::G_SET_FPENV, {},
2512 {getOrCreateVReg(*FPEnv)});
2513 return true;
2514 }
2515 case Intrinsic::reset_fpenv: {
2516 MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPENV, {}, {});
2517 return true;
2518 }
2519 case Intrinsic::set_fpmode: {
2520 Value *FPState = CI.getOperand(0);
2521 MIRBuilder.buildInstr(TargetOpcode::G_SET_FPMODE, {},
2522 { getOrCreateVReg(*FPState) });
2523 return true;
2524 }
2525 case Intrinsic::reset_fpmode: {
2526 MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {});
2527 return true;
2528 }
2529 case Intrinsic::prefetch: {
2530 Value *Addr = CI.getOperand(0);
2531 unsigned RW = cast<ConstantInt>(CI.getOperand(1))->getZExtValue();
2532 unsigned Locality = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
2533 unsigned CacheType = cast<ConstantInt>(CI.getOperand(3))->getZExtValue();
2534
2536 auto &MMO = *MF->getMachineMemOperand(MachinePointerInfo(Addr), Flags,
2537 LLT(), Align());
2538
2539 MIRBuilder.buildPrefetch(getOrCreateVReg(*Addr), RW, Locality, CacheType,
2540 MMO);
2541
2542 return true;
2543 }
2544
2545 case Intrinsic::experimental_vector_interleave2:
2546 case Intrinsic::experimental_vector_deinterleave2: {
2547 // Both intrinsics have at least one operand.
2548 Value *Op0 = CI.getOperand(0);
2549 LLT ResTy = getLLTForType(*Op0->getType(), MIRBuilder.getDataLayout());
2550 if (!ResTy.isFixedVector())
2551 return false;
2552
2553 if (CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2)
2554 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
2555
2556 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
2557 }
2558
2559#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2560 case Intrinsic::INTRINSIC:
2561#include "llvm/IR/ConstrainedOps.def"
2562 return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2563 MIRBuilder);
2564 case Intrinsic::experimental_convergence_anchor:
2565 case Intrinsic::experimental_convergence_entry:
2566 case Intrinsic::experimental_convergence_loop:
2567 return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
2568 }
2569 return false;
2570}
2571
2572bool IRTranslator::translateInlineAsm(const CallBase &CB,
2573 MachineIRBuilder &MIRBuilder) {
2574
2576
2577 if (!ALI) {
2578 LLVM_DEBUG(
2579 dbgs() << "Inline asm lowering is not supported for this target yet\n");
2580 return false;
2581 }
2582
2583 return ALI->lowerInlineAsm(
2584 MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });
2585}
2586
2587bool IRTranslator::translateCallBase(const CallBase &CB,
2588 MachineIRBuilder &MIRBuilder) {
2589 ArrayRef<Register> Res = getOrCreateVRegs(CB);
2590
2592 Register SwiftInVReg = 0;
2593 Register SwiftErrorVReg = 0;
2594 for (const auto &Arg : CB.args()) {
2595 if (CLI->supportSwiftError() && isSwiftError(Arg)) {
2596 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2597 LLT Ty = getLLTForType(*Arg->getType(), *DL);
2598 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2599 MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
2600 &CB, &MIRBuilder.getMBB(), Arg));
2601 Args.emplace_back(ArrayRef(SwiftInVReg));
2602 SwiftErrorVReg =
2603 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2604 continue;
2605 }
2606 Args.push_back(getOrCreateVRegs(*Arg));
2607 }
2608
2609 if (auto *CI = dyn_cast<CallInst>(&CB)) {
2610 if (ORE->enabled()) {
2611 if (MemoryOpRemark::canHandle(CI, *LibInfo)) {
2612 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2613 R.visit(CI);
2614 }
2615 }
2616 }
2617
2618 Register ConvergenceCtrlToken = 0;
2619 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
2620 const auto &Token = *Bundle->Inputs[0].get();
2621 ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
2622 }
2623
2624 // We don't set HasCalls on MFI here yet because call lowering may decide to
2625 // optimize into tail calls. Instead, we defer that to selection where a final
2626 // scan is done to check if any instructions are calls.
2627 bool Success = CLI->lowerCall(
2628 MIRBuilder, CB, Res, Args, SwiftErrorVReg, ConvergenceCtrlToken,
2629 [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
2630
2631 // Check if we just inserted a tail call.
2632 if (Success) {
2633 assert(!HasTailCall && "Can't tail call return twice from block?");
2635 HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
2636 }
2637
2638 return Success;
2639}
2640
2641bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2642 const CallInst &CI = cast<CallInst>(U);
2643 auto TII = MF->getTarget().getIntrinsicInfo();
2644 const Function *F = CI.getCalledFunction();
2645
2646 // FIXME: support Windows dllimport function calls and calls through
2647 // weak symbols.
2648 if (F && (F->hasDLLImportStorageClass() ||
2650 F->hasExternalWeakLinkage())))
2651 return false;
2652
2653 // FIXME: support control flow guard targets.
2655 return false;
2656
2657 // FIXME: support statepoints and related.
2658 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
2659 return false;
2660
2661 if (CI.isInlineAsm())
2662 return translateInlineAsm(CI, MIRBuilder);
2663
2664 diagnoseDontCall(CI);
2665
2667 if (F && F->isIntrinsic()) {
2668 ID = F->getIntrinsicID();
2670 ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2671 }
2672
2673 if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2674 return translateCallBase(CI, MIRBuilder);
2675
2676 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2677
2678 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2679 return true;
2680
2681 ArrayRef<Register> ResultRegs;
2682 if (!CI.getType()->isVoidTy())
2683 ResultRegs = getOrCreateVRegs(CI);
2684
2685 // Ignore the callsite attributes. Backend code is most likely not expecting
2686 // an intrinsic to sometimes have side effects and sometimes not.
2687 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);
2688 if (isa<FPMathOperator>(CI))
2689 MIB->copyIRFlags(CI);
2690
2691 for (const auto &Arg : enumerate(CI.args())) {
2692 // If this is required to be an immediate, don't materialize it in a
2693 // register.
2694 if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2695 if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
2696 // imm arguments are more convenient than cimm (and realistically
2697 // probably sufficient), so use them.
2698 assert(CI->getBitWidth() <= 64 &&
2699 "large intrinsic immediates not handled");
2700 MIB.addImm(CI->getSExtValue());
2701 } else {
2702 MIB.addFPImm(cast<ConstantFP>(Arg.value()));
2703 }
2704 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {
2705 auto *MD = MDVal->getMetadata();
2706 auto *MDN = dyn_cast<MDNode>(MD);
2707 if (!MDN) {
2708 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))
2709 MDN = MDNode::get(MF->getFunction().getContext(), ConstMD);
2710 else // This was probably an MDString.
2711 return false;
2712 }
2713 MIB.addMetadata(MDN);
2714 } else {
2715 ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
2716 if (VRegs.size() > 1)
2717 return false;
2718 MIB.addUse(VRegs[0]);
2719 }
2720 }
2721
2722 // Add a MachineMemOperand if it is a target mem intrinsic.
2724 // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2725 if (TLI->getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2726 Align Alignment = Info.align.value_or(
2727 DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
2728 LLT MemTy = Info.memVT.isSimple()
2729 ? getLLTForMVT(Info.memVT.getSimpleVT())
2730 : LLT::scalar(Info.memVT.getStoreSizeInBits());
2731
2732 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
2733 // didn't yield anything useful.
2735 if (Info.ptrVal)
2736 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2737 else if (Info.fallbackAddressSpace)
2738 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2739 MIB.addMemOperand(
2740 MF->getMachineMemOperand(MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata()));
2741 }
2742
2743 if (CI.isConvergent()) {
2744 if (auto Bundle = CI.getOperandBundle(LLVMContext::OB_convergencectrl)) {
2745 auto *Token = Bundle->Inputs[0].get();
2746 Register TokenReg = getOrCreateVReg(*Token);
2747 MIB.addUse(TokenReg, RegState::Implicit);
2748 }
2749 }
2750
2751 return true;
2752}
2753
2754bool IRTranslator::findUnwindDestinations(
2755 const BasicBlock *EHPadBB,
2756 BranchProbability Prob,
2757 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2758 &UnwindDests) {
2760 EHPadBB->getParent()->getFunction().getPersonalityFn());
2761 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2762 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2763 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2764 bool IsSEH = isAsynchronousEHPersonality(Personality);
2765
2766 if (IsWasmCXX) {
2767 // Ignore this for now.
2768 return false;
2769 }
2770
2771 while (EHPadBB) {
2772 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2773 BasicBlock *NewEHPadBB = nullptr;
2774 if (isa<LandingPadInst>(Pad)) {
2775 // Stop on landingpads. They are not funclets.
2776 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2777 break;
2778 }
2779 if (isa<CleanupPadInst>(Pad)) {
2780 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2781 // personalities.
2782 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2783 UnwindDests.back().first->setIsEHScopeEntry();
2784 UnwindDests.back().first->setIsEHFuncletEntry();
2785 break;
2786 }
2787 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2788 // Add the catchpad handlers to the possible destinations.
2789 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2790 UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2791 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2792 if (IsMSVCCXX || IsCoreCLR)
2793 UnwindDests.back().first->setIsEHFuncletEntry();
2794 if (!IsSEH)
2795 UnwindDests.back().first->setIsEHScopeEntry();
2796 }
2797 NewEHPadBB = CatchSwitch->getUnwindDest();
2798 } else {
2799 continue;
2800 }
2801
2802 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2803 if (BPI && NewEHPadBB)
2804 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2805 EHPadBB = NewEHPadBB;
2806 }
2807 return true;
2808}
2809
2810bool IRTranslator::translateInvoke(const User &U,
2811 MachineIRBuilder &MIRBuilder) {
2812 const InvokeInst &I = cast<InvokeInst>(U);
2813 MCContext &Context = MF->getContext();
2814
2815 const BasicBlock *ReturnBB = I.getSuccessor(0);
2816 const BasicBlock *EHPadBB = I.getSuccessor(1);
2817
2818 const Function *Fn = I.getCalledFunction();
2819
2820 // FIXME: support invoking patchpoint and statepoint intrinsics.
2821 if (Fn && Fn->isIntrinsic())
2822 return false;
2823
2824 // FIXME: support whatever these are.
2825 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
2826 return false;
2827
2828 // FIXME: support control flow guard targets.
2829 if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
2830 return false;
2831
2832 // FIXME: support Windows exception handling.
2833 if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHI()))
2834 return false;
2835
2836 // FIXME: support Windows dllimport function calls and calls through
2837 // weak symbols.
2838 if (Fn && (Fn->hasDLLImportStorageClass() ||
2840 Fn->hasExternalWeakLinkage())))
2841 return false;
2842
2843 bool LowerInlineAsm = I.isInlineAsm();
2844 bool NeedEHLabel = true;
2845
2846 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2847 // the region covered by the try.
2848 MCSymbol *BeginSymbol = nullptr;
2849 if (NeedEHLabel) {
2850 MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START);
2851 BeginSymbol = Context.createTempSymbol();
2852 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
2853 }
2854
2855 if (LowerInlineAsm) {
2856 if (!translateInlineAsm(I, MIRBuilder))
2857 return false;
2858 } else if (!translateCallBase(I, MIRBuilder))
2859 return false;
2860
2861 MCSymbol *EndSymbol = nullptr;
2862 if (NeedEHLabel) {
2863 EndSymbol = Context.createTempSymbol();
2864 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
2865 }
2866
2868 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2869 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2870 BranchProbability EHPadBBProb =
2871 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2873
2874 if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2875 return false;
2876
2877 MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
2878 &ReturnMBB = getMBB(*ReturnBB);
2879 // Update successor info.
2880 addSuccessorWithProb(InvokeMBB, &ReturnMBB);
2881 for (auto &UnwindDest : UnwindDests) {
2882 UnwindDest.first->setIsEHPad();
2883 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2884 }
2885 InvokeMBB->normalizeSuccProbs();
2886
2887 if (NeedEHLabel) {
2888 assert(BeginSymbol && "Expected a begin symbol!");
2889 assert(EndSymbol && "Expected an end symbol!");
2890 MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
2891 }
2892
2893 MIRBuilder.buildBr(ReturnMBB);
2894 return true;
2895}
2896
2897bool IRTranslator::translateCallBr(const User &U,
2898 MachineIRBuilder &MIRBuilder) {
2899 // FIXME: Implement this.
2900 return false;
2901}
2902
2903bool IRTranslator::translateLandingPad(const User &U,
2904 MachineIRBuilder &MIRBuilder) {
2905 const LandingPadInst &LP = cast<LandingPadInst>(U);
2906
2907 MachineBasicBlock &MBB = MIRBuilder.getMBB();
2908
2909 MBB.setIsEHPad();
2910
2911 // If there aren't registers to copy the values into (e.g., during SjLj
2912 // exceptions), then don't bother.
2913 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
2914 if (TLI->getExceptionPointerRegister(PersonalityFn) == 0 &&
2915 TLI->getExceptionSelectorRegister(PersonalityFn) == 0)
2916 return true;
2917
2918 // If landingpad's return type is token type, we don't create DAG nodes
2919 // for its exception pointer and selector value. The extraction of exception
2920 // pointer or selector value from token type landingpads is not currently
2921 // supported.
2922 if (LP.getType()->isTokenTy())
2923 return true;
2924
2925 // Add a label to mark the beginning of the landing pad. Deletion of the
2926 // landing pad can thus be detected via the MachineModuleInfo.
2927 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
2928 .addSym(MF->addLandingPad(&MBB));
2929
2930 // If the unwinder does not preserve all registers, ensure that the
2931 // function marks the clobbered registers as used.
2933 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
2935
2936 LLT Ty = getLLTForType(*LP.getType(), *DL);
2938 MIRBuilder.buildUndef(Undef);
2939
2941 for (Type *Ty : cast<StructType>(LP.getType())->elements())
2942 Tys.push_back(getLLTForType(*Ty, *DL));
2943 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
2944
2945 // Mark exception register as live in.
2946 Register ExceptionReg = TLI->getExceptionPointerRegister(PersonalityFn);
2947 if (!ExceptionReg)
2948 return false;
2949
2950 MBB.addLiveIn(ExceptionReg);
2951 ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
2952 MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
2953
2954 Register SelectorReg = TLI->getExceptionSelectorRegister(PersonalityFn);
2955 if (!SelectorReg)
2956 return false;
2957
2958 MBB.addLiveIn(SelectorReg);
2959 Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
2960 MIRBuilder.buildCopy(PtrVReg, SelectorReg);
2961 MIRBuilder.buildCast(ResRegs[1], PtrVReg);
2962
2963 return true;
2964}
2965
2966bool IRTranslator::translateAlloca(const User &U,
2967 MachineIRBuilder &MIRBuilder) {
2968 auto &AI = cast<AllocaInst>(U);
2969
2970 if (AI.isSwiftError())
2971 return true;
2972
2973 if (AI.isStaticAlloca()) {
2974 Register Res = getOrCreateVReg(AI);
2975 int FI = getOrCreateFrameIndex(AI);
2976 MIRBuilder.buildFrameIndex(Res, FI);
2977 return true;
2978 }
2979
2980 // FIXME: support stack probing for Windows.
2982 return false;
2983
2984 // Now we're in the harder dynamic case.
2985 Register NumElts = getOrCreateVReg(*AI.getArraySize());
2986 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
2987 LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
2988 if (MRI->getType(NumElts) != IntPtrTy) {
2989 Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
2990 MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
2991 NumElts = ExtElts;
2992 }
2993
2994 Type *Ty = AI.getAllocatedType();
2995
2996 Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
2997 Register TySize =
2998 getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
2999 MIRBuilder.buildMul(AllocSize, NumElts, TySize);
3000
3001 // Round the size of the allocation up to the stack alignment size
3002 // by add SA-1 to the size. This doesn't overflow because we're computing
3003 // an address inside an alloca.
3004 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
3005 auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
3006 auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
3008 auto AlignCst =
3009 MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
3010 auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
3011
3012 Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
3013 if (Alignment <= StackAlign)
3014 Alignment = Align(1);
3015 MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
3016
3017 MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
3019 return true;
3020}
3021
3022bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
3023 // FIXME: We may need more info about the type. Because of how LLT works,
3024 // we're completely discarding the i64/double distinction here (amongst
3025 // others). Fortunately the ABIs I know of where that matters don't use va_arg
3026 // anyway but that's not guaranteed.
3027 MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
3028 {getOrCreateVReg(*U.getOperand(0)),
3029 DL->getABITypeAlign(U.getType()).value()});
3030 return true;
3031}
3032
3033bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
3035 return true;
3036
3037 auto &UI = cast<UnreachableInst>(U);
3038 // We may be able to ignore unreachable behind a noreturn call.
3040 const BasicBlock &BB = *UI.getParent();
3041 if (&UI != &BB.front()) {
3043 std::prev(BasicBlock::const_iterator(UI));
3044 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3045 if (Call->doesNotReturn())
3046 return true;
3047 }
3048 }
3049 }
3050
3051 MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>());
3052 return true;
3053}
3054
3055bool IRTranslator::translateInsertElement(const User &U,
3056 MachineIRBuilder &MIRBuilder) {
3057 // If it is a <1 x Ty> vector, use the scalar as it is
3058 // not a legal vector type in LLT.
3059 if (auto *FVT = dyn_cast<FixedVectorType>(U.getType());
3060 FVT && FVT->getNumElements() == 1)
3061 return translateCopy(U, *U.getOperand(1), MIRBuilder);
3062
3063 Register Res = getOrCreateVReg(U);
3064 Register Val = getOrCreateVReg(*U.getOperand(0));
3065 Register Elt = getOrCreateVReg(*U.getOperand(1));
3066 Register Idx = getOrCreateVReg(*U.getOperand(2));
3067 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
3068 return true;
3069}
3070
3071bool IRTranslator::translateExtractElement(const User &U,
3072 MachineIRBuilder &MIRBuilder) {
3073 // If it is a <1 x Ty> vector, use the scalar as it is
3074 // not a legal vector type in LLT.
3075 if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
3076 return translateCopy(U, *U.getOperand(0), MIRBuilder);
3077
3078 Register Res = getOrCreateVReg(U);
3079 Register Val = getOrCreateVReg(*U.getOperand(0));
3080 unsigned PreferredVecIdxWidth = TLI->getVectorIdxTy(*DL).getSizeInBits();
3081 Register Idx;
3082 if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
3083 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3084 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
3085 auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
3086 Idx = getOrCreateVReg(*NewIdxCI);
3087 }
3088 }
3089 if (!Idx)
3090 Idx = getOrCreateVReg(*U.getOperand(1));
3091 if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
3092 const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
3093 Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
3094 }
3095 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
3096 return true;
3097}
3098
3099bool IRTranslator::translateShuffleVector(const User &U,
3100 MachineIRBuilder &MIRBuilder) {
3101 // A ShuffleVector that has operates on scalable vectors is a splat vector
3102 // where the value of the splat vector is the 0th element of the first
3103 // operand, since the index mask operand is the zeroinitializer (undef and
3104 // poison are treated as zeroinitializer here).
3105 if (U.getOperand(0)->getType()->isScalableTy()) {
3106 Value *Op0 = U.getOperand(0);
3107 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
3109 getOrCreateVReg(*Op0), 0);
3110 MIRBuilder.buildSplatVector(getOrCreateVReg(U), SplatVal);
3111 return true;
3112 }
3113
3115 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
3116 Mask = SVI->getShuffleMask();
3117 else
3118 Mask = cast<ConstantExpr>(U).getShuffleMask();
3119 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
3120 MIRBuilder
3121 .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
3122 {getOrCreateVReg(*U.getOperand(0)),
3123 getOrCreateVReg(*U.getOperand(1))})
3124 .addShuffleMask(MaskAlloc);
3125 return true;
3126}
3127
3128bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
3129 const PHINode &PI = cast<PHINode>(U);
3130
3132 for (auto Reg : getOrCreateVRegs(PI)) {
3133 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
3134 Insts.push_back(MIB.getInstr());
3135 }
3136
3137 PendingPHIs.emplace_back(&PI, std::move(Insts));
3138 return true;
3139}
3140
3141bool IRTranslator::translateAtomicCmpXchg(const User &U,
3142 MachineIRBuilder &MIRBuilder) {
3143 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
3144
3145 auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);
3146
3147 auto Res = getOrCreateVRegs(I);
3148 Register OldValRes = Res[0];
3149 Register SuccessRes = Res[1];
3150 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3151 Register Cmp = getOrCreateVReg(*I.getCompareOperand());
3152 Register NewVal = getOrCreateVReg(*I.getNewValOperand());
3153
3155 OldValRes, SuccessRes, Addr, Cmp, NewVal,
3157 MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
3158 getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3159 I.getSuccessOrdering(), I.getFailureOrdering()));
3160 return true;
3161}
3162
3163bool IRTranslator::translateAtomicRMW(const User &U,
3164 MachineIRBuilder &MIRBuilder) {
3165 const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
3166 auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);
3167
3168 Register Res = getOrCreateVReg(I);
3169 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3170 Register Val = getOrCreateVReg(*I.getValOperand());
3171
3172 unsigned Opcode = 0;
3173 switch (I.getOperation()) {
3174 default:
3175 return false;
3177 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3178 break;
3179 case AtomicRMWInst::Add:
3180 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3181 break;
3182 case AtomicRMWInst::Sub:
3183 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3184 break;
3185 case AtomicRMWInst::And:
3186 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3187 break;
3189 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3190 break;
3191 case AtomicRMWInst::Or:
3192 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3193 break;
3194 case AtomicRMWInst::Xor:
3195 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3196 break;
3197 case AtomicRMWInst::Max:
3198 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3199 break;
3200 case AtomicRMWInst::Min:
3201 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3202 break;
3204 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3205 break;
3207 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3208 break;
3210 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3211 break;
3213 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3214 break;
3216 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3217 break;
3219 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3220 break;
3222 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3223 break;
3225 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3226 break;
3227 }
3228
3229 MIRBuilder.buildAtomicRMW(
3230 Opcode, Res, Addr, Val,
3231 *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3232 Flags, MRI->getType(Val), getMemOpAlign(I),
3233 I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3234 I.getOrdering()));
3235 return true;
3236}
3237
3238bool IRTranslator::translateFence(const User &U,
3239 MachineIRBuilder &MIRBuilder) {
3240 const FenceInst &Fence = cast<FenceInst>(U);
3241 MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
3242 Fence.getSyncScopeID());
3243 return true;
3244}
3245
3246bool IRTranslator::translateFreeze(const User &U,
3247 MachineIRBuilder &MIRBuilder) {
3248 const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);
3249 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));
3250
3251 assert(DstRegs.size() == SrcRegs.size() &&
3252 "Freeze with different source and destination type?");
3253
3254 for (unsigned I = 0; I < DstRegs.size(); ++I) {
3255 MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);
3256 }
3257
3258 return true;
3259}
3260
3261void IRTranslator::finishPendingPhis() {
3262#ifndef NDEBUG
3263 DILocationVerifier Verifier;
3264 GISelObserverWrapper WrapperObserver(&Verifier);
3265 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3266#endif // ifndef NDEBUG
3267 for (auto &Phi : PendingPHIs) {
3268 const PHINode *PI = Phi.first;
3269 if (PI->getType()->isEmptyTy())
3270 continue;
3271 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3272 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3273 EntryBuilder->setDebugLoc(PI->getDebugLoc());
3274#ifndef NDEBUG
3275 Verifier.setCurrentInst(PI);
3276#endif // ifndef NDEBUG
3277
3279 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3280 auto IRPred = PI->getIncomingBlock(i);
3281 ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
3282 for (auto *Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
3283 if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
3284 continue;
3285 SeenPreds.insert(Pred);
3286 for (unsigned j = 0; j < ValRegs.size(); ++j) {
3287 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3288 MIB.addUse(ValRegs[j]);
3289 MIB.addMBB(Pred);
3290 }
3291 }
3292 }
3293 }
3294}
3295
3296void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,
3297 const DILocalVariable *Variable,
3298 const DIExpression *Expression,
3299 const DebugLoc &DL,
3300 MachineIRBuilder &MIRBuilder) {
3301 assert(Variable->isValidLocationForIntrinsic(DL) &&
3302 "Expected inlined-at fields to agree");
3303 // Act as if we're handling a debug intrinsic.
3304 MIRBuilder.setDebugLoc(DL);
3305
3306 if (!V || HasArgList) {
3307 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
3308 // terminate any prior location.
3309 MIRBuilder.buildIndirectDbgValue(0, Variable, Expression);
3310 return;
3311 }
3312
3313 if (const auto *CI = dyn_cast<Constant>(V)) {
3314 MIRBuilder.buildConstDbgValue(*CI, Variable, Expression);
3315 return;
3316 }
3317
3318 if (auto *AI = dyn_cast<AllocaInst>(V);
3319 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
3320 // If the value is an alloca and the expression starts with a
3321 // dereference, track a stack slot instead of a register, as registers
3322 // may be clobbered.
3323 auto ExprOperands = Expression->getElements();
3324 auto *ExprDerefRemoved =
3325 DIExpression::get(AI->getContext(), ExprOperands.drop_front());
3326 MIRBuilder.buildFIDbgValue(getOrCreateFrameIndex(*AI), Variable,
3327 ExprDerefRemoved);
3328 return;
3329 }
3330 if (translateIfEntryValueArgument(false, V, Variable, Expression, DL,
3331 MIRBuilder))
3332 return;
3333 for (Register Reg : getOrCreateVRegs(*V)) {
3334 // FIXME: This does not handle register-indirect values at offset 0. The
3335 // direct/indirect thing shouldn't really be handled by something as
3336 // implicit as reg+noreg vs reg+imm in the first place, but it seems
3337 // pretty baked in right now.
3338 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expression);
3339 }
3340 return;
3341}
3342
3343void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,
3344 const DILocalVariable *Variable,
3345 const DIExpression *Expression,
3346 const DebugLoc &DL,
3347 MachineIRBuilder &MIRBuilder) {
3348 if (!Address || isa<UndefValue>(Address)) {
3349 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
3350 return;
3351 }
3352
3353 assert(Variable->isValidLocationForIntrinsic(DL) &&
3354 "Expected inlined-at fields to agree");
3355 auto AI = dyn_cast<AllocaInst>(Address);
3356 if (AI && AI->isStaticAlloca()) {
3357 // Static allocas are tracked at the MF level, no need for DBG_VALUE
3358 // instructions (in fact, they get ignored if they *do* exist).
3359 MF->setVariableDbgInfo(Variable, Expression,
3360 getOrCreateFrameIndex(*AI), DL);
3361 return;
3362 }
3363
3364 if (translateIfEntryValueArgument(true, Address, Variable,
3365 Expression, DL,
3366 MIRBuilder))
3367 return;
3368
3369 // A dbg.declare describes the address of a source variable, so lower it
3370 // into an indirect DBG_VALUE.
3371 MIRBuilder.setDebugLoc(DL);
3372 MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
3373 Variable, Expression);
3374 return;
3375}
3376
3377void IRTranslator::translateDbgInfo(const Instruction &Inst,
3378 MachineIRBuilder &MIRBuilder) {
3379 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
3380 if (DPLabel *DPL = dyn_cast<DPLabel>(&DR)) {
3381 MIRBuilder.setDebugLoc(DPL->getDebugLoc());
3382 assert(DPL->getLabel() && "Missing label");
3383 assert(DPL->getLabel()->isValidLocationForIntrinsic(
3384 MIRBuilder.getDebugLoc()) &&
3385 "Expected inlined-at fields to agree");
3386 MIRBuilder.buildDbgLabel(DPL->getLabel());
3387 continue;
3388 }
3389 DPValue &DPV = cast<DPValue>(DR);
3390 const DILocalVariable *Variable = DPV.getVariable();
3391 const DIExpression *Expression = DPV.getExpression();
3392 Value *V = DPV.getVariableLocationOp(0);
3393 if (DPV.isDbgDeclare())
3394 translateDbgDeclareRecord(V, DPV.hasArgList(), Variable,
3395 Expression, DPV.getDebugLoc(), MIRBuilder);
3396 else
3397 translateDbgValueRecord(V, DPV.hasArgList(), Variable,
3398 Expression, DPV.getDebugLoc(), MIRBuilder);
3399 }
3400}
3401
3402bool IRTranslator::translate(const Instruction &Inst) {
3403 CurBuilder->setDebugLoc(Inst.getDebugLoc());
3404 CurBuilder->setPCSections(Inst.getMetadata(LLVMContext::MD_pcsections));
3405
3406 if (TLI->fallBackToDAGISel(Inst))
3407 return false;
3408
3409 switch (Inst.getOpcode()) {
3410#define HANDLE_INST(NUM, OPCODE, CLASS) \
3411 case Instruction::OPCODE: \
3412 return translate##OPCODE(Inst, *CurBuilder.get());
3413#include "llvm/IR/Instruction.def"
3414 default:
3415 return false;
3416 }
3417}
3418
3419bool IRTranslator::translate(const Constant &C, Register Reg) {
3420 // We only emit constants into the entry block from here. To prevent jumpy
3421 // debug behaviour remove debug line.
3422 if (auto CurrInstDL = CurBuilder->getDL())
3423 EntryBuilder->setDebugLoc(DebugLoc());
3424
3425 if (auto CI = dyn_cast<ConstantInt>(&C))
3426 EntryBuilder->buildConstant(Reg, *CI);
3427 else if (auto CF = dyn_cast<ConstantFP>(&C))
3428 EntryBuilder->buildFConstant(Reg, *CF);
3429 else if (isa<UndefValue>(C))
3430 EntryBuilder->buildUndef(Reg);
3431 else if (isa<ConstantPointerNull>(C))
3432 EntryBuilder->buildConstant(Reg, 0);
3433 else if (auto GV = dyn_cast<GlobalValue>(&C))
3434 EntryBuilder->buildGlobalValue(Reg, GV);
3435 else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
3436 if (!isa<FixedVectorType>(CAZ->getType()))
3437 return false;
3438 // Return the scalar if it is a <1 x Ty> vector.
3439 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3440 if (NumElts == 1)
3441 return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder);
3443 for (unsigned I = 0; I < NumElts; ++I) {
3444 Constant &Elt = *CAZ->getElementValue(I);
3445 Ops.push_back(getOrCreateVReg(Elt));
3446 }
3447 EntryBuilder->buildBuildVector(Reg, Ops);
3448 } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
3449 // Return the scalar if it is a <1 x Ty> vector.
3450 if (CV->getNumElements() == 1)
3451 return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder);
3453 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3454 Constant &Elt = *CV->getElementAsConstant(i);
3455 Ops.push_back(getOrCreateVReg(Elt));
3456 }
3457 EntryBuilder->buildBuildVector(Reg, Ops);
3458 } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
3459 switch(CE->getOpcode()) {
3460#define HANDLE_INST(NUM, OPCODE, CLASS) \
3461 case Instruction::OPCODE: \
3462 return translate##OPCODE(*CE, *EntryBuilder.get());
3463#include "llvm/IR/Instruction.def"
3464 default:
3465 return false;
3466 }
3467 } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
3468 if (CV->getNumOperands() == 1)
3469 return translateCopy(C, *CV->getOperand(0), *EntryBuilder);
3471 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3472 Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
3473 }
3474 EntryBuilder->buildBuildVector(Reg, Ops);
3475 } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
3476 EntryBuilder->buildBlockAddress(Reg, BA);
3477 } else
3478 return false;
3479
3480 return true;
3481}
3482
3483bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3485 for (auto &BTB : SL->BitTestCases) {
3486 // Emit header first, if it wasn't already emitted.
3487 if (!BTB.Emitted)
3488 emitBitTestHeader(BTB, BTB.Parent);
3489
3490 BranchProbability UnhandledProb = BTB.Prob;
3491 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3492 UnhandledProb -= BTB.Cases[j].ExtraProb;
3493 // Set the current basic block to the mbb we wish to insert the code into
3494 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3495 // If all cases cover a contiguous range, it is not necessary to jump to
3496 // the default block after the last bit test fails. This is because the
3497 // range check during bit test header creation has guaranteed that every
3498 // case here doesn't go outside the range. In this case, there is no need
3499 // to perform the last bit test, as it will always be true. Instead, make
3500 // the second-to-last bit-test fall through to the target of the last bit
3501 // test, and delete the last bit test.
3502
3503 MachineBasicBlock *NextMBB;
3504 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3505 // Second-to-last bit-test with contiguous range: fall through to the
3506 // target of the final bit test.
3507 NextMBB = BTB.Cases[j + 1].TargetBB;
3508 } else if (j + 1 == ej) {
3509 // For the last bit test, fall through to Default.
3510 NextMBB = BTB.Default;
3511 } else {
3512 // Otherwise, fall through to the next bit test.
3513 NextMBB = BTB.Cases[j + 1].ThisBB;
3514 }
3515
3516 emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
3517
3518 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3519 // We need to record the replacement phi edge here that normally
3520 // happens in emitBitTestCase before we delete the case, otherwise the
3521 // phi edge will be lost.
3522 addMachineCFGPred({BTB.Parent->getBasicBlock(),
3523 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3524 MBB);
3525 // Since we're not going to use the final bit test, remove it.
3526 BTB.Cases.pop_back();
3527 break;
3528 }
3529 }
3530 // This is "default" BB. We have two jumps to it. From "header" BB and from
3531 // last "case" BB, unless the latter was skipped.
3532 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3533 BTB.Default->getBasicBlock()};
3534 addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
3535 if (!BTB.ContiguousRange) {
3536 addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
3537 }
3538 }
3539 SL->BitTestCases.clear();
3540
3541 for (auto &JTCase : SL->JTCases) {
3542 // Emit header first, if it wasn't already emitted.
3543 if (!JTCase.first.Emitted)
3544 emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
3545
3546 emitJumpTable(JTCase.second, JTCase.second.MBB);
3547 }
3548 SL->JTCases.clear();
3549
3550 for (auto &SwCase : SL->SwitchCases)
3551 emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
3552 SL->SwitchCases.clear();
3553
3554 // Check if we need to generate stack-protector guard checks.
3555 StackProtector &SP = getAnalysis<StackProtector>();
3556 if (SP.shouldEmitSDCheck(BB)) {
3557 bool FunctionBasedInstrumentation =
3559 SPDescriptor.initialize(&BB, &MBB, FunctionBasedInstrumentation);
3560 }
3561 // Handle stack protector.
3562 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3563 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3564 return false;
3565 } else if (SPDescriptor.shouldEmitStackProtector()) {
3566 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3567 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3568
3569 // Find the split point to split the parent mbb. At the same time copy all
3570 // physical registers used in the tail of parent mbb into virtual registers
3571 // before the split point and back into physical registers after the split
3572 // point. This prevents us needing to deal with Live-ins and many other
3573 // register allocation issues caused by us splitting the parent mbb. The
3574 // register allocator will clean up said virtual copies later on.
3576 ParentMBB, *MF->getSubtarget().getInstrInfo());
3577
3578 // Splice the terminator of ParentMBB into SuccessMBB.
3579 SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
3580 ParentMBB->end());
3581
3582 // Add compare/jump on neq/jump to the parent BB.
3583 if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))
3584 return false;
3585
3586 // CodeGen Failure MBB if we have not codegened it yet.
3587 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3588 if (FailureMBB->empty()) {
3589 if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))
3590 return false;
3591 }
3592
3593 // Clear the Per-BB State.
3594 SPDescriptor.resetPerBBState();
3595 }
3596 return true;
3597}
3598
3599bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3600 MachineBasicBlock *ParentBB) {
3601 CurBuilder->setInsertPt(*ParentBB, ParentBB->end());
3602 // First create the loads to the guard/stack slot for the comparison.
3604 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
3605 LLT PtrMemTy = getLLTForMVT(TLI->getPointerMemTy(*DL));
3606
3607 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3608 int FI = MFI.getStackProtectorIndex();
3609
3610 Register Guard;
3611 Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);
3612 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3613 Align Align = DL->getPrefTypeAlign(PointerType::getUnqual(M.getContext()));
3614
3615 // Generate code to load the content of the guard slot.
3616 Register GuardVal =
3617 CurBuilder
3618 ->buildLoad(PtrMemTy, StackSlotPtr,
3621 .getReg(0);
3622
3623 if (TLI->useStackGuardXorFP()) {
3624 LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3625 return false;
3626 }
3627
3628 // Retrieve guard check function, nullptr if instrumentation is inlined.
3629 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M)) {
3630 // This path is currently untestable on GlobalISel, since the only platform
3631 // that needs this seems to be Windows, and we fall back on that currently.
3632 // The code still lives here in case that changes.
3633 // Silence warning about unused variable until the code below that uses
3634 // 'GuardCheckFn' is enabled.
3635 (void)GuardCheckFn;
3636 return false;
3637#if 0
3638 // The target provides a guard check function to validate the guard value.
3639 // Generate a call to that function with the content of the guard slot as
3640 // argument.
3641 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3642 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3644 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3645 Flags.setInReg();
3646 CallLowering::ArgInfo GuardArgInfo(
3647 {GuardVal, FnTy->getParamType(0), {Flags}});
3648
3650 Info.OrigArgs.push_back(GuardArgInfo);
3651 Info.CallConv = GuardCheckFn->getCallingConv();
3652 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
3653 Info.OrigRet = {Register(), FnTy->getReturnType()};
3654 if (!CLI->lowerCall(MIRBuilder, Info)) {
3655 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
3656 return false;
3657 }
3658 return true;
3659#endif
3660 }
3661
3662 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3663 // Otherwise, emit a volatile load to retrieve the stack guard value.
3664 if (TLI->useLoadStackGuardNode()) {
3665 Guard =
3667 getStackGuard(Guard, *CurBuilder);
3668 } else {
3669 // TODO: test using android subtarget when we support @llvm.thread.pointer.
3670 const Value *IRGuard = TLI->getSDagStackGuard(M);
3671 Register GuardPtr = getOrCreateVReg(*IRGuard);
3672
3673 Guard = CurBuilder
3674 ->buildLoad(PtrMemTy, GuardPtr,
3678 .getReg(0);
3679 }
3680
3681 // Perform the comparison.
3682 auto Cmp =
3683 CurBuilder->buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Guard, GuardVal);
3684 // If the guard/stackslot do not equal, branch to failure MBB.
3685 CurBuilder->buildBrCond(Cmp, *SPD.getFailureMBB());
3686 // Otherwise branch to success MBB.
3687 CurBuilder->buildBr(*SPD.getSuccessMBB());
3688 return true;
3689}
3690
3691bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
3692 MachineBasicBlock *FailureBB) {
3693 CurBuilder->setInsertPt(*FailureBB, FailureBB->end());
3694
3695 const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
3696 const char *Name = TLI->getLibcallName(Libcall);
3697
3699 Info.CallConv = TLI->getLibcallCallingConv(Libcall);
3701 Info.OrigRet = {Register(), Type::getVoidTy(MF->getFunction().getContext()),
3702 0};
3703 if (!CLI->lowerCall(*CurBuilder, Info)) {
3704 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
3705 return false;
3706 }
3707
3708 // On PS4/PS5, the "return address" must still be within the calling
3709 // function, even if it's at the very end, so emit an explicit TRAP here.
3710 // WebAssembly needs an unreachable instruction after a non-returning call,
3711 // because the function return type can be different from __stack_chk_fail's
3712 // return type (void).
3713 const TargetMachine &TM = MF->getTarget();
3714 if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) {
3715 LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
3716 return false;
3717 }
3718 return true;
3719}
3720
3721void IRTranslator::finalizeFunction() {
3722 // Release the memory used by the different maps we
3723 // needed during the translation.
3724 PendingPHIs.clear();
3725 VMap.reset();
3726 FrameIndices.clear();
3727 MachinePreds.clear();
3728 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3729 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3730 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
3731 EntryBuilder.reset();
3732 CurBuilder.reset();
3733 FuncInfo.clear();
3734 SPDescriptor.resetPerFunctionState();
3735}
3736
3737/// Returns true if a BasicBlock \p BB within a variadic function contains a
3738/// variadic musttail call.
3739static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3740 if (!IsVarArg)
3741 return false;
3742
3743 // Walk the block backwards, because tail calls usually only appear at the end
3744 // of a block.
3745 return llvm::any_of(llvm::reverse(BB), [](const Instruction &I) {
3746 const auto *CI = dyn_cast<CallInst>(&I);
3747 return CI && CI->isMustTailCall();
3748 });
3749}
3750
3752 MF = &CurMF;
3753 const Function &F = MF->getFunction();
3755 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3756 // Set the CSEConfig and run the analysis.
3757 GISelCSEInfo *CSEInfo = nullptr;
3758 TPC = &getAnalysis<TargetPassConfig>();
3759 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
3761 : TPC->isGISelCSEEnabled();
3762 TLI = MF->getSubtarget().getTargetLowering();
3763
3764 if (EnableCSE) {
3765 EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3766 CSEInfo = &Wrapper.get(TPC->getCSEConfig());
3767 EntryBuilder->setCSEInfo(CSEInfo);
3768 CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3769 CurBuilder->setCSEInfo(CSEInfo);
3770 } else {
3771 EntryBuilder = std::make_unique<MachineIRBuilder>();
3772 CurBuilder = std::make_unique<MachineIRBuilder>();
3773 }
3774 CLI = MF->getSubtarget().getCallLowering();
3775 CurBuilder->setMF(*MF);
3776 EntryBuilder->setMF(*MF);
3777 MRI = &MF->getRegInfo();
3778 DL = &F.getParent()->getDataLayout();
3779 ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
3780 const TargetMachine &TM = MF->getTarget();
3781 TM.resetTargetOptions(F);
3782 EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
3783 FuncInfo.MF = MF;
3784 if (EnableOpts) {
3785 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3786 FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3787 } else {
3788 AA = nullptr;
3789 FuncInfo.BPI = nullptr;
3790 }
3791
3792 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
3793 MF->getFunction());
3794 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
3795 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
3796
3797 SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
3798 SL->init(*TLI, TM, *DL);
3799
3800 assert(PendingPHIs.empty() && "stale PHIs");
3801
3802 // Targets which want to use big endian can enable it using
3803 // enableBigEndian()
3804 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
3805 // Currently we don't properly handle big endian code.
3806 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3807 F.getSubprogram(), &F.getEntryBlock());
3808 R << "unable to translate in big endian mode";
3809 reportTranslationError(*MF, *TPC, *ORE, R);
3810 }
3811
3812 // Release the per-function state when we return, whether we succeeded or not.
3813 auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
3814
3815 // Setup a separate basic-block for the arguments and constants
3817 MF->push_back(EntryBB);
3818 EntryBuilder->setMBB(*EntryBB);
3819
3820 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3821 SwiftError.setFunction(CurMF);
3822 SwiftError.createEntriesInEntryBlock(DbgLoc);
3823
3824 bool IsVarArg = F.isVarArg();
3825 bool HasMustTailInVarArgFn = false;
3826
3827 // Create all blocks, in IR order, to preserve the layout.
3828 for (const BasicBlock &BB: F) {
3829 auto *&MBB = BBToMBB[&BB];
3830
3831 MBB = MF->CreateMachineBasicBlock(&BB);
3832 MF->push_back(MBB);
3833
3834 if (BB.hasAddressTaken())
3835 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
3836
3837 if (!HasMustTailInVarArgFn)
3838 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
3839 }
3840
3841 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
3842
3843 // Make our arguments/constants entry block fallthrough to the IR entry block.
3844 EntryBB->addSuccessor(&getMBB(F.front()));
3845
3846 if (CLI->fallBackToDAGISel(*MF)) {
3847 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3848 F.getSubprogram(), &F.getEntryBlock());
3849 R << "unable to lower function: " << ore::NV("Prototype", F.getType());
3850 reportTranslationError(*MF, *TPC, *ORE, R);
3851 return false;
3852 }
3853
3854 // Lower the actual args into this basic block.
3855 SmallVector<ArrayRef<Register>, 8> VRegArgs;
3856 for (const Argument &Arg: F.args()) {
3857 if (DL->getTypeStoreSize(Arg.getType()).isZero())
3858 continue; // Don't handle zero sized types.
3859 ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
3860 VRegArgs.push_back(VRegs);
3861
3862 if (Arg.hasSwiftErrorAttr()) {
3863 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
3864 SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
3865 }
3866 }
3867
3868 if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) {
3869 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3870 F.getSubprogram(), &F.getEntryBlock());
3871 R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
3872 reportTranslationError(*MF, *TPC, *ORE, R);
3873 return false;
3874 }
3875
3876 // Need to visit defs before uses when translating instructions.
3877 GISelObserverWrapper WrapperObserver;
3878 if (EnableCSE && CSEInfo)
3879 WrapperObserver.addObserver(CSEInfo);
3880 {
3882#ifndef NDEBUG
3883 DILocationVerifier Verifier;
3884 WrapperObserver.addObserver(&Verifier);
3885#endif // ifndef NDEBUG
3886 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3887 RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
3888 for (const BasicBlock *BB : RPOT) {
3889 MachineBasicBlock &MBB = getMBB(*BB);
3890 // Set the insertion point of all the following translations to
3891 // the end of this basic block.
3892 CurBuilder->setMBB(MBB);
3893 HasTailCall = false;
3894 for (const Instruction &Inst : *BB) {
3895 // If we translated a tail call in the last step, then we know
3896 // everything after the call is either a return, or something that is
3897 // handled by the call itself. (E.g. a lifetime marker or assume
3898 // intrinsic.) In this case, we should stop translating the block and
3899 // move on.
3900 if (HasTailCall)
3901 break;
3902#ifndef NDEBUG
3903 Verifier.setCurrentInst(&Inst);
3904#endif // ifndef NDEBUG
3905
3906 // Translate any debug-info attached to the instruction.
3907 translateDbgInfo(Inst, *CurBuilder.get());
3908
3909 if (translate(Inst))
3910 continue;
3911
3912 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3913 Inst.getDebugLoc(), BB);
3914 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
3915
3916 if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
3917 std::string InstStrStorage;
3918 raw_string_ostream InstStr(InstStrStorage);
3919 InstStr << Inst;
3920
3921 R << ": '" << InstStr.str() << "'";
3922 }
3923
3924 reportTranslationError(*MF, *TPC, *ORE, R);
3925 return false;
3926 }
3927
3928 if (!finalizeBasicBlock(*BB, MBB)) {
3929 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3930 BB->getTerminator()->getDebugLoc(), BB);
3931 R << "unable to translate basic block";
3932 reportTranslationError(*MF, *TPC, *ORE, R);
3933 return false;
3934 }
3935 }
3936#ifndef NDEBUG
3937 WrapperObserver.removeObserver(&Verifier);
3938#endif
3939 }
3940
3941 finishPendingPhis();
3942
3943 SwiftError.propagateVRegs();
3944
3945 // Merge the argument lowering and constants block with its single
3946 // successor, the LLVM-IR entry block. We want the basic block to
3947 // be maximal.
3948 assert(EntryBB->succ_size() == 1 &&
3949 "Custom BB used for lowering should have only one successor");
3950 // Get the successor of the current entry block.
3951 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
3952 assert(NewEntryBB.pred_size() == 1 &&
3953 "LLVM-IR entry block has a predecessor!?");
3954 // Move all the instruction from the current entry block to the
3955 // new entry block.
3956 NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
3957 EntryBB->end());
3958
3959 // Update the live-in information for the new entry block.
3960 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
3961 NewEntryBB.addLiveIn(LiveIn);
3962 NewEntryBB.sortUniqueLiveIns();
3963
3964 // Get rid of the now empty basic block.
3965 EntryBB->removeSuccessor(&NewEntryBB);
3966 MF->remove(EntryBB);
3967 MF->deleteMachineBasicBlock(EntryBB);
3968
3969 assert(&MF->front() == &NewEntryBB &&
3970 "New entry wasn't next in the list of basic block!");
3971
3972 // Initialize stack protector information.
3973 StackProtector &SP = getAnalysis<StackProtector>();
3975
3976 return false;
3977}
unsigned SubReg
#define Success
aarch64 promote const
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Provides analysis for continuously CSEing during GISel passes.
This file implements a version of MachineIRBuilder which CSEs insts within a MachineBasicBlock.
This file describes how to lower LLVM calls to machine code calls.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:301
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
std::string Name
uint64_t Size
This contains common code to allow clients to notify changes to machine instr.
const HexagonInstrInfo * TII
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB)
Returns true if a BasicBlock BB within a variadic function contains a variadic musttail call.
static unsigned getConvOpcode(Intrinsic::ID ID)
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
static unsigned getConstrainedOpcode(Intrinsic::ID ID)
IRTranslator LLVM IR MI
#define DEBUG_TYPE
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
static bool isValInBlock(const Value *V, const BasicBlock *BB)
static bool isSwiftError(const Value *V)
This file declares the IRTranslator pass.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file describes how to lower LLVM inline asm to machine code INLINEASM.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:81
Implement a low-level type suitable for MachineInstr level instruction selection.
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
This file contains the declarations for metadata subclasses.
uint64_t High
IntegerType * Int32Ty
LLVMContext & Context
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
verify safepoint Safepoint IR Verifier
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
Value * RHS
Value * LHS
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:76
an instruction to allocate memory on the stack
Definition: Instructions.h:59
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:157
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:132
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:107
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:125
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:103
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
An immutable pass that tracks lazily created AssumptionCache objects.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
@ Add
*p = old + v
Definition: Instructions.h:764
@ FAdd
*p = old + v
Definition: Instructions.h:785
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:778
@ Or
*p = old | v
Definition: Instructions.h:772
@ Sub
*p = old - v
Definition: Instructions.h:766
@ And
*p = old & v
Definition: Instructions.h:768
@ Xor
*p = old ^ v
Definition: Instructions.h:774
@ FSub
*p = old - v
Definition: Instructions.h:788
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:800
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:776
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:782
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:796
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:780
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:792
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:804
@ Nand
*p = ~(old & v)
Definition: Instructions.h:770
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
Definition: Attributes.h:842
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:349
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
Definition: BasicBlock.h:639
InstListType::const_iterator const_iterator
Definition: BasicBlock.h:165
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:347
const Instruction & front() const
Definition: BasicBlock.h:452
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:366
const Instruction & back() const
Definition: BasicBlock.h:454
Legacy analysis pass which computes BlockFrequencyInfo.
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Legacy analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1455
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1770
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2357
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1703
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1623
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2333
Value * getCalledOperand() const
Definition: InstrTypes.h:1696
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1648
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1629
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:2241
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1639
unsigned arg_size() const
Definition: InstrTypes.h:1646
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1780
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
bool isMustTailCall() const
bool checkReturnTypeForCallConv(MachineFunction &MF) const
Toplevel function to check the return type based on the target calling convention.
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
Definition: CallLowering.h:546
virtual bool enableBigEndian() const
For targets which want to use big-endian can enable it with enableBigEndian() hook.
Definition: CallLowering.h:595
virtual bool supportSwiftError() const
Definition: CallLowering.h:449
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
Definition: CallLowering.h:514
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const
This hook must be implemented to lower the given call instruction, including argument and return valu...
Definition: CallLowering.h:558
virtual bool fallBackToDAGISel(const MachineFunction &MF) const
Definition: CallLowering.h:532
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:955
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:965
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:982
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:994
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:995
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:988
@ ICMP_EQ
equal
Definition: InstrTypes.h:986
@ ICMP_NE
not equal
Definition: InstrTypes.h:987
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:991
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:967
bool isFPPredicate() const
Definition: InstrTypes.h:1083
bool isIntPredicate() const
Definition: InstrTypes.h:1084
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:204
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:153
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:144
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:417
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Records a position in IR for a source label (DILabel).
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:410
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:720
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:905
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:504
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:672
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
Align getPointerABIAlignment(unsigned AS) const
Layout pointer alignment.
Definition: DataLayout.cpp:742
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
Value * getValue(unsigned OpIdx=0) const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
A debug info location.
Definition: DebugLoc.h:33
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
An instruction for ordering other memory operations.
Definition: Instructions.h:460
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:498
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:487
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:692
BranchProbabilityInfo * BPI
void clear()
clear - Clear out all the function-specific state.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:178
const BasicBlock & getEntryBlock() const
Definition: Function.h:782
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1828
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:677
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1874
const Function & getFunction() const
Definition: Function.h:160
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:235
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:342
The actual analysis pass wrapper.
Definition: CSEInfo.h:222
Simple wrapper that does the following.
Definition: CSEInfo.h:204
The CSE Analysis object.
Definition: CSEInfo.h:69
Abstract class that contains various methods for clients to notify about changes.
Simple wrapper observer that takes several observers, and calls each one for each event.
void removeObserver(GISelChangeObserver *O)
void addObserver(GISelChangeObserver *O)
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:566
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:528
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:278
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
bool isTailCall(const MachineInstr &MI) const override
This instruction compares its operands according to the predicate given to the constructor.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
IRTranslator(CodeGenOptLevel OptLevel=CodeGenOptLevel::None)
static char ID
Definition: IRTranslator.h:68
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Indirect Branch Instruction.
bool lowerInlineAsm(MachineIRBuilder &MIRBuilder, const CallBase &CB, std::function< ArrayRef< Register >(const Value &Val)> GetOrCreateVRegs) const
Lower the given inline asm call instruction GetOrCreateVRegs is a callback to materialize a register ...
This instruction inserts a struct field of array element value into an aggregate value.
iterator_range< simple_ilist< DbgRecord >::iterator > getDbgRecordRange() const
Return a range over the DbgRecords attached to this instruction.
Definition: Instruction.h:83
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:453
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:340
const BasicBlock * getParent() const
Definition: Instruction.h:151
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:358
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1704
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:251
bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
Invoke instruction.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
Definition: LowLevelType.h:214
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelType.h:159
constexpr bool isVector() const
Definition: LowLevelType.h:148
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelType.h:57
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:193
constexpr bool isPointer() const
Definition: LowLevelType.h:149
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelType.h:100
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
Definition: LowLevelType.h:178
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:184
Value * getPointerOperand()
Definition: Instructions.h:280
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:245
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:255
static LocationSize precise(uint64_t Value)
Context object for machine code objects.
Definition: MCContext.h:76
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
unsigned pred_size() const
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
void setAddressTakenIRBlock(BasicBlock *BB)
Set this block to reflect that it corresponds to an IR-level basic block with a BlockAddress.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
void setHasMustTailInVarArgFunc(bool B)
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
ArrayRef< int > allocateShuffleMask(ArrayRef< int > Mask)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
void push_back(MachineBasicBlock *MBB)
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
void deleteMachineBasicBlock(MachineBasicBlock *MBB)
DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
void remove(iterator MBBI)
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
Helper class to build MachineInstr.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPEXT Op.
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, const SrcOp &Src2, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FMA Op0, Op1, Op2.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr,...
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
MachineInstrBuilder buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, Align Alignment)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
MachineInstrBuilder buildFIDbgValue(int FI, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in th...
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
const DebugLoc & getDebugLoc()
Get the current instruction's debug location.
MachineInstrBuilder buildFFrexp(const DstOp &Fract, const DstOp &Exp, const SrcOp &Src, std::optional< unsigned > Flags=std::nullopt)
Build and insert Fract, Exp = G_FFREXP Src.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef< int > Mask)
Build and insert Res = G_SHUFFLE_VECTOR Src1, Src2, Mask.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildPrefetch(const SrcOp &Addr, unsigned RW, unsigned Locality, unsigned CacheType, MachineMemOperand &MMO)
Build and insert G_PREFETCH Addr, RW, Locality, CacheType.
const DataLayout & getDataLayout() const
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Val)
Build and insert Res = G_SPLAT_VECTOR Val.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:69
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
static uint32_t copyFlagsFromInstruction(const Instruction &I)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:554
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MCContext & getContext() const
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
Representation for a specific memory location.
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
The optimization diagnostic interface.
Diagnostic information for missed-optimization remarks.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
A simple RAII based Delegate installer.
A simple RAII based Observer installer.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Definition: Register.h:110
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents the LLVM 'select' instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
void initialize(const BasicBlock *BB, MachineBasicBlock *MBB, bool FunctionBasedInstrumentation)
Initialize the stack protector descriptor structure for a new basic block.
MachineBasicBlock * getSuccessMBB()
void resetPerBBState()
Reset state that changes when we handle different basic blocks.
void resetPerFunctionState()
Reset state that only changes when we switch functions.
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitStackProtector() const
Returns true if all fields of the stack protector descriptor are initialized implying that we should/...
bool shouldEmitFunctionBasedCheckStackProtector() const
bool shouldEmitSDCheck(const BasicBlock &BB) const
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
An instruction for storing to memory.
Definition: Instructions.h:317
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Multiway switch.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual unsigned getVaListSizeInBits(const DataLayout &DL) const
Returns the size of the platform's va_list object.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool fallBackToDAGISel(const Instruction &Inst) const
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
const Triple & getTargetTriple() const
TargetOptions Options
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
Target-Independent Code Generator Pass Configuration Options.
virtual std::unique_ptr< CSEConfigBase > getCSEConfig() const
Returns the CSEConfig object to use for the current optimization level.
virtual bool isGISelCSEEnabled() const
Check whether continuous CSE should be enabled in GISel passes.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const InlineAsmLowering * getInlineAsmLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const CallLowering * getCallLowering() const
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetLowering * getTargetLowering() const
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:608
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:302
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:295
static IntegerType * getInt32Ty(LLVMContext &C)
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:225
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Value * getOperand(unsigned i) const
Definition: User.h:169
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:693
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
constexpr bool isZero() const
Definition: TypeSize.h:156
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:316
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:678
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:86
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:821
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Undef
Value of the register doesn't matter.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1522
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
@ CE
Windows NT (Windows on ARM)
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< PhiNode * > Phi
Definition: RDFGraph.h:390
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
@ Offset
Definition: DWP.cpp:456
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Definition: STLExtras.h:2415
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
gep_type_iterator gep_type_end(const User *GEP)
MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition: Local.h:241
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:428
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
Definition: Analysis.cpp:141
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
@ Global
Append to llvm.global_dtors.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:1071
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1963
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ FMul
Product of floats.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1833
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
unsigned succ_size(const MachineBasicBlock *BB)
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Pair of physical register and lane mask.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static bool canHandle(const Instruction *I, const TargetLibraryInfo &TLI)
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
struct PredInfoPair PredInfo