LLVM 19.0.0git
IRTranslator.cpp
Go to the documentation of this file.
1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallSet.h"
21#include "llvm/Analysis/Loads.h"
52#include "llvm/IR/BasicBlock.h"
53#include "llvm/IR/CFG.h"
54#include "llvm/IR/Constant.h"
55#include "llvm/IR/Constants.h"
56#include "llvm/IR/DataLayout.h"
59#include "llvm/IR/Function.h"
61#include "llvm/IR/InlineAsm.h"
62#include "llvm/IR/InstrTypes.h"
65#include "llvm/IR/Intrinsics.h"
66#include "llvm/IR/IntrinsicsAMDGPU.h"
67#include "llvm/IR/LLVMContext.h"
68#include "llvm/IR/Metadata.h"
70#include "llvm/IR/Statepoint.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
75#include "llvm/MC/MCContext.h"
76#include "llvm/Pass.h"
79#include "llvm/Support/Debug.h"
87#include <algorithm>
88#include <cassert>
89#include <cstdint>
90#include <iterator>
91#include <optional>
92#include <string>
93#include <utility>
94#include <vector>
95
96#define DEBUG_TYPE "irtranslator"
97
98using namespace llvm;
99
100static cl::opt<bool>
101 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
102 cl::desc("Should enable CSE in irtranslator"),
103 cl::Optional, cl::init(false));
104char IRTranslator::ID = 0;
105
106INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
107 false, false)
115
120 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
121
122 // Print the function name explicitly if we don't have a debug location (which
123 // makes the diagnostic less useful) or if we're going to emit a raw error.
124 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
125 R << (" (in function: " + MF.getName() + ")").str();
126
127 if (TPC.isGlobalISelAbortEnabled())
128 report_fatal_error(Twine(R.getMsg()));
129 else
130 ORE.emit(R);
131}
132
134 : MachineFunctionPass(ID), OptLevel(optlevel) {}
135
136#ifndef NDEBUG
137namespace {
138/// Verify that every instruction created has the same DILocation as the
139/// instruction being translated.
140class DILocationVerifier : public GISelChangeObserver {
141 const Instruction *CurrInst = nullptr;
142
143public:
144 DILocationVerifier() = default;
145 ~DILocationVerifier() = default;
146
147 const Instruction *getCurrentInst() const { return CurrInst; }
148 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
149
150 void erasingInstr(MachineInstr &MI) override {}
151 void changingInstr(MachineInstr &MI) override {}
152 void changedInstr(MachineInstr &MI) override {}
153
154 void createdInstr(MachineInstr &MI) override {
155 assert(getCurrentInst() && "Inserted instruction without a current MI");
156
157 // Only print the check message if we're actually checking it.
158#ifndef NDEBUG
159 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
160 << " was copied to " << MI);
161#endif
162 // We allow insts in the entry block to have no debug loc because
163 // they could have originated from constants, and we don't want a jumpy
164 // debug experience.
165 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
166 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
167 (MI.isDebugInstr())) &&
168 "Line info was not transferred to all instructions");
169 }
170};
171} // namespace
172#endif // ifndef NDEBUG
173
174
180 if (OptLevel != CodeGenOptLevel::None) {
183 }
188}
189
191IRTranslator::allocateVRegs(const Value &Val) {
192 auto VRegsIt = VMap.findVRegs(Val);
193 if (VRegsIt != VMap.vregs_end())
194 return *VRegsIt->second;
195 auto *Regs = VMap.getVRegs(Val);
196 auto *Offsets = VMap.getOffsets(Val);
197 SmallVector<LLT, 4> SplitTys;
198 computeValueLLTs(*DL, *Val.getType(), SplitTys,
199 Offsets->empty() ? Offsets : nullptr);
200 for (unsigned i = 0; i < SplitTys.size(); ++i)
201 Regs->push_back(0);
202 return *Regs;
203}
204
205ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
206 auto VRegsIt = VMap.findVRegs(Val);
207 if (VRegsIt != VMap.vregs_end())
208 return *VRegsIt->second;
209
210 if (Val.getType()->isVoidTy())
211 return *VMap.getVRegs(Val);
212
213 // Create entry for this type.
214 auto *VRegs = VMap.getVRegs(Val);
215 auto *Offsets = VMap.getOffsets(Val);
216
217 if (!Val.getType()->isTokenTy())
218 assert(Val.getType()->isSized() &&
219 "Don't know how to create an empty vreg");
220
221 SmallVector<LLT, 4> SplitTys;
222 computeValueLLTs(*DL, *Val.getType(), SplitTys,
223 Offsets->empty() ? Offsets : nullptr);
224
225 if (!isa<Constant>(Val)) {
226 for (auto Ty : SplitTys)
227 VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
228 return *VRegs;
229 }
230
231 if (Val.getType()->isAggregateType()) {
232 // UndefValue, ConstantAggregateZero
233 auto &C = cast<Constant>(Val);
234 unsigned Idx = 0;
235 while (auto Elt = C.getAggregateElement(Idx++)) {
236 auto EltRegs = getOrCreateVRegs(*Elt);
237 llvm::copy(EltRegs, std::back_inserter(*VRegs));
238 }
239 } else {
240 assert(SplitTys.size() == 1 && "unexpectedly split LLT");
241 VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
242 bool Success = translate(cast<Constant>(Val), VRegs->front());
243 if (!Success) {
244 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
246 &MF->getFunction().getEntryBlock());
247 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
248 reportTranslationError(*MF, *TPC, *ORE, R);
249 return *VRegs;
250 }
251 }
252
253 return *VRegs;
254}
255
256int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
257 auto MapEntry = FrameIndices.find(&AI);
258 if (MapEntry != FrameIndices.end())
259 return MapEntry->second;
260
261 uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
262 uint64_t Size =
263 ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
264
265 // Always allocate at least one byte.
266 Size = std::max<uint64_t>(Size, 1u);
267
268 int &FI = FrameIndices[&AI];
269 FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
270 return FI;
271}
272
273Align IRTranslator::getMemOpAlign(const Instruction &I) {
274 if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
275 return SI->getAlign();
276 if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
277 return LI->getAlign();
278 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
279 return AI->getAlign();
280 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
281 return AI->getAlign();
282
283 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
284 R << "unable to translate memop: " << ore::NV("Opcode", &I);
285 reportTranslationError(*MF, *TPC, *ORE, R);
286 return Align(1);
287}
288
289MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
290 MachineBasicBlock *&MBB = BBToMBB[&BB];
291 assert(MBB && "BasicBlock was not encountered before");
292 return *MBB;
293}
294
295void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
296 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
297 MachinePreds[Edge].push_back(NewPred);
298}
299
300bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
301 MachineIRBuilder &MIRBuilder) {
302 // Get or create a virtual register for each value.
303 // Unless the value is a Constant => loadimm cst?
304 // or inline constant each time?
305 // Creation of a virtual register needs to have a size.
306 Register Op0 = getOrCreateVReg(*U.getOperand(0));
307 Register Op1 = getOrCreateVReg(*U.getOperand(1));
308 Register Res = getOrCreateVReg(U);
309 uint32_t Flags = 0;
310 if (isa<Instruction>(U)) {
311 const Instruction &I = cast<Instruction>(U);
313 }
314
315 MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
316 return true;
317}
318
319bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
320 MachineIRBuilder &MIRBuilder) {
321 Register Op0 = getOrCreateVReg(*U.getOperand(0));
322 Register Res = getOrCreateVReg(U);
323 uint32_t Flags = 0;
324 if (isa<Instruction>(U)) {
325 const Instruction &I = cast<Instruction>(U);
327 }
328 MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);
329 return true;
330}
331
332bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
333 return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
334}
335
336bool IRTranslator::translateCompare(const User &U,
337 MachineIRBuilder &MIRBuilder) {
338 auto *CI = dyn_cast<CmpInst>(&U);
339 Register Op0 = getOrCreateVReg(*U.getOperand(0));
340 Register Op1 = getOrCreateVReg(*U.getOperand(1));
341 Register Res = getOrCreateVReg(U);
342 CmpInst::Predicate Pred =
343 CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
344 cast<ConstantExpr>(U).getPredicate());
345 if (CmpInst::isIntPredicate(Pred))
346 MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
347 else if (Pred == CmpInst::FCMP_FALSE)
348 MIRBuilder.buildCopy(
349 Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
350 else if (Pred == CmpInst::FCMP_TRUE)
351 MIRBuilder.buildCopy(
352 Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
353 else {
354 uint32_t Flags = 0;
355 if (CI)
357 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
358 }
359
360 return true;
361}
362
363bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
364 const ReturnInst &RI = cast<ReturnInst>(U);
365 const Value *Ret = RI.getReturnValue();
366 if (Ret && DL->getTypeStoreSize(Ret->getType()).isZero())
367 Ret = nullptr;
368
369 ArrayRef<Register> VRegs;
370 if (Ret)
371 VRegs = getOrCreateVRegs(*Ret);
372
373 Register SwiftErrorVReg = 0;
374 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
375 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
376 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
377 }
378
379 // The target may mess up with the insertion point, but
380 // this is not important as a return is the last instruction
381 // of the block anyway.
382 return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
383}
384
385void IRTranslator::emitBranchForMergedCondition(
387 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
388 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
389 // If the leaf of the tree is a comparison, merge the condition into
390 // the caseblock.
391 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
392 CmpInst::Predicate Condition;
393 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
394 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
395 } else {
396 const FCmpInst *FC = cast<FCmpInst>(Cond);
397 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
398 }
399
400 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),
401 BOp->getOperand(1), nullptr, TBB, FBB, CurBB,
402 CurBuilder->getDebugLoc(), TProb, FProb);
403 SL->SwitchCases.push_back(CB);
404 return;
405 }
406
407 // Create a CaseBlock record representing this branch.
410 Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),
411 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
412 SL->SwitchCases.push_back(CB);
413}
414
415static bool isValInBlock(const Value *V, const BasicBlock *BB) {
416 if (const Instruction *I = dyn_cast<Instruction>(V))
417 return I->getParent() == BB;
418 return true;
419}
420
421void IRTranslator::findMergedConditions(
423 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
425 BranchProbability FProb, bool InvertCond) {
426 using namespace PatternMatch;
427 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
428 "Expected Opc to be AND/OR");
429 // Skip over not part of the tree and remember to invert op and operands at
430 // next level.
431 Value *NotCond;
432 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
433 isValInBlock(NotCond, CurBB->getBasicBlock())) {
434 findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
435 !InvertCond);
436 return;
437 }
438
439 const Instruction *BOp = dyn_cast<Instruction>(Cond);
440 const Value *BOpOp0, *BOpOp1;
441 // Compute the effective opcode for Cond, taking into account whether it needs
442 // to be inverted, e.g.
443 // and (not (or A, B)), C
444 // gets lowered as
445 // and (and (not A, not B), C)
447 if (BOp) {
448 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
449 ? Instruction::And
450 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
451 ? Instruction::Or
453 if (InvertCond) {
454 if (BOpc == Instruction::And)
455 BOpc = Instruction::Or;
456 else if (BOpc == Instruction::Or)
457 BOpc = Instruction::And;
458 }
459 }
460
461 // If this node is not part of the or/and tree, emit it as a branch.
462 // Note that all nodes in the tree should have same opcode.
463 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
464 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
465 !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||
466 !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {
467 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
468 InvertCond);
469 return;
470 }
471
472 // Create TmpBB after CurBB.
473 MachineFunction::iterator BBI(CurBB);
474 MachineBasicBlock *TmpBB =
476 CurBB->getParent()->insert(++BBI, TmpBB);
477
478 if (Opc == Instruction::Or) {
479 // Codegen X | Y as:
480 // BB1:
481 // jmp_if_X TBB
482 // jmp TmpBB
483 // TmpBB:
484 // jmp_if_Y TBB
485 // jmp FBB
486 //
487
488 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
489 // The requirement is that
490 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
491 // = TrueProb for original BB.
492 // Assuming the original probabilities are A and B, one choice is to set
493 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
494 // A/(1+B) and 2B/(1+B). This choice assumes that
495 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
496 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
497 // TmpBB, but the math is more complicated.
498
499 auto NewTrueProb = TProb / 2;
500 auto NewFalseProb = TProb / 2 + FProb;
501 // Emit the LHS condition.
502 findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
503 NewFalseProb, InvertCond);
504
505 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
506 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
507 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
508 // Emit the RHS condition into TmpBB.
509 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
510 Probs[1], InvertCond);
511 } else {
512 assert(Opc == Instruction::And && "Unknown merge op!");
513 // Codegen X & Y as:
514 // BB1:
515 // jmp_if_X TmpBB
516 // jmp FBB
517 // TmpBB:
518 // jmp_if_Y TBB
519 // jmp FBB
520 //
521 // This requires creation of TmpBB after CurBB.
522
523 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
524 // The requirement is that
525 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
526 // = FalseProb for original BB.
527 // Assuming the original probabilities are A and B, one choice is to set
528 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
529 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
530 // TrueProb for BB1 * FalseProb for TmpBB.
531
532 auto NewTrueProb = TProb + FProb / 2;
533 auto NewFalseProb = FProb / 2;
534 // Emit the LHS condition.
535 findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
536 NewFalseProb, InvertCond);
537
538 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
539 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
540 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
541 // Emit the RHS condition into TmpBB.
542 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
543 Probs[1], InvertCond);
544 }
545}
546
547bool IRTranslator::shouldEmitAsBranches(
548 const std::vector<SwitchCG::CaseBlock> &Cases) {
549 // For multiple cases, it's better to emit as branches.
550 if (Cases.size() != 2)
551 return true;
552
553 // If this is two comparisons of the same values or'd or and'd together, they
554 // will get folded into a single comparison, so don't emit two blocks.
555 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
556 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
557 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
558 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
559 return false;
560 }
561
562 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
563 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
564 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
565 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
566 isa<Constant>(Cases[0].CmpRHS) &&
567 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
568 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
569 Cases[0].TrueBB == Cases[1].ThisBB)
570 return false;
571 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
572 Cases[0].FalseBB == Cases[1].ThisBB)
573 return false;
574 }
575
576 return true;
577}
578
579bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
580 const BranchInst &BrInst = cast<BranchInst>(U);
581 auto &CurMBB = MIRBuilder.getMBB();
582 auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));
583
584 if (BrInst.isUnconditional()) {
585 // If the unconditional target is the layout successor, fallthrough.
586 if (OptLevel == CodeGenOptLevel::None ||
587 !CurMBB.isLayoutSuccessor(Succ0MBB))
588 MIRBuilder.buildBr(*Succ0MBB);
589
590 // Link successors.
591 for (const BasicBlock *Succ : successors(&BrInst))
592 CurMBB.addSuccessor(&getMBB(*Succ));
593 return true;
594 }
595
596 // If this condition is one of the special cases we handle, do special stuff
597 // now.
598 const Value *CondVal = BrInst.getCondition();
599 MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));
600
601 // If this is a series of conditions that are or'd or and'd together, emit
602 // this as a sequence of branches instead of setcc's with and/or operations.
603 // As long as jumps are not expensive (exceptions for multi-use logic ops,
604 // unpredictable branches, and vector extracts because those jumps are likely
605 // expensive for any target), this should improve performance.
606 // For example, instead of something like:
607 // cmp A, B
608 // C = seteq
609 // cmp D, E
610 // F = setle
611 // or C, F
612 // jnz foo
613 // Emit:
614 // cmp A, B
615 // je foo
616 // cmp D, E
617 // jle foo
618 using namespace PatternMatch;
619 const Instruction *CondI = dyn_cast<Instruction>(CondVal);
620 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
621 !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {
623 Value *Vec;
624 const Value *BOp0, *BOp1;
625 if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
626 Opcode = Instruction::And;
627 else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
628 Opcode = Instruction::Or;
629
630 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
631 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
632 findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
633 getEdgeProbability(&CurMBB, Succ0MBB),
634 getEdgeProbability(&CurMBB, Succ1MBB),
635 /*InvertCond=*/false);
636 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
637
638 // Allow some cases to be rejected.
639 if (shouldEmitAsBranches(SL->SwitchCases)) {
640 // Emit the branch for this block.
641 emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
642 SL->SwitchCases.erase(SL->SwitchCases.begin());
643 return true;
644 }
645
646 // Okay, we decided not to do this, remove any inserted MBB's and clear
647 // SwitchCases.
648 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
649 MF->erase(SL->SwitchCases[I].ThisBB);
650
651 SL->SwitchCases.clear();
652 }
653 }
654
655 // Create a CaseBlock record representing this branch.
656 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
658 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
659 CurBuilder->getDebugLoc());
660
661 // Use emitSwitchCase to actually insert the fast branch sequence for this
662 // cond branch.
663 emitSwitchCase(CB, &CurMBB, *CurBuilder);
664 return true;
665}
666
667void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
669 BranchProbability Prob) {
670 if (!FuncInfo.BPI) {
671 Src->addSuccessorWithoutProb(Dst);
672 return;
673 }
674 if (Prob.isUnknown())
675 Prob = getEdgeProbability(Src, Dst);
676 Src->addSuccessor(Dst, Prob);
677}
678
680IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
681 const MachineBasicBlock *Dst) const {
682 const BasicBlock *SrcBB = Src->getBasicBlock();
683 const BasicBlock *DstBB = Dst->getBasicBlock();
684 if (!FuncInfo.BPI) {
685 // If BPI is not available, set the default probability as 1 / N, where N is
686 // the number of successors.
687 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
688 return BranchProbability(1, SuccSize);
689 }
690 return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
691}
692
693bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
694 using namespace SwitchCG;
695 // Extract cases from the switch.
696 const SwitchInst &SI = cast<SwitchInst>(U);
697 BranchProbabilityInfo *BPI = FuncInfo.BPI;
698 CaseClusterVector Clusters;
699 Clusters.reserve(SI.getNumCases());
700 for (const auto &I : SI.cases()) {
701 MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
702 assert(Succ && "Could not find successor mbb in mapping");
703 const ConstantInt *CaseVal = I.getCaseValue();
704 BranchProbability Prob =
705 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
706 : BranchProbability(1, SI.getNumCases() + 1);
707 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
708 }
709
710 MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
711
712 // Cluster adjacent cases with the same destination. We do this at all
713 // optimization levels because it's cheap to do and will make codegen faster
714 // if there are many clusters.
715 sortAndRangeify(Clusters);
716
717 MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
718
719 // If there is only the default destination, jump there directly.
720 if (Clusters.empty()) {
721 SwitchMBB->addSuccessor(DefaultMBB);
722 if (DefaultMBB != SwitchMBB->getNextNode())
723 MIB.buildBr(*DefaultMBB);
724 return true;
725 }
726
727 SL->findJumpTables(Clusters, &SI, std::nullopt, DefaultMBB, nullptr, nullptr);
728 SL->findBitTestClusters(Clusters, &SI);
729
730 LLVM_DEBUG({
731 dbgs() << "Case clusters: ";
732 for (const CaseCluster &C : Clusters) {
733 if (C.Kind == CC_JumpTable)
734 dbgs() << "JT:";
735 if (C.Kind == CC_BitTests)
736 dbgs() << "BT:";
737
738 C.Low->getValue().print(dbgs(), true);
739 if (C.Low != C.High) {
740 dbgs() << '-';
741 C.High->getValue().print(dbgs(), true);
742 }
743 dbgs() << ' ';
744 }
745 dbgs() << '\n';
746 });
747
748 assert(!Clusters.empty());
749 SwitchWorkList WorkList;
750 CaseClusterIt First = Clusters.begin();
751 CaseClusterIt Last = Clusters.end() - 1;
752 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
753 WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
754
755 while (!WorkList.empty()) {
756 SwitchWorkListItem W = WorkList.pop_back_val();
757
758 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
759 // For optimized builds, lower large range as a balanced binary tree.
760 if (NumClusters > 3 &&
762 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
763 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB, MIB);
764 continue;
765 }
766
767 if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
768 return false;
769 }
770 return true;
771}
772
773void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
775 Value *Cond, MachineBasicBlock *SwitchMBB,
776 MachineIRBuilder &MIB) {
777 using namespace SwitchCG;
778 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
779 "Clusters not sorted?");
780 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
781
782 auto [LastLeft, FirstRight, LeftProb, RightProb] =
783 SL->computeSplitWorkItemInfo(W);
784
785 // Use the first element on the right as pivot since we will make less-than
786 // comparisons against it.
787 CaseClusterIt PivotCluster = FirstRight;
788 assert(PivotCluster > W.FirstCluster);
789 assert(PivotCluster <= W.LastCluster);
790
791 CaseClusterIt FirstLeft = W.FirstCluster;
792 CaseClusterIt LastRight = W.LastCluster;
793
794 const ConstantInt *Pivot = PivotCluster->Low;
795
796 // New blocks will be inserted immediately after the current one.
798 ++BBI;
799
800 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
801 // we can branch to its destination directly if it's squeezed exactly in
802 // between the known lower bound and Pivot - 1.
803 MachineBasicBlock *LeftMBB;
804 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
805 FirstLeft->Low == W.GE &&
806 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
807 LeftMBB = FirstLeft->MBB;
808 } else {
809 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
810 FuncInfo.MF->insert(BBI, LeftMBB);
811 WorkList.push_back(
812 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
813 }
814
815 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
816 // single cluster, RHS.Low == Pivot, and we can branch to its destination
817 // directly if RHS.High equals the current upper bound.
818 MachineBasicBlock *RightMBB;
819 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
820 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
821 RightMBB = FirstRight->MBB;
822 } else {
823 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
824 FuncInfo.MF->insert(BBI, RightMBB);
825 WorkList.push_back(
826 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
827 }
828
829 // Create the CaseBlock record that will be used to lower the branch.
830 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
831 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
832 RightProb);
833
834 if (W.MBB == SwitchMBB)
835 emitSwitchCase(CB, SwitchMBB, MIB);
836 else
837 SL->SwitchCases.push_back(CB);
838}
839
840void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
842 // Emit the code for the jump table
843 assert(JT.Reg != -1U && "Should lower JT Header first!");
845 MIB.setMBB(*MBB);
846 MIB.setDebugLoc(CurBuilder->getDebugLoc());
847
849 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
850
851 auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
852 MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
853}
854
855bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
857 MachineBasicBlock *HeaderBB) {
858 MachineIRBuilder MIB(*HeaderBB->getParent());
859 MIB.setMBB(*HeaderBB);
860 MIB.setDebugLoc(CurBuilder->getDebugLoc());
861
862 const Value &SValue = *JTH.SValue;
863 // Subtract the lowest switch case value from the value being switched on.
864 const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
865 Register SwitchOpReg = getOrCreateVReg(SValue);
866 auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
867 auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
868
869 // This value may be smaller or larger than the target's pointer type, and
870 // therefore require extension or truncating.
871 auto *PtrIRTy = PointerType::getUnqual(SValue.getContext());
872 const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
873 Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
874
875 JT.Reg = Sub.getReg(0);
876
877 if (JTH.FallthroughUnreachable) {
878 if (JT.MBB != HeaderBB->getNextNode())
879 MIB.buildBr(*JT.MBB);
880 return true;
881 }
882
883 // Emit the range check for the jump table, and branch to the default block
884 // for the switch statement if the value being switched on exceeds the
885 // largest case in the switch.
886 auto Cst = getOrCreateVReg(
887 *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
888 Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
889 auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
890
891 auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
892
893 // Avoid emitting unnecessary branches to the next block.
894 if (JT.MBB != HeaderBB->getNextNode())
895 BrCond = MIB.buildBr(*JT.MBB);
896 return true;
897}
898
899void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
900 MachineBasicBlock *SwitchBB,
901 MachineIRBuilder &MIB) {
902 Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
904 DebugLoc OldDbgLoc = MIB.getDebugLoc();
905 MIB.setDebugLoc(CB.DbgLoc);
906 MIB.setMBB(*CB.ThisBB);
907
908 if (CB.PredInfo.NoCmp) {
909 // Branch or fall through to TrueBB.
910 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
911 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
912 CB.ThisBB);
914 if (CB.TrueBB != CB.ThisBB->getNextNode())
915 MIB.buildBr(*CB.TrueBB);
916 MIB.setDebugLoc(OldDbgLoc);
917 return;
918 }
919
920 const LLT i1Ty = LLT::scalar(1);
921 // Build the compare.
922 if (!CB.CmpMHS) {
923 const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);
924 // For conditional branch lowering, we might try to do something silly like
925 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
926 // just re-use the existing condition vreg.
927 if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
929 Cond = CondLHS;
930 } else {
931 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
933 Cond =
934 MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
935 else
936 Cond =
937 MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
938 }
939 } else {
941 "Can only handle SLE ranges");
942
943 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
944 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
945
946 Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
947 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
948 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
949 Cond =
950 MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
951 } else {
952 const LLT CmpTy = MRI->getType(CmpOpReg);
953 auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
954 auto Diff = MIB.buildConstant(CmpTy, High - Low);
955 Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
956 }
957 }
958
959 // Update successor info
960 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
961
962 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
963 CB.ThisBB);
964
965 // TrueBB and FalseBB are always different unless the incoming IR is
966 // degenerate. This only happens when running llc on weird IR.
967 if (CB.TrueBB != CB.FalseBB)
968 addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
970
971 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
972 CB.ThisBB);
973
974 MIB.buildBrCond(Cond, *CB.TrueBB);
975 MIB.buildBr(*CB.FalseBB);
976 MIB.setDebugLoc(OldDbgLoc);
977}
978
979bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
980 MachineBasicBlock *SwitchMBB,
981 MachineBasicBlock *CurMBB,
982 MachineBasicBlock *DefaultMBB,
983 MachineIRBuilder &MIB,
985 BranchProbability UnhandledProbs,
987 MachineBasicBlock *Fallthrough,
988 bool FallthroughUnreachable) {
989 using namespace SwitchCG;
990 MachineFunction *CurMF = SwitchMBB->getParent();
991 // FIXME: Optimize away range check based on pivot comparisons.
992 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
993 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
994 BranchProbability DefaultProb = W.DefaultProb;
995
996 // The jump block hasn't been inserted yet; insert it here.
997 MachineBasicBlock *JumpMBB = JT->MBB;
998 CurMF->insert(BBI, JumpMBB);
999
1000 // Since the jump table block is separate from the switch block, we need
1001 // to keep track of it as a machine predecessor to the default block,
1002 // otherwise we lose the phi edges.
1003 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1004 CurMBB);
1005 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1006 JumpMBB);
1007
1008 auto JumpProb = I->Prob;
1009 auto FallthroughProb = UnhandledProbs;
1010
1011 // If the default statement is a target of the jump table, we evenly
1012 // distribute the default probability to successors of CurMBB. Also
1013 // update the probability on the edge from JumpMBB to Fallthrough.
1014 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1015 SE = JumpMBB->succ_end();
1016 SI != SE; ++SI) {
1017 if (*SI == DefaultMBB) {
1018 JumpProb += DefaultProb / 2;
1019 FallthroughProb -= DefaultProb / 2;
1020 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
1021 JumpMBB->normalizeSuccProbs();
1022 } else {
1023 // Also record edges from the jump table block to it's successors.
1024 addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1025 JumpMBB);
1026 }
1027 }
1028
1029 if (FallthroughUnreachable)
1030 JTH->FallthroughUnreachable = true;
1031
1032 if (!JTH->FallthroughUnreachable)
1033 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
1034 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
1035 CurMBB->normalizeSuccProbs();
1036
1037 // The jump table header will be inserted in our current block, do the
1038 // range check, and fall through to our fallthrough block.
1039 JTH->HeaderBB = CurMBB;
1040 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1041
1042 // If we're in the right place, emit the jump table header right now.
1043 if (CurMBB == SwitchMBB) {
1044 if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
1045 return false;
1046 JTH->Emitted = true;
1047 }
1048 return true;
1049}
1050bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
1051 Value *Cond,
1052 MachineBasicBlock *Fallthrough,
1053 bool FallthroughUnreachable,
1054 BranchProbability UnhandledProbs,
1055 MachineBasicBlock *CurMBB,
1056 MachineIRBuilder &MIB,
1057 MachineBasicBlock *SwitchMBB) {
1058 using namespace SwitchCG;
1059 const Value *RHS, *LHS, *MHS;
1060 CmpInst::Predicate Pred;
1061 if (I->Low == I->High) {
1062 // Check Cond == I->Low.
1063 Pred = CmpInst::ICMP_EQ;
1064 LHS = Cond;
1065 RHS = I->Low;
1066 MHS = nullptr;
1067 } else {
1068 // Check I->Low <= Cond <= I->High.
1069 Pred = CmpInst::ICMP_SLE;
1070 LHS = I->Low;
1071 MHS = Cond;
1072 RHS = I->High;
1073 }
1074
1075 // If Fallthrough is unreachable, fold away the comparison.
1076 // The false probability is the sum of all unhandled cases.
1077 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1078 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1079
1080 emitSwitchCase(CB, SwitchMBB, MIB);
1081 return true;
1082}
1083
1084void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1085 MachineBasicBlock *SwitchBB) {
1086 MachineIRBuilder &MIB = *CurBuilder;
1087 MIB.setMBB(*SwitchBB);
1088
1089 // Subtract the minimum value.
1090 Register SwitchOpReg = getOrCreateVReg(*B.SValue);
1091
1092 LLT SwitchOpTy = MRI->getType(SwitchOpReg);
1093 Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);
1094 auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1095
1097 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1098
1099 LLT MaskTy = SwitchOpTy;
1100 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1101 !llvm::has_single_bit<uint32_t>(MaskTy.getSizeInBits()))
1102 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1103 else {
1104 // Ensure that the type will fit the mask value.
1105 for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1106 if (!isUIntN(SwitchOpTy.getSizeInBits(), B.Cases[I].Mask)) {
1107 // Switch table case range are encoded into series of masks.
1108 // Just use pointer type, it's guaranteed to fit.
1109 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1110 break;
1111 }
1112 }
1113 }
1114 Register SubReg = RangeSub.getReg(0);
1115 if (SwitchOpTy != MaskTy)
1116 SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);
1117
1118 B.RegVT = getMVTForLLT(MaskTy);
1119 B.Reg = SubReg;
1120
1121 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1122
1123 if (!B.FallthroughUnreachable)
1124 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
1125 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
1126
1127 SwitchBB->normalizeSuccProbs();
1128
1129 if (!B.FallthroughUnreachable) {
1130 // Conditional branch to the default block.
1131 auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);
1132 auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),
1133 RangeSub, RangeCst);
1134 MIB.buildBrCond(RangeCmp, *B.Default);
1135 }
1136
1137 // Avoid emitting unnecessary branches to the next block.
1138 if (MBB != SwitchBB->getNextNode())
1139 MIB.buildBr(*MBB);
1140}
1141
1142void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1143 MachineBasicBlock *NextMBB,
1144 BranchProbability BranchProbToNext,
1146 MachineBasicBlock *SwitchBB) {
1147 MachineIRBuilder &MIB = *CurBuilder;
1148 MIB.setMBB(*SwitchBB);
1149
1150 LLT SwitchTy = getLLTForMVT(BB.RegVT);
1151 Register Cmp;
1152 unsigned PopCount = llvm::popcount(B.Mask);
1153 if (PopCount == 1) {
1154 // Testing for a single bit; just compare the shift count with what it
1155 // would need to be to shift a 1 bit in that position.
1156 auto MaskTrailingZeros =
1157 MIB.buildConstant(SwitchTy, llvm::countr_zero(B.Mask));
1158 Cmp =
1159 MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
1160 .getReg(0);
1161 } else if (PopCount == BB.Range) {
1162 // There is only one zero bit in the range, test for it directly.
1163 auto MaskTrailingOnes =
1164 MIB.buildConstant(SwitchTy, llvm::countr_one(B.Mask));
1165 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
1166 .getReg(0);
1167 } else {
1168 // Make desired shift.
1169 auto CstOne = MIB.buildConstant(SwitchTy, 1);
1170 auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);
1171
1172 // Emit bit tests and jumps.
1173 auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);
1174 auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);
1175 auto CstZero = MIB.buildConstant(SwitchTy, 0);
1176 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)
1177 .getReg(0);
1178 }
1179
1180 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1181 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
1182 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1183 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1184 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1185 // one as they are relative probabilities (and thus work more like weights),
1186 // and hence we need to normalize them to let the sum of them become one.
1187 SwitchBB->normalizeSuccProbs();
1188
1189 // Record the fact that the IR edge from the header to the bit test target
1190 // will go through our new block. Neeeded for PHIs to have nodes added.
1191 addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1192 SwitchBB);
1193
1194 MIB.buildBrCond(Cmp, *B.TargetBB);
1195
1196 // Avoid emitting unnecessary branches to the next block.
1197 if (NextMBB != SwitchBB->getNextNode())
1198 MIB.buildBr(*NextMBB);
1199}
1200
1201bool IRTranslator::lowerBitTestWorkItem(
1203 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1205 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1207 bool FallthroughUnreachable) {
1208 using namespace SwitchCG;
1209 MachineFunction *CurMF = SwitchMBB->getParent();
1210 // FIXME: Optimize away range check based on pivot comparisons.
1211 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1212 // The bit test blocks haven't been inserted yet; insert them here.
1213 for (BitTestCase &BTC : BTB->Cases)
1214 CurMF->insert(BBI, BTC.ThisBB);
1215
1216 // Fill in fields of the BitTestBlock.
1217 BTB->Parent = CurMBB;
1218 BTB->Default = Fallthrough;
1219
1220 BTB->DefaultProb = UnhandledProbs;
1221 // If the cases in bit test don't form a contiguous range, we evenly
1222 // distribute the probability on the edge to Fallthrough to two
1223 // successors of CurMBB.
1224 if (!BTB->ContiguousRange) {
1225 BTB->Prob += DefaultProb / 2;
1226 BTB->DefaultProb -= DefaultProb / 2;
1227 }
1228
1229 if (FallthroughUnreachable)
1230 BTB->FallthroughUnreachable = true;
1231
1232 // If we're in the right place, emit the bit test header right now.
1233 if (CurMBB == SwitchMBB) {
1234 emitBitTestHeader(*BTB, SwitchMBB);
1235 BTB->Emitted = true;
1236 }
1237 return true;
1238}
1239
1240bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1241 Value *Cond,
1242 MachineBasicBlock *SwitchMBB,
1243 MachineBasicBlock *DefaultMBB,
1244 MachineIRBuilder &MIB) {
1245 using namespace SwitchCG;
1246 MachineFunction *CurMF = FuncInfo.MF;
1247 MachineBasicBlock *NextMBB = nullptr;
1249 if (++BBI != FuncInfo.MF->end())
1250 NextMBB = &*BBI;
1251
1252 if (EnableOpts) {
1253 // Here, we order cases by probability so the most likely case will be
1254 // checked first. However, two clusters can have the same probability in
1255 // which case their relative ordering is non-deterministic. So we use Low
1256 // as a tie-breaker as clusters are guaranteed to never overlap.
1257 llvm::sort(W.FirstCluster, W.LastCluster + 1,
1258 [](const CaseCluster &a, const CaseCluster &b) {
1259 return a.Prob != b.Prob
1260 ? a.Prob > b.Prob
1261 : a.Low->getValue().slt(b.Low->getValue());
1262 });
1263
1264 // Rearrange the case blocks so that the last one falls through if possible
1265 // without changing the order of probabilities.
1266 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1267 --I;
1268 if (I->Prob > W.LastCluster->Prob)
1269 break;
1270 if (I->Kind == CC_Range && I->MBB == NextMBB) {
1271 std::swap(*I, *W.LastCluster);
1272 break;
1273 }
1274 }
1275 }
1276
1277 // Compute total probability.
1278 BranchProbability DefaultProb = W.DefaultProb;
1279 BranchProbability UnhandledProbs = DefaultProb;
1280 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1281 UnhandledProbs += I->Prob;
1282
1283 MachineBasicBlock *CurMBB = W.MBB;
1284 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1285 bool FallthroughUnreachable = false;
1286 MachineBasicBlock *Fallthrough;
1287 if (I == W.LastCluster) {
1288 // For the last cluster, fall through to the default destination.
1289 Fallthrough = DefaultMBB;
1290 FallthroughUnreachable = isa<UnreachableInst>(
1291 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1292 } else {
1293 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
1294 CurMF->insert(BBI, Fallthrough);
1295 }
1296 UnhandledProbs -= I->Prob;
1297
1298 switch (I->Kind) {
1299 case CC_BitTests: {
1300 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1301 DefaultProb, UnhandledProbs, I, Fallthrough,
1302 FallthroughUnreachable)) {
1303 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1304 return false;
1305 }
1306 break;
1307 }
1308
1309 case CC_JumpTable: {
1310 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1311 UnhandledProbs, I, Fallthrough,
1312 FallthroughUnreachable)) {
1313 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1314 return false;
1315 }
1316 break;
1317 }
1318 case CC_Range: {
1319 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1320 FallthroughUnreachable, UnhandledProbs,
1321 CurMBB, MIB, SwitchMBB)) {
1322 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1323 return false;
1324 }
1325 break;
1326 }
1327 }
1328 CurMBB = Fallthrough;
1329 }
1330
1331 return true;
1332}
1333
1334bool IRTranslator::translateIndirectBr(const User &U,
1335 MachineIRBuilder &MIRBuilder) {
1336 const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
1337
1338 const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
1339 MIRBuilder.buildBrIndirect(Tgt);
1340
1341 // Link successors.
1343 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1344 for (const BasicBlock *Succ : successors(&BrInst)) {
1345 // It's legal for indirectbr instructions to have duplicate blocks in the
1346 // destination list. We don't allow this in MIR. Skip anything that's
1347 // already a successor.
1348 if (!AddedSuccessors.insert(Succ).second)
1349 continue;
1350 CurBB.addSuccessor(&getMBB(*Succ));
1351 }
1352
1353 return true;
1354}
1355
1356static bool isSwiftError(const Value *V) {
1357 if (auto Arg = dyn_cast<Argument>(V))
1358 return Arg->hasSwiftErrorAttr();
1359 if (auto AI = dyn_cast<AllocaInst>(V))
1360 return AI->isSwiftError();
1361 return false;
1362}
1363
1364bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1365 const LoadInst &LI = cast<LoadInst>(U);
1366 TypeSize StoreSize = DL->getTypeStoreSize(LI.getType());
1367 if (StoreSize.isZero())
1368 return true;
1369
1370 ArrayRef<Register> Regs = getOrCreateVRegs(LI);
1371 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
1372 Register Base = getOrCreateVReg(*LI.getPointerOperand());
1373 AAMDNodes AAInfo = LI.getAAMetadata();
1374
1375 const Value *Ptr = LI.getPointerOperand();
1376 Type *OffsetIRTy = DL->getIndexType(Ptr->getType());
1377 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1378
1379 if (CLI->supportSwiftError() && isSwiftError(Ptr)) {
1380 assert(Regs.size() == 1 && "swifterror should be single pointer");
1381 Register VReg =
1382 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1383 MIRBuilder.buildCopy(Regs[0], VReg);
1384 return true;
1385 }
1386
1388 TLI->getLoadMemOperandFlags(LI, *DL, AC, LibInfo);
1389 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1390 if (AA->pointsToConstantMemory(
1391 MemoryLocation(Ptr, LocationSize::precise(StoreSize), AAInfo))) {
1393 }
1394 }
1395
1396 const MDNode *Ranges =
1397 Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
1398 for (unsigned i = 0; i < Regs.size(); ++i) {
1399 Register Addr;
1400 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1401
1402 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
1403 Align BaseAlign = getMemOpAlign(LI);
1404 auto MMO = MF->getMachineMemOperand(
1405 Ptr, Flags, MRI->getType(Regs[i]),
1406 commonAlignment(BaseAlign, Offsets[i] / 8), AAInfo, Ranges,
1407 LI.getSyncScopeID(), LI.getOrdering());
1408 MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
1409 }
1410
1411 return true;
1412}
1413
1414bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1415 const StoreInst &SI = cast<StoreInst>(U);
1416 if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
1417 return true;
1418
1419 ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
1420 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
1421 Register Base = getOrCreateVReg(*SI.getPointerOperand());
1422
1423 Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());
1424 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1425
1426 if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
1427 assert(Vals.size() == 1 && "swifterror should be single pointer");
1428
1429 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1430 SI.getPointerOperand());
1431 MIRBuilder.buildCopy(VReg, Vals[0]);
1432 return true;
1433 }
1434
1436
1437 for (unsigned i = 0; i < Vals.size(); ++i) {
1438 Register Addr;
1439 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1440
1441 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1442 Align BaseAlign = getMemOpAlign(SI);
1443 auto MMO = MF->getMachineMemOperand(
1444 Ptr, Flags, MRI->getType(Vals[i]),
1445 commonAlignment(BaseAlign, Offsets[i] / 8), SI.getAAMetadata(), nullptr,
1446 SI.getSyncScopeID(), SI.getOrdering());
1447 MIRBuilder.buildStore(Vals[i], Addr, *MMO);
1448 }
1449 return true;
1450}
1451
1453 const Value *Src = U.getOperand(0);
1454 Type *Int32Ty = Type::getInt32Ty(U.getContext());
1455
1456 // getIndexedOffsetInType is designed for GEPs, so the first index is the
1457 // usual array element rather than looking into the actual aggregate.
1459 Indices.push_back(ConstantInt::get(Int32Ty, 0));
1460
1461 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
1462 for (auto Idx : EVI->indices())
1463 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1464 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1465 for (auto Idx : IVI->indices())
1466 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1467 } else {
1468 for (unsigned i = 1; i < U.getNumOperands(); ++i)
1469 Indices.push_back(U.getOperand(i));
1470 }
1471
1472 return 8 * static_cast<uint64_t>(
1473 DL.getIndexedOffsetInType(Src->getType(), Indices));
1474}
1475
1476bool IRTranslator::translateExtractValue(const User &U,
1477 MachineIRBuilder &MIRBuilder) {
1478 const Value *Src = U.getOperand(0);
1480 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1481 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
1482 unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
1483 auto &DstRegs = allocateVRegs(U);
1484
1485 for (unsigned i = 0; i < DstRegs.size(); ++i)
1486 DstRegs[i] = SrcRegs[Idx++];
1487
1488 return true;
1489}
1490
1491bool IRTranslator::translateInsertValue(const User &U,
1492 MachineIRBuilder &MIRBuilder) {
1493 const Value *Src = U.getOperand(0);
1495 auto &DstRegs = allocateVRegs(U);
1496 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
1497 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1498 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
1499 auto *InsertedIt = InsertedRegs.begin();
1500
1501 for (unsigned i = 0; i < DstRegs.size(); ++i) {
1502 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1503 DstRegs[i] = *InsertedIt++;
1504 else
1505 DstRegs[i] = SrcRegs[i];
1506 }
1507
1508 return true;
1509}
1510
1511bool IRTranslator::translateSelect(const User &U,
1512 MachineIRBuilder &MIRBuilder) {
1513 Register Tst = getOrCreateVReg(*U.getOperand(0));
1514 ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1515 ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1516 ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1517
1518 uint32_t Flags = 0;
1519 if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
1521
1522 for (unsigned i = 0; i < ResRegs.size(); ++i) {
1523 MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
1524 }
1525
1526 return true;
1527}
1528
1529bool IRTranslator::translateCopy(const User &U, const Value &V,
1530 MachineIRBuilder &MIRBuilder) {
1531 Register Src = getOrCreateVReg(V);
1532 auto &Regs = *VMap.getVRegs(U);
1533 if (Regs.empty()) {
1534 Regs.push_back(Src);
1535 VMap.getOffsets(U)->push_back(0);
1536 } else {
1537 // If we already assigned a vreg for this instruction, we can't change that.
1538 // Emit a copy to satisfy the users we already emitted.
1539 MIRBuilder.buildCopy(Regs[0], Src);
1540 }
1541 return true;
1542}
1543
1544bool IRTranslator::translateBitCast(const User &U,
1545 MachineIRBuilder &MIRBuilder) {
1546 // If we're bitcasting to the source type, we can reuse the source vreg.
1547 if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1548 getLLTForType(*U.getType(), *DL)) {
1549 // If the source is a ConstantInt then it was probably created by
1550 // ConstantHoisting and we should leave it alone.
1551 if (isa<ConstantInt>(U.getOperand(0)))
1552 return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1553 MIRBuilder);
1554 return translateCopy(U, *U.getOperand(0), MIRBuilder);
1555 }
1556
1557 return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1558}
1559
1560bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1561 MachineIRBuilder &MIRBuilder) {
1562 if (U.getType()->getScalarType()->isBFloatTy() ||
1563 U.getOperand(0)->getType()->getScalarType()->isBFloatTy())
1564 return false;
1565
1566 uint32_t Flags = 0;
1567 if (const Instruction *I = dyn_cast<Instruction>(&U))
1569
1570 Register Op = getOrCreateVReg(*U.getOperand(0));
1571 Register Res = getOrCreateVReg(U);
1572 MIRBuilder.buildInstr(Opcode, {Res}, {Op}, Flags);
1573 return true;
1574}
1575
1576bool IRTranslator::translateGetElementPtr(const User &U,
1577 MachineIRBuilder &MIRBuilder) {
1578 Value &Op0 = *U.getOperand(0);
1579 Register BaseReg = getOrCreateVReg(Op0);
1580 Type *PtrIRTy = Op0.getType();
1581 LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1582 Type *OffsetIRTy = DL->getIndexType(PtrIRTy);
1583 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1584
1585 uint32_t Flags = 0;
1586 if (isa<Instruction>(U)) {
1587 const Instruction &I = cast<Instruction>(U);
1589 }
1590
1591 // Normalize Vector GEP - all scalar operands should be converted to the
1592 // splat vector.
1593 unsigned VectorWidth = 0;
1594
1595 // True if we should use a splat vector; using VectorWidth alone is not
1596 // sufficient.
1597 bool WantSplatVector = false;
1598 if (auto *VT = dyn_cast<VectorType>(U.getType())) {
1599 VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1600 // We don't produce 1 x N vectors; those are treated as scalars.
1601 WantSplatVector = VectorWidth > 1;
1602 }
1603
1604 // We might need to splat the base pointer into a vector if the offsets
1605 // are vectors.
1606 if (WantSplatVector && !PtrTy.isVector()) {
1607 BaseReg = MIRBuilder
1608 .buildSplatBuildVector(LLT::fixed_vector(VectorWidth, PtrTy),
1609 BaseReg)
1610 .getReg(0);
1611 PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
1612 PtrTy = getLLTForType(*PtrIRTy, *DL);
1613 OffsetIRTy = DL->getIndexType(PtrIRTy);
1614 OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1615 }
1616
1617 int64_t Offset = 0;
1618 for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1619 GTI != E; ++GTI) {
1620 const Value *Idx = GTI.getOperand();
1621 if (StructType *StTy = GTI.getStructTypeOrNull()) {
1622 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1624 continue;
1625 } else {
1626 uint64_t ElementSize = GTI.getSequentialElementStride(*DL);
1627
1628 // If this is a scalar constant or a splat vector of constants,
1629 // handle it quickly.
1630 if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1631 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1632 Offset += ElementSize * *Val;
1633 continue;
1634 }
1635 }
1636
1637 if (Offset != 0) {
1638 auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1639 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1640 .getReg(0);
1641 Offset = 0;
1642 }
1643
1644 Register IdxReg = getOrCreateVReg(*Idx);
1645 LLT IdxTy = MRI->getType(IdxReg);
1646 if (IdxTy != OffsetTy) {
1647 if (!IdxTy.isVector() && WantSplatVector) {
1648 IdxReg = MIRBuilder
1650 IdxReg)
1651 .getReg(0);
1652 }
1653
1654 IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1655 }
1656
1657 // N = N + Idx * ElementSize;
1658 // Avoid doing it for ElementSize of 1.
1659 Register GepOffsetReg;
1660 if (ElementSize != 1) {
1661 auto ElementSizeMIB = MIRBuilder.buildConstant(
1662 getLLTForType(*OffsetIRTy, *DL), ElementSize);
1663 GepOffsetReg =
1664 MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
1665 } else
1666 GepOffsetReg = IdxReg;
1667
1668 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1669 }
1670 }
1671
1672 if (Offset != 0) {
1673 auto OffsetMIB =
1674 MIRBuilder.buildConstant(OffsetTy, Offset);
1675
1676 if (int64_t(Offset) >= 0 && cast<GEPOperator>(U).isInBounds())
1678
1679 MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0),
1680 Flags);
1681 return true;
1682 }
1683
1684 MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1685 return true;
1686}
1687
1688bool IRTranslator::translateMemFunc(const CallInst &CI,
1689 MachineIRBuilder &MIRBuilder,
1690 unsigned Opcode) {
1691 const Value *SrcPtr = CI.getArgOperand(1);
1692 // If the source is undef, then just emit a nop.
1693 if (isa<UndefValue>(SrcPtr))
1694 return true;
1695
1697
1698 unsigned MinPtrSize = UINT_MAX;
1699 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {
1700 Register SrcReg = getOrCreateVReg(**AI);
1701 LLT SrcTy = MRI->getType(SrcReg);
1702 if (SrcTy.isPointer())
1703 MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);
1704 SrcRegs.push_back(SrcReg);
1705 }
1706
1707 LLT SizeTy = LLT::scalar(MinPtrSize);
1708
1709 // The size operand should be the minimum of the pointer sizes.
1710 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1711 if (MRI->getType(SizeOpReg) != SizeTy)
1712 SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);
1713
1714 auto ICall = MIRBuilder.buildInstr(Opcode);
1715 for (Register SrcReg : SrcRegs)
1716 ICall.addUse(SrcReg);
1717
1718 Align DstAlign;
1719 Align SrcAlign;
1720 unsigned IsVol =
1721 cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();
1722
1723 ConstantInt *CopySize = nullptr;
1724
1725 if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1726 DstAlign = MCI->getDestAlign().valueOrOne();
1727 SrcAlign = MCI->getSourceAlign().valueOrOne();
1728 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1729 } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
1730 DstAlign = MCI->getDestAlign().valueOrOne();
1731 SrcAlign = MCI->getSourceAlign().valueOrOne();
1732 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1733 } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1734 DstAlign = MMI->getDestAlign().valueOrOne();
1735 SrcAlign = MMI->getSourceAlign().valueOrOne();
1736 CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2));
1737 } else {
1738 auto *MSI = cast<MemSetInst>(&CI);
1739 DstAlign = MSI->getDestAlign().valueOrOne();
1740 }
1741
1742 if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1743 // We need to propagate the tail call flag from the IR inst as an argument.
1744 // Otherwise, we have to pessimize and assume later that we cannot tail call
1745 // any memory intrinsics.
1746 ICall.addImm(CI.isTailCall() ? 1 : 0);
1747 }
1748
1749 // Create mem operands to store the alignment and volatile info.
1752 if (IsVol) {
1753 LoadFlags |= MachineMemOperand::MOVolatile;
1754 StoreFlags |= MachineMemOperand::MOVolatile;
1755 }
1756
1757 AAMDNodes AAInfo = CI.getAAMetadata();
1758 if (AA && CopySize &&
1760 SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) {
1761 LoadFlags |= MachineMemOperand::MOInvariant;
1762
1763 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1764 // but the previous usage implied it did. Probably should check
1765 // isDereferenceableAndAlignedPointer.
1767 }
1768
1769 ICall.addMemOperand(
1771 StoreFlags, 1, DstAlign, AAInfo));
1772 if (Opcode != TargetOpcode::G_MEMSET)
1773 ICall.addMemOperand(MF->getMachineMemOperand(
1774 MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
1775
1776 return true;
1777}
1778
1779bool IRTranslator::translateTrap(const CallInst &CI,
1780 MachineIRBuilder &MIRBuilder,
1781 unsigned Opcode) {
1782 StringRef TrapFuncName =
1783 CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
1784 if (TrapFuncName.empty()) {
1785 if (Opcode == TargetOpcode::G_UBSANTRAP) {
1786 uint64_t Code = cast<ConstantInt>(CI.getOperand(0))->getZExtValue();
1787 MIRBuilder.buildInstr(Opcode, {}, ArrayRef<llvm::SrcOp>{Code});
1788 } else {
1789 MIRBuilder.buildInstr(Opcode);
1790 }
1791 return true;
1792 }
1793
1795 if (Opcode == TargetOpcode::G_UBSANTRAP)
1796 Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
1797 CI.getArgOperand(0)->getType(), 0});
1798
1799 Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
1800 Info.CB = &CI;
1801 Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
1802 return CLI->lowerCall(MIRBuilder, Info);
1803}
1804
1805bool IRTranslator::translateVectorInterleave2Intrinsic(
1806 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1807 assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2 &&
1808 "This function can only be called on the interleave2 intrinsic!");
1809 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
1810 Register Op0 = getOrCreateVReg(*CI.getOperand(0));
1811 Register Op1 = getOrCreateVReg(*CI.getOperand(1));
1812 Register Res = getOrCreateVReg(CI);
1813
1814 LLT OpTy = MRI->getType(Op0);
1815 MIRBuilder.buildShuffleVector(Res, Op0, Op1,
1817
1818 return true;
1819}
1820
1821bool IRTranslator::translateVectorDeinterleave2Intrinsic(
1822 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1823 assert(CI.getIntrinsicID() == Intrinsic::experimental_vector_deinterleave2 &&
1824 "This function can only be called on the deinterleave2 intrinsic!");
1825 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
1826 // SelectionDAG).
1827 Register Op = getOrCreateVReg(*CI.getOperand(0));
1828 auto Undef = MIRBuilder.buildUndef(MRI->getType(Op));
1829 ArrayRef<Register> Res = getOrCreateVRegs(CI);
1830
1831 LLT ResTy = MRI->getType(Res[0]);
1832 MIRBuilder.buildShuffleVector(Res[0], Op, Undef,
1833 createStrideMask(0, 2, ResTy.getNumElements()));
1834 MIRBuilder.buildShuffleVector(Res[1], Op, Undef,
1835 createStrideMask(1, 2, ResTy.getNumElements()));
1836
1837 return true;
1838}
1839
1840void IRTranslator::getStackGuard(Register DstReg,
1841 MachineIRBuilder &MIRBuilder) {
1843 MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1844 auto MIB =
1845 MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1846
1848 if (!Global)
1849 return;
1850
1851 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1852 LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
1853
1854 MachinePointerInfo MPInfo(Global);
1858 MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
1859 MIB.setMemRefs({MemRef});
1860}
1861
1862bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1863 MachineIRBuilder &MIRBuilder) {
1864 ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1865 MIRBuilder.buildInstr(
1866 Op, {ResRegs[0], ResRegs[1]},
1867 {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});
1868
1869 return true;
1870}
1871
1872bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1873 MachineIRBuilder &MIRBuilder) {
1874 Register Dst = getOrCreateVReg(CI);
1875 Register Src0 = getOrCreateVReg(*CI.getOperand(0));
1876 Register Src1 = getOrCreateVReg(*CI.getOperand(1));
1877 uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
1878 MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
1879 return true;
1880}
1881
1882unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1883 switch (ID) {
1884 default:
1885 break;
1886 case Intrinsic::bswap:
1887 return TargetOpcode::G_BSWAP;
1888 case Intrinsic::bitreverse:
1889 return TargetOpcode::G_BITREVERSE;
1890 case Intrinsic::fshl:
1891 return TargetOpcode::G_FSHL;
1892 case Intrinsic::fshr:
1893 return TargetOpcode::G_FSHR;
1894 case Intrinsic::ceil:
1895 return TargetOpcode::G_FCEIL;
1896 case Intrinsic::cos:
1897 return TargetOpcode::G_FCOS;
1898 case Intrinsic::ctpop:
1899 return TargetOpcode::G_CTPOP;
1900 case Intrinsic::exp:
1901 return TargetOpcode::G_FEXP;
1902 case Intrinsic::exp2:
1903 return TargetOpcode::G_FEXP2;
1904 case Intrinsic::exp10:
1905 return TargetOpcode::G_FEXP10;
1906 case Intrinsic::fabs:
1907 return TargetOpcode::G_FABS;
1908 case Intrinsic::copysign:
1909 return TargetOpcode::G_FCOPYSIGN;
1910 case Intrinsic::minnum:
1911 return TargetOpcode::G_FMINNUM;
1912 case Intrinsic::maxnum:
1913 return TargetOpcode::G_FMAXNUM;
1914 case Intrinsic::minimum:
1915 return TargetOpcode::G_FMINIMUM;
1916 case Intrinsic::maximum:
1917 return TargetOpcode::G_FMAXIMUM;
1918 case Intrinsic::canonicalize:
1919 return TargetOpcode::G_FCANONICALIZE;
1920 case Intrinsic::floor:
1921 return TargetOpcode::G_FFLOOR;
1922 case Intrinsic::fma:
1923 return TargetOpcode::G_FMA;
1924 case Intrinsic::log:
1925 return TargetOpcode::G_FLOG;
1926 case Intrinsic::log2:
1927 return TargetOpcode::G_FLOG2;
1928 case Intrinsic::log10:
1929 return TargetOpcode::G_FLOG10;
1930 case Intrinsic::ldexp:
1931 return TargetOpcode::G_FLDEXP;
1932 case Intrinsic::nearbyint:
1933 return TargetOpcode::G_FNEARBYINT;
1934 case Intrinsic::pow:
1935 return TargetOpcode::G_FPOW;
1936 case Intrinsic::powi:
1937 return TargetOpcode::G_FPOWI;
1938 case Intrinsic::rint:
1939 return TargetOpcode::G_FRINT;
1940 case Intrinsic::round:
1941 return TargetOpcode::G_INTRINSIC_ROUND;
1942 case Intrinsic::roundeven:
1943 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1944 case Intrinsic::sin:
1945 return TargetOpcode::G_FSIN;
1946 case Intrinsic::sqrt:
1947 return TargetOpcode::G_FSQRT;
1948 case Intrinsic::trunc:
1949 return TargetOpcode::G_INTRINSIC_TRUNC;
1950 case Intrinsic::readcyclecounter:
1951 return TargetOpcode::G_READCYCLECOUNTER;
1952 case Intrinsic::readsteadycounter:
1953 return TargetOpcode::G_READSTEADYCOUNTER;
1954 case Intrinsic::ptrmask:
1955 return TargetOpcode::G_PTRMASK;
1956 case Intrinsic::lrint:
1957 return TargetOpcode::G_INTRINSIC_LRINT;
1958 // FADD/FMUL require checking the FMF, so are handled elsewhere.
1959 case Intrinsic::vector_reduce_fmin:
1960 return TargetOpcode::G_VECREDUCE_FMIN;
1961 case Intrinsic::vector_reduce_fmax:
1962 return TargetOpcode::G_VECREDUCE_FMAX;
1963 case Intrinsic::vector_reduce_fminimum:
1964 return TargetOpcode::G_VECREDUCE_FMINIMUM;
1965 case Intrinsic::vector_reduce_fmaximum:
1966 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
1967 case Intrinsic::vector_reduce_add:
1968 return TargetOpcode::G_VECREDUCE_ADD;
1969 case Intrinsic::vector_reduce_mul:
1970 return TargetOpcode::G_VECREDUCE_MUL;
1971 case Intrinsic::vector_reduce_and:
1972 return TargetOpcode::G_VECREDUCE_AND;
1973 case Intrinsic::vector_reduce_or:
1974 return TargetOpcode::G_VECREDUCE_OR;
1975 case Intrinsic::vector_reduce_xor:
1976 return TargetOpcode::G_VECREDUCE_XOR;
1977 case Intrinsic::vector_reduce_smax:
1978 return TargetOpcode::G_VECREDUCE_SMAX;
1979 case Intrinsic::vector_reduce_smin:
1980 return TargetOpcode::G_VECREDUCE_SMIN;
1981 case Intrinsic::vector_reduce_umax:
1982 return TargetOpcode::G_VECREDUCE_UMAX;
1983 case Intrinsic::vector_reduce_umin:
1984 return TargetOpcode::G_VECREDUCE_UMIN;
1985 case Intrinsic::lround:
1986 return TargetOpcode::G_LROUND;
1987 case Intrinsic::llround:
1988 return TargetOpcode::G_LLROUND;
1989 case Intrinsic::get_fpenv:
1990 return TargetOpcode::G_GET_FPENV;
1991 case Intrinsic::get_fpmode:
1992 return TargetOpcode::G_GET_FPMODE;
1993 }
1995}
1996
1997bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1999 MachineIRBuilder &MIRBuilder) {
2000
2001 unsigned Op = getSimpleIntrinsicOpcode(ID);
2002
2003 // Is this a simple intrinsic?
2005 return false;
2006
2007 // Yes. Let's translate it.
2009 for (const auto &Arg : CI.args())
2010 VRegs.push_back(getOrCreateVReg(*Arg));
2011
2012 MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
2014 return true;
2015}
2016
2017// TODO: Include ConstainedOps.def when all strict instructions are defined.
2019 switch (ID) {
2020 case Intrinsic::experimental_constrained_fadd:
2021 return TargetOpcode::G_STRICT_FADD;
2022 case Intrinsic::experimental_constrained_fsub:
2023 return TargetOpcode::G_STRICT_FSUB;
2024 case Intrinsic::experimental_constrained_fmul:
2025 return TargetOpcode::G_STRICT_FMUL;
2026 case Intrinsic::experimental_constrained_fdiv:
2027 return TargetOpcode::G_STRICT_FDIV;
2028 case Intrinsic::experimental_constrained_frem:
2029 return TargetOpcode::G_STRICT_FREM;
2030 case Intrinsic::experimental_constrained_fma:
2031 return TargetOpcode::G_STRICT_FMA;
2032 case Intrinsic::experimental_constrained_sqrt:
2033 return TargetOpcode::G_STRICT_FSQRT;
2034 case Intrinsic::experimental_constrained_ldexp:
2035 return TargetOpcode::G_STRICT_FLDEXP;
2036 default:
2037 return 0;
2038 }
2039}
2040
2041bool IRTranslator::translateConstrainedFPIntrinsic(
2042 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2044
2045 unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
2046 if (!Opcode)
2047 return false;
2048
2052
2054 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(0)));
2055 if (!FPI.isUnaryOp())
2056 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(1)));
2057 if (FPI.isTernaryOp())
2058 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(2)));
2059
2060 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
2061 return true;
2062}
2063
2064std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
2065 auto VRegs = getOrCreateVRegs(Arg);
2066 if (VRegs.size() != 1)
2067 return std::nullopt;
2068
2069 // Arguments are lowered as a copy of a livein physical register.
2070 auto *VRegDef = MF->getRegInfo().getVRegDef(VRegs[0]);
2071 if (!VRegDef || !VRegDef->isCopy())
2072 return std::nullopt;
2073 return VRegDef->getOperand(1).getReg().asMCReg();
2074}
2075
2076bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,
2077 const DILocalVariable *Var,
2078 const DIExpression *Expr,
2079 const DebugLoc &DL,
2080 MachineIRBuilder &MIRBuilder) {
2081 auto *Arg = dyn_cast<Argument>(Val);
2082 if (!Arg)
2083 return false;
2084
2085 if (!Expr->isEntryValue())
2086 return false;
2087
2088 std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
2089 if (!PhysReg) {
2090 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2091 << ": expression is entry_value but "
2092 << "couldn't find a physical register\n");
2093 LLVM_DEBUG(dbgs() << *Var << "\n");
2094 return true;
2095 }
2096
2097 if (isDeclare) {
2098 // Append an op deref to account for the fact that this is a dbg_declare.
2099 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
2100 MF->setVariableDbgInfo(Var, Expr, *PhysReg, DL);
2101 } else {
2102 MIRBuilder.buildDirectDbgValue(*PhysReg, Var, Expr);
2103 }
2104
2105 return true;
2106}
2107
2109 switch (ID) {
2110 default:
2111 llvm_unreachable("Unexpected intrinsic");
2112 case Intrinsic::experimental_convergence_anchor:
2113 return TargetOpcode::CONVERGENCECTRL_ANCHOR;
2114 case Intrinsic::experimental_convergence_entry:
2115 return TargetOpcode::CONVERGENCECTRL_ENTRY;
2116 case Intrinsic::experimental_convergence_loop:
2117 return TargetOpcode::CONVERGENCECTRL_LOOP;
2118 }
2119}
2120
2121bool IRTranslator::translateConvergenceControlIntrinsic(
2122 const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
2124 Register OutputReg = getOrCreateConvergenceTokenVReg(CI);
2125 MIB.addDef(OutputReg);
2126
2127 if (ID == Intrinsic::experimental_convergence_loop) {
2129 assert(Bundle && "Expected a convergence control token.");
2130 Register InputReg =
2131 getOrCreateConvergenceTokenVReg(*Bundle->Inputs[0].get());
2132 MIB.addUse(InputReg);
2133 }
2134
2135 return true;
2136}
2137
2138bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
2139 MachineIRBuilder &MIRBuilder) {
2140 if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
2141 if (ORE->enabled()) {
2142 if (MemoryOpRemark::canHandle(MI, *LibInfo)) {
2143 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2144 R.visit(MI);
2145 }
2146 }
2147 }
2148
2149 // If this is a simple intrinsic (that is, we just need to add a def of
2150 // a vreg, and uses for each arg operand, then translate it.
2151 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
2152 return true;
2153
2154 switch (ID) {
2155 default:
2156 break;
2157 case Intrinsic::lifetime_start:
2158 case Intrinsic::lifetime_end: {
2159 // No stack colouring in O0, discard region information.
2161 return true;
2162
2163 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2164 : TargetOpcode::LIFETIME_END;
2165
2166 // Get the underlying objects for the location passed on the lifetime
2167 // marker.
2169 getUnderlyingObjects(CI.getArgOperand(1), Allocas);
2170
2171 // Iterate over each underlying object, creating lifetime markers for each
2172 // static alloca. Quit if we find a non-static alloca.
2173 for (const Value *V : Allocas) {
2174 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
2175 if (!AI)
2176 continue;
2177
2178 if (!AI->isStaticAlloca())
2179 return true;
2180
2181 MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
2182 }
2183 return true;
2184 }
2185 case Intrinsic::dbg_declare: {
2186 const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
2187 assert(DI.getVariable() && "Missing variable");
2188 translateDbgDeclareRecord(DI.getAddress(), DI.hasArgList(), DI.getVariable(),
2189 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2190 return true;
2191 }
2192 case Intrinsic::dbg_label: {
2193 const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
2194 assert(DI.getLabel() && "Missing label");
2195
2197 MIRBuilder.getDebugLoc()) &&
2198 "Expected inlined-at fields to agree");
2199
2200 MIRBuilder.buildDbgLabel(DI.getLabel());
2201 return true;
2202 }
2203 case Intrinsic::vaend:
2204 // No target I know of cares about va_end. Certainly no in-tree target
2205 // does. Simplest intrinsic ever!
2206 return true;
2207 case Intrinsic::vastart: {
2208 Value *Ptr = CI.getArgOperand(0);
2209 unsigned ListSize = TLI->getVaListSizeInBits(*DL) / 8;
2210 Align Alignment = getKnownAlignment(Ptr, *DL);
2211
2212 MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
2213 .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),
2215 ListSize, Alignment));
2216 return true;
2217 }
2218 case Intrinsic::dbg_assign:
2219 // A dbg.assign is a dbg.value with more information about stack locations,
2220 // typically produced during optimisation of variables with leaked
2221 // addresses. We can treat it like a normal dbg_value intrinsic here; to
2222 // benefit from the full analysis of stack/SSA locations, GlobalISel would
2223 // need to register for and use the AssignmentTrackingAnalysis pass.
2225 case Intrinsic::dbg_value: {
2226 // This form of DBG_VALUE is target-independent.
2227 const DbgValueInst &DI = cast<DbgValueInst>(CI);
2228 translateDbgValueRecord(DI.getValue(), DI.hasArgList(), DI.getVariable(),
2229 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2230 return true;
2231 }
2232 case Intrinsic::uadd_with_overflow:
2233 return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
2234 case Intrinsic::sadd_with_overflow:
2235 return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
2236 case Intrinsic::usub_with_overflow:
2237 return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
2238 case Intrinsic::ssub_with_overflow:
2239 return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
2240 case Intrinsic::umul_with_overflow:
2241 return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
2242 case Intrinsic::smul_with_overflow:
2243 return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
2244 case Intrinsic::uadd_sat:
2245 return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
2246 case Intrinsic::sadd_sat:
2247 return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
2248 case Intrinsic::usub_sat:
2249 return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
2250 case Intrinsic::ssub_sat:
2251 return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
2252 case Intrinsic::ushl_sat:
2253 return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
2254 case Intrinsic::sshl_sat:
2255 return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
2256 case Intrinsic::umin:
2257 return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
2258 case Intrinsic::umax:
2259 return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
2260 case Intrinsic::smin:
2261 return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
2262 case Intrinsic::smax:
2263 return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
2264 case Intrinsic::abs:
2265 // TODO: Preserve "int min is poison" arg in GMIR?
2266 return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
2267 case Intrinsic::smul_fix:
2268 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2269 case Intrinsic::umul_fix:
2270 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2271 case Intrinsic::smul_fix_sat:
2272 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2273 case Intrinsic::umul_fix_sat:
2274 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2275 case Intrinsic::sdiv_fix:
2276 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2277 case Intrinsic::udiv_fix:
2278 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2279 case Intrinsic::sdiv_fix_sat:
2280 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2281 case Intrinsic::udiv_fix_sat:
2282 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2283 case Intrinsic::fmuladd: {
2284 const TargetMachine &TM = MF->getTarget();
2285 Register Dst = getOrCreateVReg(CI);
2286 Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
2287 Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
2288 Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
2289 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2291 TLI->getValueType(*DL, CI.getType()))) {
2292 // TODO: Revisit this to see if we should move this part of the
2293 // lowering to the combiner.
2294 MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,
2296 } else {
2297 LLT Ty = getLLTForType(*CI.getType(), *DL);
2298 auto FMul = MIRBuilder.buildFMul(
2299 Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));
2300 MIRBuilder.buildFAdd(Dst, FMul, Op2,
2302 }
2303 return true;
2304 }
2305 case Intrinsic::convert_from_fp16:
2306 // FIXME: This intrinsic should probably be removed from the IR.
2307 MIRBuilder.buildFPExt(getOrCreateVReg(CI),
2308 getOrCreateVReg(*CI.getArgOperand(0)),
2310 return true;
2311 case Intrinsic::convert_to_fp16:
2312 // FIXME: This intrinsic should probably be removed from the IR.
2313 MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
2314 getOrCreateVReg(*CI.getArgOperand(0)),
2316 return true;
2317 case Intrinsic::frexp: {
2318 ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
2319 MIRBuilder.buildFFrexp(VRegs[0], VRegs[1],
2320 getOrCreateVReg(*CI.getArgOperand(0)),
2322 return true;
2323 }
2324 case Intrinsic::memcpy_inline:
2325 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
2326 case Intrinsic::memcpy:
2327 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2328 case Intrinsic::memmove:
2329 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2330 case Intrinsic::memset:
2331 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2332 case Intrinsic::eh_typeid_for: {
2334 Register Reg = getOrCreateVReg(CI);
2335 unsigned TypeID = MF->getTypeIDFor(GV);
2336 MIRBuilder.buildConstant(Reg, TypeID);
2337 return true;
2338 }
2339 case Intrinsic::objectsize:
2340 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2341
2342 case Intrinsic::is_constant:
2343 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2344
2345 case Intrinsic::stackguard:
2346 getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2347 return true;
2348 case Intrinsic::stackprotector: {
2349 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2350 Register GuardVal;
2351 if (TLI->useLoadStackGuardNode()) {
2352 GuardVal = MRI->createGenericVirtualRegister(PtrTy);
2353 getStackGuard(GuardVal, MIRBuilder);
2354 } else
2355 GuardVal = getOrCreateVReg(*CI.getArgOperand(0)); // The guard's value.
2356
2357 AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
2358 int FI = getOrCreateFrameIndex(*Slot);
2360
2361 MIRBuilder.buildStore(
2362 GuardVal, getOrCreateVReg(*Slot),
2366 PtrTy, Align(8)));
2367 return true;
2368 }
2369 case Intrinsic::stacksave: {
2370 MIRBuilder.buildInstr(TargetOpcode::G_STACKSAVE, {getOrCreateVReg(CI)}, {});
2371 return true;
2372 }
2373 case Intrinsic::stackrestore: {
2374 MIRBuilder.buildInstr(TargetOpcode::G_STACKRESTORE, {},
2375 {getOrCreateVReg(*CI.getArgOperand(0))});
2376 return true;
2377 }
2378 case Intrinsic::cttz:
2379 case Intrinsic::ctlz: {
2380 ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
2381 bool isTrailing = ID == Intrinsic::cttz;
2382 unsigned Opcode = isTrailing
2383 ? Cst->isZero() ? TargetOpcode::G_CTTZ
2384 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2385 : Cst->isZero() ? TargetOpcode::G_CTLZ
2386 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2387 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
2388 {getOrCreateVReg(*CI.getArgOperand(0))});
2389 return true;
2390 }
2391 case Intrinsic::invariant_start: {
2392 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2394 MIRBuilder.buildUndef(Undef);
2395 return true;
2396 }
2397 case Intrinsic::invariant_end:
2398 return true;
2399 case Intrinsic::expect:
2400 case Intrinsic::annotation:
2401 case Intrinsic::ptr_annotation:
2402 case Intrinsic::launder_invariant_group:
2403 case Intrinsic::strip_invariant_group: {
2404 // Drop the intrinsic, but forward the value.
2405 MIRBuilder.buildCopy(getOrCreateVReg(CI),
2406 getOrCreateVReg(*CI.getArgOperand(0)));
2407 return true;
2408 }
2409 case Intrinsic::assume:
2410 case Intrinsic::experimental_noalias_scope_decl:
2411 case Intrinsic::var_annotation:
2412 case Intrinsic::sideeffect:
2413 // Discard annotate attributes, assumptions, and artificial side-effects.
2414 return true;
2415 case Intrinsic::read_volatile_register:
2416 case Intrinsic::read_register: {
2417 Value *Arg = CI.getArgOperand(0);
2418 MIRBuilder
2419 .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2420 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
2421 return true;
2422 }
2423 case Intrinsic::write_register: {
2424 Value *Arg = CI.getArgOperand(0);
2425 MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
2426 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
2427 .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
2428 return true;
2429 }
2430 case Intrinsic::localescape: {
2431 MachineBasicBlock &EntryMBB = MF->front();
2433
2434 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2435 // is the same on all targets.
2436 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2438 if (isa<ConstantPointerNull>(Arg))
2439 continue; // Skip null pointers. They represent a hole in index space.
2440
2441 int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
2442 MCSymbol *FrameAllocSym =
2443 MF->getMMI().getContext().getOrCreateFrameAllocSymbol(EscapedName,
2444 Idx);
2445
2446 // This should be inserted at the start of the entry block.
2447 auto LocalEscape =
2448 MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)
2449 .addSym(FrameAllocSym)
2450 .addFrameIndex(FI);
2451
2452 EntryMBB.insert(EntryMBB.begin(), LocalEscape);
2453 }
2454
2455 return true;
2456 }
2457 case Intrinsic::vector_reduce_fadd:
2458 case Intrinsic::vector_reduce_fmul: {
2459 // Need to check for the reassoc flag to decide whether we want a
2460 // sequential reduction opcode or not.
2461 Register Dst = getOrCreateVReg(CI);
2462 Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));
2463 Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));
2464 unsigned Opc = 0;
2465 if (!CI.hasAllowReassoc()) {
2466 // The sequential ordering case.
2467 Opc = ID == Intrinsic::vector_reduce_fadd
2468 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2469 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2470 MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2472 return true;
2473 }
2474 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2475 // since the associativity doesn't matter.
2476 unsigned ScalarOpc;
2477 if (ID == Intrinsic::vector_reduce_fadd) {
2478 Opc = TargetOpcode::G_VECREDUCE_FADD;
2479 ScalarOpc = TargetOpcode::G_FADD;
2480 } else {
2481 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2482 ScalarOpc = TargetOpcode::G_FMUL;
2483 }
2484 LLT DstTy = MRI->getType(Dst);
2485 auto Rdx = MIRBuilder.buildInstr(
2486 Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));
2487 MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2489
2490 return true;
2491 }
2492 case Intrinsic::trap:
2493 return translateTrap(CI, MIRBuilder, TargetOpcode::G_TRAP);
2494 case Intrinsic::debugtrap:
2495 return translateTrap(CI, MIRBuilder, TargetOpcode::G_DEBUGTRAP);
2496 case Intrinsic::ubsantrap:
2497 return translateTrap(CI, MIRBuilder, TargetOpcode::G_UBSANTRAP);
2498 case Intrinsic::amdgcn_cs_chain:
2499 return translateCallBase(CI, MIRBuilder);
2500 case Intrinsic::fptrunc_round: {
2502
2503 // Convert the metadata argument to a constant integer
2504 Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata();
2505 std::optional<RoundingMode> RoundMode =
2506 convertStrToRoundingMode(cast<MDString>(MD)->getString());
2507
2508 // Add the Rounding mode as an integer
2509 MIRBuilder
2510 .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2511 {getOrCreateVReg(CI)},
2512 {getOrCreateVReg(*CI.getArgOperand(0))}, Flags)
2513 .addImm((int)*RoundMode);
2514
2515 return true;
2516 }
2517 case Intrinsic::is_fpclass: {
2518 Value *FpValue = CI.getOperand(0);
2519 ConstantInt *TestMaskValue = cast<ConstantInt>(CI.getOperand(1));
2520
2521 MIRBuilder
2522 .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},
2523 {getOrCreateVReg(*FpValue)})
2524 .addImm(TestMaskValue->getZExtValue());
2525
2526 return true;
2527 }
2528 case Intrinsic::set_fpenv: {
2529 Value *FPEnv = CI.getOperand(0);
2530 MIRBuilder.buildInstr(TargetOpcode::G_SET_FPENV, {},
2531 {getOrCreateVReg(*FPEnv)});
2532 return true;
2533 }
2534 case Intrinsic::reset_fpenv: {
2535 MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPENV, {}, {});
2536 return true;
2537 }
2538 case Intrinsic::set_fpmode: {
2539 Value *FPState = CI.getOperand(0);
2540 MIRBuilder.buildInstr(TargetOpcode::G_SET_FPMODE, {},
2541 { getOrCreateVReg(*FPState) });
2542 return true;
2543 }
2544 case Intrinsic::reset_fpmode: {
2545 MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {});
2546 return true;
2547 }
2548 case Intrinsic::prefetch: {
2549 Value *Addr = CI.getOperand(0);
2550 unsigned RW = cast<ConstantInt>(CI.getOperand(1))->getZExtValue();
2551 unsigned Locality = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
2552 unsigned CacheType = cast<ConstantInt>(CI.getOperand(3))->getZExtValue();
2553
2555 auto &MMO = *MF->getMachineMemOperand(MachinePointerInfo(Addr), Flags,
2556 LLT(), Align());
2557
2558 MIRBuilder.buildPrefetch(getOrCreateVReg(*Addr), RW, Locality, CacheType,
2559 MMO);
2560
2561 return true;
2562 }
2563
2564 case Intrinsic::experimental_vector_interleave2:
2565 case Intrinsic::experimental_vector_deinterleave2: {
2566 // Both intrinsics have at least one operand.
2567 Value *Op0 = CI.getOperand(0);
2568 LLT ResTy = getLLTForType(*Op0->getType(), MIRBuilder.getDataLayout());
2569 if (!ResTy.isFixedVector())
2570 return false;
2571
2572 if (CI.getIntrinsicID() == Intrinsic::experimental_vector_interleave2)
2573 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
2574
2575 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
2576 }
2577
2578#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2579 case Intrinsic::INTRINSIC:
2580#include "llvm/IR/ConstrainedOps.def"
2581 return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2582 MIRBuilder);
2583 case Intrinsic::experimental_convergence_anchor:
2584 case Intrinsic::experimental_convergence_entry:
2585 case Intrinsic::experimental_convergence_loop:
2586 return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
2587 }
2588 return false;
2589}
2590
2591bool IRTranslator::translateInlineAsm(const CallBase &CB,
2592 MachineIRBuilder &MIRBuilder) {
2593
2595
2596 if (!ALI) {
2597 LLVM_DEBUG(
2598 dbgs() << "Inline asm lowering is not supported for this target yet\n");
2599 return false;
2600 }
2601
2602 return ALI->lowerInlineAsm(
2603 MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });
2604}
2605
2606bool IRTranslator::translateCallBase(const CallBase &CB,
2607 MachineIRBuilder &MIRBuilder) {
2608 ArrayRef<Register> Res = getOrCreateVRegs(CB);
2609
2611 Register SwiftInVReg = 0;
2612 Register SwiftErrorVReg = 0;
2613 for (const auto &Arg : CB.args()) {
2614 if (CLI->supportSwiftError() && isSwiftError(Arg)) {
2615 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2616 LLT Ty = getLLTForType(*Arg->getType(), *DL);
2617 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2618 MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
2619 &CB, &MIRBuilder.getMBB(), Arg));
2620 Args.emplace_back(ArrayRef(SwiftInVReg));
2621 SwiftErrorVReg =
2622 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2623 continue;
2624 }
2625 Args.push_back(getOrCreateVRegs(*Arg));
2626 }
2627
2628 if (auto *CI = dyn_cast<CallInst>(&CB)) {
2629 if (ORE->enabled()) {
2630 if (MemoryOpRemark::canHandle(CI, *LibInfo)) {
2631 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2632 R.visit(CI);
2633 }
2634 }
2635 }
2636
2637 Register ConvergenceCtrlToken = 0;
2638 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
2639 const auto &Token = *Bundle->Inputs[0].get();
2640 ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
2641 }
2642
2643 // We don't set HasCalls on MFI here yet because call lowering may decide to
2644 // optimize into tail calls. Instead, we defer that to selection where a final
2645 // scan is done to check if any instructions are calls.
2646 bool Success = CLI->lowerCall(
2647 MIRBuilder, CB, Res, Args, SwiftErrorVReg, ConvergenceCtrlToken,
2648 [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
2649
2650 // Check if we just inserted a tail call.
2651 if (Success) {
2652 assert(!HasTailCall && "Can't tail call return twice from block?");
2654 HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
2655 }
2656
2657 return Success;
2658}
2659
2660bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2661 const CallInst &CI = cast<CallInst>(U);
2662 auto TII = MF->getTarget().getIntrinsicInfo();
2663 const Function *F = CI.getCalledFunction();
2664
2665 // FIXME: support Windows dllimport function calls and calls through
2666 // weak symbols.
2667 if (F && (F->hasDLLImportStorageClass() ||
2669 F->hasExternalWeakLinkage())))
2670 return false;
2671
2672 // FIXME: support control flow guard targets.
2674 return false;
2675
2676 // FIXME: support statepoints and related.
2677 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
2678 return false;
2679
2680 if (CI.isInlineAsm())
2681 return translateInlineAsm(CI, MIRBuilder);
2682
2683 diagnoseDontCall(CI);
2684
2686 if (F && F->isIntrinsic()) {
2687 ID = F->getIntrinsicID();
2689 ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2690 }
2691
2692 if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2693 return translateCallBase(CI, MIRBuilder);
2694
2695 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2696
2697 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2698 return true;
2699
2700 ArrayRef<Register> ResultRegs;
2701 if (!CI.getType()->isVoidTy())
2702 ResultRegs = getOrCreateVRegs(CI);
2703
2704 // Ignore the callsite attributes. Backend code is most likely not expecting
2705 // an intrinsic to sometimes have side effects and sometimes not.
2706 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);
2707 if (isa<FPMathOperator>(CI))
2708 MIB->copyIRFlags(CI);
2709
2710 for (const auto &Arg : enumerate(CI.args())) {
2711 // If this is required to be an immediate, don't materialize it in a
2712 // register.
2713 if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2714 if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
2715 // imm arguments are more convenient than cimm (and realistically
2716 // probably sufficient), so use them.
2717 assert(CI->getBitWidth() <= 64 &&
2718 "large intrinsic immediates not handled");
2719 MIB.addImm(CI->getSExtValue());
2720 } else {
2721 MIB.addFPImm(cast<ConstantFP>(Arg.value()));
2722 }
2723 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {
2724 auto *MD = MDVal->getMetadata();
2725 auto *MDN = dyn_cast<MDNode>(MD);
2726 if (!MDN) {
2727 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))
2728 MDN = MDNode::get(MF->getFunction().getContext(), ConstMD);
2729 else // This was probably an MDString.
2730 return false;
2731 }
2732 MIB.addMetadata(MDN);
2733 } else {
2734 ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
2735 if (VRegs.size() > 1)
2736 return false;
2737 MIB.addUse(VRegs[0]);
2738 }
2739 }
2740
2741 // Add a MachineMemOperand if it is a target mem intrinsic.
2743 // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2744 if (TLI->getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2745 Align Alignment = Info.align.value_or(
2746 DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
2747 LLT MemTy = Info.memVT.isSimple()
2748 ? getLLTForMVT(Info.memVT.getSimpleVT())
2749 : LLT::scalar(Info.memVT.getStoreSizeInBits());
2750
2751 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
2752 // didn't yield anything useful.
2754 if (Info.ptrVal)
2755 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2756 else if (Info.fallbackAddressSpace)
2757 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2758 MIB.addMemOperand(
2759 MF->getMachineMemOperand(MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata()));
2760 }
2761
2762 if (CI.isConvergent()) {
2763 if (auto Bundle = CI.getOperandBundle(LLVMContext::OB_convergencectrl)) {
2764 auto *Token = Bundle->Inputs[0].get();
2765 Register TokenReg = getOrCreateVReg(*Token);
2766 MIB.addUse(TokenReg, RegState::Implicit);
2767 }
2768 }
2769
2770 return true;
2771}
2772
2773bool IRTranslator::findUnwindDestinations(
2774 const BasicBlock *EHPadBB,
2775 BranchProbability Prob,
2776 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2777 &UnwindDests) {
2779 EHPadBB->getParent()->getFunction().getPersonalityFn());
2780 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2781 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2782 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2783 bool IsSEH = isAsynchronousEHPersonality(Personality);
2784
2785 if (IsWasmCXX) {
2786 // Ignore this for now.
2787 return false;
2788 }
2789
2790 while (EHPadBB) {
2791 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2792 BasicBlock *NewEHPadBB = nullptr;
2793 if (isa<LandingPadInst>(Pad)) {
2794 // Stop on landingpads. They are not funclets.
2795 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2796 break;
2797 }
2798 if (isa<CleanupPadInst>(Pad)) {
2799 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2800 // personalities.
2801 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2802 UnwindDests.back().first->setIsEHScopeEntry();
2803 UnwindDests.back().first->setIsEHFuncletEntry();
2804 break;
2805 }
2806 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2807 // Add the catchpad handlers to the possible destinations.
2808 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2809 UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2810 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2811 if (IsMSVCCXX || IsCoreCLR)
2812 UnwindDests.back().first->setIsEHFuncletEntry();
2813 if (!IsSEH)
2814 UnwindDests.back().first->setIsEHScopeEntry();
2815 }
2816 NewEHPadBB = CatchSwitch->getUnwindDest();
2817 } else {
2818 continue;
2819 }
2820
2821 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2822 if (BPI && NewEHPadBB)
2823 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2824 EHPadBB = NewEHPadBB;
2825 }
2826 return true;
2827}
2828
2829bool IRTranslator::translateInvoke(const User &U,
2830 MachineIRBuilder &MIRBuilder) {
2831 const InvokeInst &I = cast<InvokeInst>(U);
2832 MCContext &Context = MF->getContext();
2833
2834 const BasicBlock *ReturnBB = I.getSuccessor(0);
2835 const BasicBlock *EHPadBB = I.getSuccessor(1);
2836
2837 const Function *Fn = I.getCalledFunction();
2838
2839 // FIXME: support invoking patchpoint and statepoint intrinsics.
2840 if (Fn && Fn->isIntrinsic())
2841 return false;
2842
2843 // FIXME: support whatever these are.
2844 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
2845 return false;
2846
2847 // FIXME: support control flow guard targets.
2848 if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
2849 return false;
2850
2851 // FIXME: support Windows exception handling.
2852 if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHI()))
2853 return false;
2854
2855 // FIXME: support Windows dllimport function calls and calls through
2856 // weak symbols.
2857 if (Fn && (Fn->hasDLLImportStorageClass() ||
2859 Fn->hasExternalWeakLinkage())))
2860 return false;
2861
2862 bool LowerInlineAsm = I.isInlineAsm();
2863 bool NeedEHLabel = true;
2864
2865 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2866 // the region covered by the try.
2867 MCSymbol *BeginSymbol = nullptr;
2868 if (NeedEHLabel) {
2869 MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START);
2870 BeginSymbol = Context.createTempSymbol();
2871 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
2872 }
2873
2874 if (LowerInlineAsm) {
2875 if (!translateInlineAsm(I, MIRBuilder))
2876 return false;
2877 } else if (!translateCallBase(I, MIRBuilder))
2878 return false;
2879
2880 MCSymbol *EndSymbol = nullptr;
2881 if (NeedEHLabel) {
2882 EndSymbol = Context.createTempSymbol();
2883 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
2884 }
2885
2887 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2888 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2889 BranchProbability EHPadBBProb =
2890 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2892
2893 if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2894 return false;
2895
2896 MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
2897 &ReturnMBB = getMBB(*ReturnBB);
2898 // Update successor info.
2899 addSuccessorWithProb(InvokeMBB, &ReturnMBB);
2900 for (auto &UnwindDest : UnwindDests) {
2901 UnwindDest.first->setIsEHPad();
2902 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2903 }
2904 InvokeMBB->normalizeSuccProbs();
2905
2906 if (NeedEHLabel) {
2907 assert(BeginSymbol && "Expected a begin symbol!");
2908 assert(EndSymbol && "Expected an end symbol!");
2909 MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
2910 }
2911
2912 MIRBuilder.buildBr(ReturnMBB);
2913 return true;
2914}
2915
2916bool IRTranslator::translateCallBr(const User &U,
2917 MachineIRBuilder &MIRBuilder) {
2918 // FIXME: Implement this.
2919 return false;
2920}
2921
2922bool IRTranslator::translateLandingPad(const User &U,
2923 MachineIRBuilder &MIRBuilder) {
2924 const LandingPadInst &LP = cast<LandingPadInst>(U);
2925
2926 MachineBasicBlock &MBB = MIRBuilder.getMBB();
2927
2928 MBB.setIsEHPad();
2929
2930 // If there aren't registers to copy the values into (e.g., during SjLj
2931 // exceptions), then don't bother.
2932 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
2933 if (TLI->getExceptionPointerRegister(PersonalityFn) == 0 &&
2934 TLI->getExceptionSelectorRegister(PersonalityFn) == 0)
2935 return true;
2936
2937 // If landingpad's return type is token type, we don't create DAG nodes
2938 // for its exception pointer and selector value. The extraction of exception
2939 // pointer or selector value from token type landingpads is not currently
2940 // supported.
2941 if (LP.getType()->isTokenTy())
2942 return true;
2943
2944 // Add a label to mark the beginning of the landing pad. Deletion of the
2945 // landing pad can thus be detected via the MachineModuleInfo.
2946 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
2947 .addSym(MF->addLandingPad(&MBB));
2948
2949 // If the unwinder does not preserve all registers, ensure that the
2950 // function marks the clobbered registers as used.
2952 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
2954
2955 LLT Ty = getLLTForType(*LP.getType(), *DL);
2957 MIRBuilder.buildUndef(Undef);
2958
2960 for (Type *Ty : cast<StructType>(LP.getType())->elements())
2961 Tys.push_back(getLLTForType(*Ty, *DL));
2962 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
2963
2964 // Mark exception register as live in.
2965 Register ExceptionReg = TLI->getExceptionPointerRegister(PersonalityFn);
2966 if (!ExceptionReg)
2967 return false;
2968
2969 MBB.addLiveIn(ExceptionReg);
2970 ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
2971 MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
2972
2973 Register SelectorReg = TLI->getExceptionSelectorRegister(PersonalityFn);
2974 if (!SelectorReg)
2975 return false;
2976
2977 MBB.addLiveIn(SelectorReg);
2978 Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
2979 MIRBuilder.buildCopy(PtrVReg, SelectorReg);
2980 MIRBuilder.buildCast(ResRegs[1], PtrVReg);
2981
2982 return true;
2983}
2984
2985bool IRTranslator::translateAlloca(const User &U,
2986 MachineIRBuilder &MIRBuilder) {
2987 auto &AI = cast<AllocaInst>(U);
2988
2989 if (AI.isSwiftError())
2990 return true;
2991
2992 if (AI.isStaticAlloca()) {
2993 Register Res = getOrCreateVReg(AI);
2994 int FI = getOrCreateFrameIndex(AI);
2995 MIRBuilder.buildFrameIndex(Res, FI);
2996 return true;
2997 }
2998
2999 // FIXME: support stack probing for Windows.
3001 return false;
3002
3003 // Now we're in the harder dynamic case.
3004 Register NumElts = getOrCreateVReg(*AI.getArraySize());
3005 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
3006 LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
3007 if (MRI->getType(NumElts) != IntPtrTy) {
3008 Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
3009 MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
3010 NumElts = ExtElts;
3011 }
3012
3013 Type *Ty = AI.getAllocatedType();
3014
3015 Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
3016 Register TySize =
3017 getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
3018 MIRBuilder.buildMul(AllocSize, NumElts, TySize);
3019
3020 // Round the size of the allocation up to the stack alignment size
3021 // by add SA-1 to the size. This doesn't overflow because we're computing
3022 // an address inside an alloca.
3023 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
3024 auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
3025 auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
3027 auto AlignCst =
3028 MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
3029 auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
3030
3031 Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
3032 if (Alignment <= StackAlign)
3033 Alignment = Align(1);
3034 MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
3035
3036 MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
3038 return true;
3039}
3040
3041bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
3042 // FIXME: We may need more info about the type. Because of how LLT works,
3043 // we're completely discarding the i64/double distinction here (amongst
3044 // others). Fortunately the ABIs I know of where that matters don't use va_arg
3045 // anyway but that's not guaranteed.
3046 MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
3047 {getOrCreateVReg(*U.getOperand(0)),
3048 DL->getABITypeAlign(U.getType()).value()});
3049 return true;
3050}
3051
3052bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
3054 return true;
3055
3056 auto &UI = cast<UnreachableInst>(U);
3057 // We may be able to ignore unreachable behind a noreturn call.
3059 const BasicBlock &BB = *UI.getParent();
3060 if (&UI != &BB.front()) {
3062 std::prev(BasicBlock::const_iterator(UI));
3063 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3064 if (Call->doesNotReturn())
3065 return true;
3066 }
3067 }
3068 }
3069
3070 MIRBuilder.buildTrap();
3071 return true;
3072}
3073
3074bool IRTranslator::translateInsertElement(const User &U,
3075 MachineIRBuilder &MIRBuilder) {
3076 // If it is a <1 x Ty> vector, use the scalar as it is
3077 // not a legal vector type in LLT.
3078 if (auto *FVT = dyn_cast<FixedVectorType>(U.getType());
3079 FVT && FVT->getNumElements() == 1)
3080 return translateCopy(U, *U.getOperand(1), MIRBuilder);
3081
3082 Register Res = getOrCreateVReg(U);
3083 Register Val = getOrCreateVReg(*U.getOperand(0));
3084 Register Elt = getOrCreateVReg(*U.getOperand(1));
3085 Register Idx = getOrCreateVReg(*U.getOperand(2));
3086 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
3087 return true;
3088}
3089
3090bool IRTranslator::translateExtractElement(const User &U,
3091 MachineIRBuilder &MIRBuilder) {
3092 // If it is a <1 x Ty> vector, use the scalar as it is
3093 // not a legal vector type in LLT.
3094 if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
3095 return translateCopy(U, *U.getOperand(0), MIRBuilder);
3096
3097 Register Res = getOrCreateVReg(U);
3098 Register Val = getOrCreateVReg(*U.getOperand(0));
3099 unsigned PreferredVecIdxWidth = TLI->getVectorIdxTy(*DL).getSizeInBits();
3100 Register Idx;
3101 if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
3102 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3103 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
3104 auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
3105 Idx = getOrCreateVReg(*NewIdxCI);
3106 }
3107 }
3108 if (!Idx)
3109 Idx = getOrCreateVReg(*U.getOperand(1));
3110 if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
3111 const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
3112 Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
3113 }
3114 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
3115 return true;
3116}
3117
3118bool IRTranslator::translateShuffleVector(const User &U,
3119 MachineIRBuilder &MIRBuilder) {
3120 // A ShuffleVector that has operates on scalable vectors is a splat vector
3121 // where the value of the splat vector is the 0th element of the first
3122 // operand, since the index mask operand is the zeroinitializer (undef and
3123 // poison are treated as zeroinitializer here).
3124 if (U.getOperand(0)->getType()->isScalableTy()) {
3125 Value *Op0 = U.getOperand(0);
3126 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
3128 getOrCreateVReg(*Op0), 0);
3129 MIRBuilder.buildSplatVector(getOrCreateVReg(U), SplatVal);
3130 return true;
3131 }
3132
3134 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
3135 Mask = SVI->getShuffleMask();
3136 else
3137 Mask = cast<ConstantExpr>(U).getShuffleMask();
3138 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
3139 MIRBuilder
3140 .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
3141 {getOrCreateVReg(*U.getOperand(0)),
3142 getOrCreateVReg(*U.getOperand(1))})
3143 .addShuffleMask(MaskAlloc);
3144 return true;
3145}
3146
3147bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
3148 const PHINode &PI = cast<PHINode>(U);
3149
3151 for (auto Reg : getOrCreateVRegs(PI)) {
3152 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
3153 Insts.push_back(MIB.getInstr());
3154 }
3155
3156 PendingPHIs.emplace_back(&PI, std::move(Insts));
3157 return true;
3158}
3159
3160bool IRTranslator::translateAtomicCmpXchg(const User &U,
3161 MachineIRBuilder &MIRBuilder) {
3162 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
3163
3164 auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);
3165
3166 auto Res = getOrCreateVRegs(I);
3167 Register OldValRes = Res[0];
3168 Register SuccessRes = Res[1];
3169 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3170 Register Cmp = getOrCreateVReg(*I.getCompareOperand());
3171 Register NewVal = getOrCreateVReg(*I.getNewValOperand());
3172
3174 OldValRes, SuccessRes, Addr, Cmp, NewVal,
3176 MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
3177 getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3178 I.getSuccessOrdering(), I.getFailureOrdering()));
3179 return true;
3180}
3181
3182bool IRTranslator::translateAtomicRMW(const User &U,
3183 MachineIRBuilder &MIRBuilder) {
3184 const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
3185 auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);
3186
3187 Register Res = getOrCreateVReg(I);
3188 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3189 Register Val = getOrCreateVReg(*I.getValOperand());
3190
3191 unsigned Opcode = 0;
3192 switch (I.getOperation()) {
3193 default:
3194 return false;
3196 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3197 break;
3198 case AtomicRMWInst::Add:
3199 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3200 break;
3201 case AtomicRMWInst::Sub:
3202 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3203 break;
3204 case AtomicRMWInst::And:
3205 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3206 break;
3208 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3209 break;
3210 case AtomicRMWInst::Or:
3211 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3212 break;
3213 case AtomicRMWInst::Xor:
3214 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3215 break;
3216 case AtomicRMWInst::Max:
3217 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3218 break;
3219 case AtomicRMWInst::Min:
3220 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3221 break;
3223 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3224 break;
3226 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3227 break;
3229 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3230 break;
3232 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3233 break;
3235 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3236 break;
3238 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3239 break;
3241 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3242 break;
3244 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3245 break;
3246 }
3247
3248 MIRBuilder.buildAtomicRMW(
3249 Opcode, Res, Addr, Val,
3250 *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3251 Flags, MRI->getType(Val), getMemOpAlign(I),
3252 I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3253 I.getOrdering()));
3254 return true;
3255}
3256
3257bool IRTranslator::translateFence(const User &U,
3258 MachineIRBuilder &MIRBuilder) {
3259 const FenceInst &Fence = cast<FenceInst>(U);
3260 MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
3261 Fence.getSyncScopeID());
3262 return true;
3263}
3264
3265bool IRTranslator::translateFreeze(const User &U,
3266 MachineIRBuilder &MIRBuilder) {
3267 const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);
3268 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));
3269
3270 assert(DstRegs.size() == SrcRegs.size() &&
3271 "Freeze with different source and destination type?");
3272
3273 for (unsigned I = 0; I < DstRegs.size(); ++I) {
3274 MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);
3275 }
3276
3277 return true;
3278}
3279
3280void IRTranslator::finishPendingPhis() {
3281#ifndef NDEBUG
3282 DILocationVerifier Verifier;
3283 GISelObserverWrapper WrapperObserver(&Verifier);
3284 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3285#endif // ifndef NDEBUG
3286 for (auto &Phi : PendingPHIs) {
3287 const PHINode *PI = Phi.first;
3288 if (PI->getType()->isEmptyTy())
3289 continue;
3290 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3291 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3292 EntryBuilder->setDebugLoc(PI->getDebugLoc());
3293#ifndef NDEBUG
3294 Verifier.setCurrentInst(PI);
3295#endif // ifndef NDEBUG
3296
3298 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3299 auto IRPred = PI->getIncomingBlock(i);
3300 ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
3301 for (auto *Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
3302 if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
3303 continue;
3304 SeenPreds.insert(Pred);
3305 for (unsigned j = 0; j < ValRegs.size(); ++j) {
3306 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3307 MIB.addUse(ValRegs[j]);
3308 MIB.addMBB(Pred);
3309 }
3310 }
3311 }
3312 }
3313}
3314
3315void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,
3316 const DILocalVariable *Variable,
3317 const DIExpression *Expression,
3318 const DebugLoc &DL,
3319 MachineIRBuilder &MIRBuilder) {
3320 assert(Variable->isValidLocationForIntrinsic(DL) &&
3321 "Expected inlined-at fields to agree");
3322 // Act as if we're handling a debug intrinsic.
3323 MIRBuilder.setDebugLoc(DL);
3324
3325 if (!V || HasArgList) {
3326 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
3327 // terminate any prior location.
3328 MIRBuilder.buildIndirectDbgValue(0, Variable, Expression);
3329 return;
3330 }
3331
3332 if (const auto *CI = dyn_cast<Constant>(V)) {
3333 MIRBuilder.buildConstDbgValue(*CI, Variable, Expression);
3334 return;
3335 }
3336
3337 if (auto *AI = dyn_cast<AllocaInst>(V);
3338 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
3339 // If the value is an alloca and the expression starts with a
3340 // dereference, track a stack slot instead of a register, as registers
3341 // may be clobbered.
3342 auto ExprOperands = Expression->getElements();
3343 auto *ExprDerefRemoved =
3344 DIExpression::get(AI->getContext(), ExprOperands.drop_front());
3345 MIRBuilder.buildFIDbgValue(getOrCreateFrameIndex(*AI), Variable,
3346 ExprDerefRemoved);
3347 return;
3348 }
3349 if (translateIfEntryValueArgument(false, V, Variable, Expression, DL,
3350 MIRBuilder))
3351 return;
3352 for (Register Reg : getOrCreateVRegs(*V)) {
3353 // FIXME: This does not handle register-indirect values at offset 0. The
3354 // direct/indirect thing shouldn't really be handled by something as
3355 // implicit as reg+noreg vs reg+imm in the first place, but it seems
3356 // pretty baked in right now.
3357 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expression);
3358 }
3359 return;
3360}
3361
3362void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,
3363 const DILocalVariable *Variable,
3364 const DIExpression *Expression,
3365 const DebugLoc &DL,
3366 MachineIRBuilder &MIRBuilder) {
3367 if (!Address || isa<UndefValue>(Address)) {
3368 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
3369 return;
3370 }
3371
3372 assert(Variable->isValidLocationForIntrinsic(DL) &&
3373 "Expected inlined-at fields to agree");
3374 auto AI = dyn_cast<AllocaInst>(Address);
3375 if (AI && AI->isStaticAlloca()) {
3376 // Static allocas are tracked at the MF level, no need for DBG_VALUE
3377 // instructions (in fact, they get ignored if they *do* exist).
3378 MF->setVariableDbgInfo(Variable, Expression,
3379 getOrCreateFrameIndex(*AI), DL);
3380 return;
3381 }
3382
3383 if (translateIfEntryValueArgument(true, Address, Variable,
3384 Expression, DL,
3385 MIRBuilder))
3386 return;
3387
3388 // A dbg.declare describes the address of a source variable, so lower it
3389 // into an indirect DBG_VALUE.
3390 MIRBuilder.setDebugLoc(DL);
3391 MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
3392 Variable, Expression);
3393 return;
3394}
3395
3396void IRTranslator::translateDbgInfo(const Instruction &Inst,
3397 MachineIRBuilder &MIRBuilder) {
3398 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
3399 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
3400 MIRBuilder.setDebugLoc(DLR->getDebugLoc());
3401 assert(DLR->getLabel() && "Missing label");
3402 assert(DLR->getLabel()->isValidLocationForIntrinsic(
3403 MIRBuilder.getDebugLoc()) &&
3404 "Expected inlined-at fields to agree");
3405 MIRBuilder.buildDbgLabel(DLR->getLabel());
3406 continue;
3407 }
3408 DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
3409 const DILocalVariable *Variable = DVR.getVariable();
3410 const DIExpression *Expression = DVR.getExpression();
3411 Value *V = DVR.getVariableLocationOp(0);
3412 if (DVR.isDbgDeclare())
3413 translateDbgDeclareRecord(V, DVR.hasArgList(), Variable, Expression,
3414 DVR.getDebugLoc(), MIRBuilder);
3415 else
3416 translateDbgValueRecord(V, DVR.hasArgList(), Variable, Expression,
3417 DVR.getDebugLoc(), MIRBuilder);
3418 }
3419}
3420
3421bool IRTranslator::translate(const Instruction &Inst) {
3422 CurBuilder->setDebugLoc(Inst.getDebugLoc());
3423 CurBuilder->setPCSections(Inst.getMetadata(LLVMContext::MD_pcsections));
3424
3425 if (TLI->fallBackToDAGISel(Inst))
3426 return false;
3427
3428 switch (Inst.getOpcode()) {
3429#define HANDLE_INST(NUM, OPCODE, CLASS) \
3430 case Instruction::OPCODE: \
3431 return translate##OPCODE(Inst, *CurBuilder.get());
3432#include "llvm/IR/Instruction.def"
3433 default:
3434 return false;
3435 }
3436}
3437
3438bool IRTranslator::translate(const Constant &C, Register Reg) {
3439 // We only emit constants into the entry block from here. To prevent jumpy
3440 // debug behaviour remove debug line.
3441 if (auto CurrInstDL = CurBuilder->getDL())
3442 EntryBuilder->setDebugLoc(DebugLoc());
3443
3444 if (auto CI = dyn_cast<ConstantInt>(&C))
3445 EntryBuilder->buildConstant(Reg, *CI);
3446 else if (auto CF = dyn_cast<ConstantFP>(&C))
3447 EntryBuilder->buildFConstant(Reg, *CF);
3448 else if (isa<UndefValue>(C))
3449 EntryBuilder->buildUndef(Reg);
3450 else if (isa<ConstantPointerNull>(C))
3451 EntryBuilder->buildConstant(Reg, 0);
3452 else if (auto GV = dyn_cast<GlobalValue>(&C))
3453 EntryBuilder->buildGlobalValue(Reg, GV);
3454 else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
3455 if (!isa<FixedVectorType>(CAZ->getType()))
3456 return false;
3457 // Return the scalar if it is a <1 x Ty> vector.
3458 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3459 if (NumElts == 1)
3460 return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder);
3462 for (unsigned I = 0; I < NumElts; ++I) {
3463 Constant &Elt = *CAZ->getElementValue(I);
3464 Ops.push_back(getOrCreateVReg(Elt));
3465 }
3466 EntryBuilder->buildBuildVector(Reg, Ops);
3467 } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
3468 // Return the scalar if it is a <1 x Ty> vector.
3469 if (CV->getNumElements() == 1)
3470 return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder);
3472 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3473 Constant &Elt = *CV->getElementAsConstant(i);
3474 Ops.push_back(getOrCreateVReg(Elt));
3475 }
3476 EntryBuilder->buildBuildVector(Reg, Ops);
3477 } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
3478 switch(CE->getOpcode()) {
3479#define HANDLE_INST(NUM, OPCODE, CLASS) \
3480 case Instruction::OPCODE: \
3481 return translate##OPCODE(*CE, *EntryBuilder.get());
3482#include "llvm/IR/Instruction.def"
3483 default:
3484 return false;
3485 }
3486 } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
3487 if (CV->getNumOperands() == 1)
3488 return translateCopy(C, *CV->getOperand(0), *EntryBuilder);
3490 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3491 Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
3492 }
3493 EntryBuilder->buildBuildVector(Reg, Ops);
3494 } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
3495 EntryBuilder->buildBlockAddress(Reg, BA);
3496 } else
3497 return false;
3498
3499 return true;
3500}
3501
3502bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3504 for (auto &BTB : SL->BitTestCases) {
3505 // Emit header first, if it wasn't already emitted.
3506 if (!BTB.Emitted)
3507 emitBitTestHeader(BTB, BTB.Parent);
3508
3509 BranchProbability UnhandledProb = BTB.Prob;
3510 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3511 UnhandledProb -= BTB.Cases[j].ExtraProb;
3512 // Set the current basic block to the mbb we wish to insert the code into
3513 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3514 // If all cases cover a contiguous range, it is not necessary to jump to
3515 // the default block after the last bit test fails. This is because the
3516 // range check during bit test header creation has guaranteed that every
3517 // case here doesn't go outside the range. In this case, there is no need
3518 // to perform the last bit test, as it will always be true. Instead, make
3519 // the second-to-last bit-test fall through to the target of the last bit
3520 // test, and delete the last bit test.
3521
3522 MachineBasicBlock *NextMBB;
3523 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3524 // Second-to-last bit-test with contiguous range: fall through to the
3525 // target of the final bit test.
3526 NextMBB = BTB.Cases[j + 1].TargetBB;
3527 } else if (j + 1 == ej) {
3528 // For the last bit test, fall through to Default.
3529 NextMBB = BTB.Default;
3530 } else {
3531 // Otherwise, fall through to the next bit test.
3532 NextMBB = BTB.Cases[j + 1].ThisBB;
3533 }
3534
3535 emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
3536
3537 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3538 // We need to record the replacement phi edge here that normally
3539 // happens in emitBitTestCase before we delete the case, otherwise the
3540 // phi edge will be lost.
3541 addMachineCFGPred({BTB.Parent->getBasicBlock(),
3542 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3543 MBB);
3544 // Since we're not going to use the final bit test, remove it.
3545 BTB.Cases.pop_back();
3546 break;
3547 }
3548 }
3549 // This is "default" BB. We have two jumps to it. From "header" BB and from
3550 // last "case" BB, unless the latter was skipped.
3551 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3552 BTB.Default->getBasicBlock()};
3553 addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
3554 if (!BTB.ContiguousRange) {
3555 addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
3556 }
3557 }
3558 SL->BitTestCases.clear();
3559
3560 for (auto &JTCase : SL->JTCases) {
3561 // Emit header first, if it wasn't already emitted.
3562 if (!JTCase.first.Emitted)
3563 emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
3564
3565 emitJumpTable(JTCase.second, JTCase.second.MBB);
3566 }
3567 SL->JTCases.clear();
3568
3569 for (auto &SwCase : SL->SwitchCases)
3570 emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
3571 SL->SwitchCases.clear();
3572
3573 // Check if we need to generate stack-protector guard checks.
3574 StackProtector &SP = getAnalysis<StackProtector>();
3575 if (SP.shouldEmitSDCheck(BB)) {
3576 bool FunctionBasedInstrumentation =
3578 SPDescriptor.initialize(&BB, &MBB, FunctionBasedInstrumentation);
3579 }
3580 // Handle stack protector.
3581 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3582 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3583 return false;
3584 } else if (SPDescriptor.shouldEmitStackProtector()) {
3585 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3586 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3587
3588 // Find the split point to split the parent mbb. At the same time copy all
3589 // physical registers used in the tail of parent mbb into virtual registers
3590 // before the split point and back into physical registers after the split
3591 // point. This prevents us needing to deal with Live-ins and many other
3592 // register allocation issues caused by us splitting the parent mbb. The
3593 // register allocator will clean up said virtual copies later on.
3595 ParentMBB, *MF->getSubtarget().getInstrInfo());
3596
3597 // Splice the terminator of ParentMBB into SuccessMBB.
3598 SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
3599 ParentMBB->end());
3600
3601 // Add compare/jump on neq/jump to the parent BB.
3602 if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))
3603 return false;
3604
3605 // CodeGen Failure MBB if we have not codegened it yet.
3606 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3607 if (FailureMBB->empty()) {
3608 if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))
3609 return false;
3610 }
3611
3612 // Clear the Per-BB State.
3613 SPDescriptor.resetPerBBState();
3614 }
3615 return true;
3616}
3617
3618bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3619 MachineBasicBlock *ParentBB) {
3620 CurBuilder->setInsertPt(*ParentBB, ParentBB->end());
3621 // First create the loads to the guard/stack slot for the comparison.
3623 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
3624 LLT PtrMemTy = getLLTForMVT(TLI->getPointerMemTy(*DL));
3625
3626 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3627 int FI = MFI.getStackProtectorIndex();
3628
3629 Register Guard;
3630 Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);
3631 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3632 Align Align = DL->getPrefTypeAlign(PointerType::getUnqual(M.getContext()));
3633
3634 // Generate code to load the content of the guard slot.
3635 Register GuardVal =
3636 CurBuilder
3637 ->buildLoad(PtrMemTy, StackSlotPtr,
3640 .getReg(0);
3641
3642 if (TLI->useStackGuardXorFP()) {
3643 LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3644 return false;
3645 }
3646
3647 // Retrieve guard check function, nullptr if instrumentation is inlined.
3648 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M)) {
3649 // This path is currently untestable on GlobalISel, since the only platform
3650 // that needs this seems to be Windows, and we fall back on that currently.
3651 // The code still lives here in case that changes.
3652 // Silence warning about unused variable until the code below that uses
3653 // 'GuardCheckFn' is enabled.
3654 (void)GuardCheckFn;
3655 return false;
3656#if 0
3657 // The target provides a guard check function to validate the guard value.
3658 // Generate a call to that function with the content of the guard slot as
3659 // argument.
3660 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3661 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3663 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3664 Flags.setInReg();
3665 CallLowering::ArgInfo GuardArgInfo(
3666 {GuardVal, FnTy->getParamType(0), {Flags}});
3667
3669 Info.OrigArgs.push_back(GuardArgInfo);
3670 Info.CallConv = GuardCheckFn->getCallingConv();
3671 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
3672 Info.OrigRet = {Register(), FnTy->getReturnType()};
3673 if (!CLI->lowerCall(MIRBuilder, Info)) {
3674 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
3675 return false;
3676 }
3677 return true;
3678#endif
3679 }
3680
3681 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3682 // Otherwise, emit a volatile load to retrieve the stack guard value.
3683 if (TLI->useLoadStackGuardNode()) {
3684 Guard =
3686 getStackGuard(Guard, *CurBuilder);
3687 } else {
3688 // TODO: test using android subtarget when we support @llvm.thread.pointer.
3689 const Value *IRGuard = TLI->getSDagStackGuard(M);
3690 Register GuardPtr = getOrCreateVReg(*IRGuard);
3691
3692 Guard = CurBuilder
3693 ->buildLoad(PtrMemTy, GuardPtr,
3697 .getReg(0);
3698 }
3699
3700 // Perform the comparison.
3701 auto Cmp =
3702 CurBuilder->buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Guard, GuardVal);
3703 // If the guard/stackslot do not equal, branch to failure MBB.
3704 CurBuilder->buildBrCond(Cmp, *SPD.getFailureMBB());
3705 // Otherwise branch to success MBB.
3706 CurBuilder->buildBr(*SPD.getSuccessMBB());
3707 return true;
3708}
3709
3710bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
3711 MachineBasicBlock *FailureBB) {
3712 CurBuilder->setInsertPt(*FailureBB, FailureBB->end());
3713
3714 const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
3715 const char *Name = TLI->getLibcallName(Libcall);
3716
3718 Info.CallConv = TLI->getLibcallCallingConv(Libcall);
3720 Info.OrigRet = {Register(), Type::getVoidTy(MF->getFunction().getContext()),
3721 0};
3722 if (!CLI->lowerCall(*CurBuilder, Info)) {
3723 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
3724 return false;
3725 }
3726
3727 // On PS4/PS5, the "return address" must still be within the calling
3728 // function, even if it's at the very end, so emit an explicit TRAP here.
3729 // WebAssembly needs an unreachable instruction after a non-returning call,
3730 // because the function return type can be different from __stack_chk_fail's
3731 // return type (void).
3732 const TargetMachine &TM = MF->getTarget();
3733 if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) {
3734 LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
3735 return false;
3736 }
3737 return true;
3738}
3739
3740void IRTranslator::finalizeFunction() {
3741 // Release the memory used by the different maps we
3742 // needed during the translation.
3743 PendingPHIs.clear();
3744 VMap.reset();
3745 FrameIndices.clear();
3746 MachinePreds.clear();
3747 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3748 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3749 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
3750 EntryBuilder.reset();
3751 CurBuilder.reset();
3752 FuncInfo.clear();
3753 SPDescriptor.resetPerFunctionState();
3754}
3755
3756/// Returns true if a BasicBlock \p BB within a variadic function contains a
3757/// variadic musttail call.
3758static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3759 if (!IsVarArg)
3760 return false;
3761
3762 // Walk the block backwards, because tail calls usually only appear at the end
3763 // of a block.
3764 return llvm::any_of(llvm::reverse(BB), [](const Instruction &I) {
3765 const auto *CI = dyn_cast<CallInst>(&I);
3766 return CI && CI->isMustTailCall();
3767 });
3768}
3769
3771 MF = &CurMF;
3772 const Function &F = MF->getFunction();
3774 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3775 // Set the CSEConfig and run the analysis.
3776 GISelCSEInfo *CSEInfo = nullptr;
3777 TPC = &getAnalysis<TargetPassConfig>();
3778 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
3780 : TPC->isGISelCSEEnabled();
3781 TLI = MF->getSubtarget().getTargetLowering();
3782
3783 if (EnableCSE) {
3784 EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3785 CSEInfo = &Wrapper.get(TPC->getCSEConfig());
3786 EntryBuilder->setCSEInfo(CSEInfo);
3787 CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3788 CurBuilder->setCSEInfo(CSEInfo);
3789 } else {
3790 EntryBuilder = std::make_unique<MachineIRBuilder>();
3791 CurBuilder = std::make_unique<MachineIRBuilder>();
3792 }
3793 CLI = MF->getSubtarget().getCallLowering();
3794 CurBuilder->setMF(*MF);
3795 EntryBuilder->setMF(*MF);
3796 MRI = &MF->getRegInfo();
3797 DL = &F.getParent()->getDataLayout();
3798 ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
3799 const TargetMachine &TM = MF->getTarget();
3800 TM.resetTargetOptions(F);
3801 EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
3802 FuncInfo.MF = MF;
3803 if (EnableOpts) {
3804 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3805 FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3806 } else {
3807 AA = nullptr;
3808 FuncInfo.BPI = nullptr;
3809 }
3810
3811 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
3812 MF->getFunction());
3813 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
3814 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
3815
3816 SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
3817 SL->init(*TLI, TM, *DL);
3818
3819 assert(PendingPHIs.empty() && "stale PHIs");
3820
3821 // Targets which want to use big endian can enable it using
3822 // enableBigEndian()
3823 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
3824 // Currently we don't properly handle big endian code.
3825 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3826 F.getSubprogram(), &F.getEntryBlock());
3827 R << "unable to translate in big endian mode";
3828 reportTranslationError(*MF, *TPC, *ORE, R);
3829 }
3830
3831 // Release the per-function state when we return, whether we succeeded or not.
3832 auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
3833
3834 // Setup a separate basic-block for the arguments and constants
3836 MF->push_back(EntryBB);
3837 EntryBuilder->setMBB(*EntryBB);
3838
3839 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3840 SwiftError.setFunction(CurMF);
3841 SwiftError.createEntriesInEntryBlock(DbgLoc);
3842
3843 bool IsVarArg = F.isVarArg();
3844 bool HasMustTailInVarArgFn = false;
3845
3846 // Create all blocks, in IR order, to preserve the layout.
3847 for (const BasicBlock &BB: F) {
3848 auto *&MBB = BBToMBB[&BB];
3849
3850 MBB = MF->CreateMachineBasicBlock(&BB);
3851 MF->push_back(MBB);
3852
3853 if (BB.hasAddressTaken())
3854 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
3855
3856 if (!HasMustTailInVarArgFn)
3857 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
3858 }
3859
3860 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
3861
3862 // Make our arguments/constants entry block fallthrough to the IR entry block.
3863 EntryBB->addSuccessor(&getMBB(F.front()));
3864
3865 if (CLI->fallBackToDAGISel(*MF)) {
3866 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3867 F.getSubprogram(), &F.getEntryBlock());
3868 R << "unable to lower function: " << ore::NV("Prototype", F.getType());
3869 reportTranslationError(*MF, *TPC, *ORE, R);
3870 return false;
3871 }
3872
3873 // Lower the actual args into this basic block.
3874 SmallVector<ArrayRef<Register>, 8> VRegArgs;
3875 for (const Argument &Arg: F.args()) {
3876 if (DL->getTypeStoreSize(Arg.getType()).isZero())
3877 continue; // Don't handle zero sized types.
3878 ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
3879 VRegArgs.push_back(VRegs);
3880
3881 if (Arg.hasSwiftErrorAttr()) {
3882 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
3883 SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
3884 }
3885 }
3886
3887 if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) {
3888 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3889 F.getSubprogram(), &F.getEntryBlock());
3890 R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
3891 reportTranslationError(*MF, *TPC, *ORE, R);
3892 return false;
3893 }
3894
3895 // Need to visit defs before uses when translating instructions.
3896 GISelObserverWrapper WrapperObserver;
3897 if (EnableCSE && CSEInfo)
3898 WrapperObserver.addObserver(CSEInfo);
3899 {
3901#ifndef NDEBUG
3902 DILocationVerifier Verifier;
3903 WrapperObserver.addObserver(&Verifier);
3904#endif // ifndef NDEBUG
3905 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3906 RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
3907 for (const BasicBlock *BB : RPOT) {
3908 MachineBasicBlock &MBB = getMBB(*BB);
3909 // Set the insertion point of all the following translations to
3910 // the end of this basic block.
3911 CurBuilder->setMBB(MBB);
3912 HasTailCall = false;
3913 for (const Instruction &Inst : *BB) {
3914 // If we translated a tail call in the last step, then we know
3915 // everything after the call is either a return, or something that is
3916 // handled by the call itself. (E.g. a lifetime marker or assume
3917 // intrinsic.) In this case, we should stop translating the block and
3918 // move on.
3919 if (HasTailCall)
3920 break;
3921#ifndef NDEBUG
3922 Verifier.setCurrentInst(&Inst);
3923#endif // ifndef NDEBUG
3924
3925 // Translate any debug-info attached to the instruction.
3926 translateDbgInfo(Inst, *CurBuilder.get());
3927
3928 if (translate(Inst))
3929 continue;
3930
3931 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3932 Inst.getDebugLoc(), BB);
3933 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
3934
3935 if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
3936 std::string InstStrStorage;
3937 raw_string_ostream InstStr(InstStrStorage);
3938 InstStr << Inst;
3939
3940 R << ": '" << InstStr.str() << "'";
3941 }
3942
3943 reportTranslationError(*MF, *TPC, *ORE, R);
3944 return false;
3945 }
3946
3947 if (!finalizeBasicBlock(*BB, MBB)) {
3948 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3949 BB->getTerminator()->getDebugLoc(), BB);
3950 R << "unable to translate basic block";
3951 reportTranslationError(*MF, *TPC, *ORE, R);
3952 return false;
3953 }
3954 }
3955#ifndef NDEBUG
3956 WrapperObserver.removeObserver(&Verifier);
3957#endif
3958 }
3959
3960 finishPendingPhis();
3961
3962 SwiftError.propagateVRegs();
3963
3964 // Merge the argument lowering and constants block with its single
3965 // successor, the LLVM-IR entry block. We want the basic block to
3966 // be maximal.
3967 assert(EntryBB->succ_size() == 1 &&
3968 "Custom BB used for lowering should have only one successor");
3969 // Get the successor of the current entry block.
3970 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
3971 assert(NewEntryBB.pred_size() == 1 &&
3972 "LLVM-IR entry block has a predecessor!?");
3973 // Move all the instruction from the current entry block to the
3974 // new entry block.
3975 NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
3976 EntryBB->end());
3977
3978 // Update the live-in information for the new entry block.
3979 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
3980 NewEntryBB.addLiveIn(LiveIn);
3981 NewEntryBB.sortUniqueLiveIns();
3982
3983 // Get rid of the now empty basic block.
3984 EntryBB->removeSuccessor(&NewEntryBB);
3985 MF->remove(EntryBB);
3986 MF->deleteMachineBasicBlock(EntryBB);
3987
3988 assert(&MF->front() == &NewEntryBB &&
3989 "New entry wasn't next in the list of basic block!");
3990
3991 // Initialize stack protector information.
3992 StackProtector &SP = getAnalysis<StackProtector>();
3994
3995 return false;
3996}
unsigned SubReg
#define Success
aarch64 promote const
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Provides analysis for continuously CSEing during GISel passes.
This file implements a version of MachineIRBuilder which CSEs insts within a MachineBasicBlock.
This file describes how to lower LLVM calls to machine code calls.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:301
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
std::string Name
uint64_t Size
This contains common code to allow clients to notify changes to machine instr.
const HexagonInstrInfo * TII
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB)
Returns true if a BasicBlock BB within a variadic function contains a variadic musttail call.
static unsigned getConvOpcode(Intrinsic::ID ID)
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
static unsigned getConstrainedOpcode(Intrinsic::ID ID)
IRTranslator LLVM IR MI
#define DEBUG_TYPE
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
static bool isValInBlock(const Value *V, const BasicBlock *BB)
static bool isSwiftError(const Value *V)
This file declares the IRTranslator pass.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file describes how to lower LLVM inline asm to machine code INLINEASM.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:81
Implement a low-level type suitable for MachineInstr level instruction selection.
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
This file contains the declarations for metadata subclasses.
uint64_t High
IntegerType * Int32Ty
LLVMContext & Context
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
verify safepoint Safepoint IR Verifier
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
Value * RHS
Value * LHS
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:76
an instruction to allocate memory on the stack
Definition: Instructions.h:59
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:157
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:132
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:107
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:125
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:103
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
An immutable pass that tracks lazily created AssumptionCache objects.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
@ Add
*p = old + v
Definition: Instructions.h:764
@ FAdd
*p = old + v
Definition: Instructions.h:785
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:778
@ Or
*p = old | v
Definition: Instructions.h:772
@ Sub
*p = old - v
Definition: Instructions.h:766
@ And
*p = old & v
Definition: Instructions.h:768
@ Xor
*p = old ^ v
Definition: Instructions.h:774
@ FSub
*p = old - v
Definition: Instructions.h:788
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:800
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:776
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:782
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:796
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:780
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:792
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:804
@ Nand
*p = ~(old & v)
Definition: Instructions.h:770
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
Definition: Attributes.h:842
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:349
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
Definition: BasicBlock.h:639
InstListType::const_iterator const_iterator
Definition: BasicBlock.h:165
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:349
const Instruction & front() const
Definition: BasicBlock.h:452
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:368
const Instruction & back() const
Definition: BasicBlock.h:454
Legacy analysis pass which computes BlockFrequencyInfo.
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Legacy analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1461
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1776
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2367
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1709
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1629
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2343
Value * getCalledOperand() const
Definition: InstrTypes.h:1702
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1654
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1635
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:2251
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1645
unsigned arg_size() const
Definition: InstrTypes.h:1652
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1786
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
bool isMustTailCall() const
bool checkReturnTypeForCallConv(MachineFunction &MF) const
Toplevel function to check the return type based on the target calling convention.
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
Definition: CallLowering.h:546
virtual bool enableBigEndian() const
For targets which want to use big-endian can enable it with enableBigEndian() hook.
Definition: CallLowering.h:595
virtual bool supportSwiftError() const
Definition: CallLowering.h:449
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
Definition: CallLowering.h:514
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const
This hook must be implemented to lower the given call instruction, including argument and return valu...
Definition: CallLowering.h:558
virtual bool fallBackToDAGISel(const MachineFunction &MF) const
Definition: CallLowering.h:532
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:950
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:960
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:977
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:989
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:990
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:983
@ ICMP_EQ
equal
Definition: InstrTypes.h:981
@ ICMP_NE
not equal
Definition: InstrTypes.h:982
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:986
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:962
bool isFPPredicate() const
Definition: InstrTypes.h:1089
bool isIntPredicate() const
Definition: InstrTypes.h:1090
This is the shared class of boolean and integer constants.
Definition: Constants.h:80
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:205
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:154
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:145
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:417
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:410
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:720
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:905
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:504
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:672
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
Align getPointerABIAlignment(unsigned AS) const
Layout pointer alignment.
Definition: DataLayout.cpp:742
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
Value * getValue(unsigned OpIdx=0) const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
A debug info location.
Definition: DebugLoc.h:33
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
An instruction for ordering other memory operations.
Definition: Instructions.h:460
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:498
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:487
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:692
BranchProbabilityInfo * BPI
void clear()
clear - Clear out all the function-specific state.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:178
const BasicBlock & getEntryBlock() const
Definition: Function.h:782
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1831
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:677
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1882
const Function & getFunction() const
Definition: Function.h:160
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:235
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:350
The actual analysis pass wrapper.
Definition: CSEInfo.h:222
Simple wrapper that does the following.
Definition: CSEInfo.h:204
The CSE Analysis object.
Definition: CSEInfo.h:69
Abstract class that contains various methods for clients to notify about changes.
Simple wrapper observer that takes several observers, and calls each one for each event.
void removeObserver(GISelChangeObserver *O)
void addObserver(GISelChangeObserver *O)
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:566
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:528
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:278
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
bool isTailCall(const MachineInstr &MI) const override
This instruction compares its operands according to the predicate given to the constructor.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
IRTranslator(CodeGenOptLevel OptLevel=CodeGenOptLevel::None)
static char ID
Definition: IRTranslator.h:68
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Indirect Branch Instruction.
bool lowerInlineAsm(MachineIRBuilder &MIRBuilder, const CallBase &CB, std::function< ArrayRef< Register >(const Value &Val)> GetOrCreateVRegs) const
Lower the given inline asm call instruction GetOrCreateVRegs is a callback to materialize a register ...
This instruction inserts a struct field of array element value into an aggregate value.
iterator_range< simple_ilist< DbgRecord >::iterator > getDbgRecordRange() const
Return a range over the DbgRecords attached to this instruction.
Definition: Instruction.h:84
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:454
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:341
const BasicBlock * getParent() const
Definition: Instruction.h:152
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:359
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1707
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:252
bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
Invoke instruction.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
Definition: LowLevelType.h:214
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelType.h:159
constexpr bool isVector() const
Definition: LowLevelType.h:148
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelType.h:57
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:193
constexpr bool isPointer() const
Definition: LowLevelType.h:149
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelType.h:100
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
Definition: LowLevelType.h:178
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:184
Value * getPointerOperand()
Definition: Instructions.h:280
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:245
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:255
static LocationSize precise(uint64_t Value)
Context object for machine code objects.
Definition: MCContext.h:76
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
unsigned pred_size() const
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
void setAddressTakenIRBlock(BasicBlock *BB)
Set this block to reflect that it corresponds to an IR-level basic block with a BlockAddress.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
void setHasMustTailInVarArgFunc(bool B)
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
ArrayRef< int > allocateShuffleMask(ArrayRef< int > Mask)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
void push_back(MachineBasicBlock *MBB)
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
void deleteMachineBasicBlock(MachineBasicBlock *MBB)
DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
void remove(iterator MBBI)
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
Helper class to build MachineInstr.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPEXT Op.
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, const SrcOp &Src2, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FMA Op0, Op1, Op2.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr,...
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
MachineInstrBuilder buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, Align Alignment)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
MachineInstrBuilder buildFIDbgValue(int FI, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in th...
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
const DebugLoc & getDebugLoc()
Get the current instruction's debug location.
MachineInstrBuilder buildTrap(bool Debug=false)
Build and insert G_TRAP or G_DEBUGTRAP.
MachineInstrBuilder buildFFrexp(const DstOp &Fract, const DstOp &Exp, const SrcOp &Src, std::optional< unsigned > Flags=std::nullopt)
Build and insert Fract, Exp = G_FFREXP Src.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef< int > Mask)
Build and insert Res = G_SHUFFLE_VECTOR Src1, Src2, Mask.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildPrefetch(const SrcOp &Addr, unsigned RW, unsigned Locality, unsigned CacheType, MachineMemOperand &MMO)
Build and insert G_PREFETCH Addr, RW, Locality, CacheType.
const DataLayout & getDataLayout() const
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Val)
Build and insert Res = G_SPLAT_VECTOR Val.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:69
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
static uint32_t copyFlagsFromInstruction(const Instruction &I)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:556
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MCContext & getContext() const
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
Representation for a specific memory location.
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
The optimization diagnostic interface.
Diagnostic information for missed-optimization remarks.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
A simple RAII based Delegate installer.
A simple RAII based Observer installer.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Definition: Register.h:110
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents the LLVM 'select' instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
void initialize(const BasicBlock *BB, MachineBasicBlock *MBB, bool FunctionBasedInstrumentation)
Initialize the stack protector descriptor structure for a new basic block.
MachineBasicBlock * getSuccessMBB()
void resetPerBBState()
Reset state that changes when we handle different basic blocks.
void resetPerFunctionState()
Reset state that only changes when we switch functions.
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitStackProtector() const
Returns true if all fields of the stack protector descriptor are initialized implying that we should/...
bool shouldEmitFunctionBasedCheckStackProtector() const
bool shouldEmitSDCheck(const BasicBlock &BB) const
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
An instruction for storing to memory.
Definition: Instructions.h:317
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:216
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Multiway switch.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual unsigned getVaListSizeInBits(const DataLayout &DL) const
Returns the size of the platform's va_list object.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool fallBackToDAGISel(const Instruction &Inst) const
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
const Triple & getTargetTriple() const
TargetOptions Options
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
Target-Independent Code Generator Pass Configuration Options.
virtual std::unique_ptr< CSEConfigBase > getCSEConfig() const
Returns the CSEConfig object to use for the current optimization level.
virtual bool isGISelCSEEnabled() const
Check whether continuous CSE should be enabled in GISel passes.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const InlineAsmLowering * getInlineAsmLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const CallLowering * getCallLowering() const
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetLowering * getTargetLowering() const
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:608
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:302
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:295
static IntegerType * getInt32Ty(LLVMContext &C)
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:225
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Value * getOperand(unsigned i) const
Definition: User.h:169
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:693
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
constexpr bool isZero() const
Definition: TypeSize.h:156
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:316
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:678
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:86
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:821
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Undef
Value of the register doesn't matter.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1522
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
@ CE
Windows NT (Windows on ARM)
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< PhiNode * > Phi
Definition: RDFGraph.h:390
NodeAddr< CodeNode * > Code
Definition: RDFGraph.h:388
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
@ Offset
Definition: DWP.cpp:456
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Definition: STLExtras.h:2415
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
gep_type_iterator gep_type_end(const User *GEP)
MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition: Local.h:241
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:428
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
Definition: Analysis.cpp:141
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
@ Global
Append to llvm.global_dtors.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:1072
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1963
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ FMul
Product of floats.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1833
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
unsigned succ_size(const MachineBasicBlock *BB)
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Pair of physical register and lane mask.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static bool canHandle(const Instruction *I, const TargetLibraryInfo &TLI)
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
struct PredInfoPair PredInfo