LLVM 18.0.0git
IRTranslator.cpp
Go to the documentation of this file.
1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallSet.h"
21#include "llvm/Analysis/Loads.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/CFG.h"
53#include "llvm/IR/Constant.h"
54#include "llvm/IR/Constants.h"
55#include "llvm/IR/DataLayout.h"
58#include "llvm/IR/Function.h"
60#include "llvm/IR/InlineAsm.h"
61#include "llvm/IR/InstrTypes.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/LLVMContext.h"
66#include "llvm/IR/Metadata.h"
68#include "llvm/IR/Statepoint.h"
69#include "llvm/IR/Type.h"
70#include "llvm/IR/User.h"
71#include "llvm/IR/Value.h"
73#include "llvm/MC/MCContext.h"
74#include "llvm/Pass.h"
77#include "llvm/Support/Debug.h"
84#include <algorithm>
85#include <cassert>
86#include <cstdint>
87#include <iterator>
88#include <optional>
89#include <string>
90#include <utility>
91#include <vector>
92
93#define DEBUG_TYPE "irtranslator"
94
95using namespace llvm;
96
97static cl::opt<bool>
98 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
99 cl::desc("Should enable CSE in irtranslator"),
100 cl::Optional, cl::init(false));
101char IRTranslator::ID = 0;
102
103INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
104 false, false)
112
117 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
118
119 // Print the function name explicitly if we don't have a debug location (which
120 // makes the diagnostic less useful) or if we're going to emit a raw error.
121 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
122 R << (" (in function: " + MF.getName() + ")").str();
123
124 if (TPC.isGlobalISelAbortEnabled())
125 report_fatal_error(Twine(R.getMsg()));
126 else
127 ORE.emit(R);
128}
129
131 : MachineFunctionPass(ID), OptLevel(optlevel) {}
132
133#ifndef NDEBUG
134namespace {
135/// Verify that every instruction created has the same DILocation as the
136/// instruction being translated.
137class DILocationVerifier : public GISelChangeObserver {
138 const Instruction *CurrInst = nullptr;
139
140public:
141 DILocationVerifier() = default;
142 ~DILocationVerifier() = default;
143
144 const Instruction *getCurrentInst() const { return CurrInst; }
145 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
146
147 void erasingInstr(MachineInstr &MI) override {}
148 void changingInstr(MachineInstr &MI) override {}
149 void changedInstr(MachineInstr &MI) override {}
150
151 void createdInstr(MachineInstr &MI) override {
152 assert(getCurrentInst() && "Inserted instruction without a current MI");
153
154 // Only print the check message if we're actually checking it.
155#ifndef NDEBUG
156 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
157 << " was copied to " << MI);
158#endif
159 // We allow insts in the entry block to have no debug loc because
160 // they could have originated from constants, and we don't want a jumpy
161 // debug experience.
162 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
163 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc())) &&
164 "Line info was not transferred to all instructions");
165 }
166};
167} // namespace
168#endif // ifndef NDEBUG
169
170
176 if (OptLevel != CodeGenOptLevel::None) {
179 }
184}
185
187IRTranslator::allocateVRegs(const Value &Val) {
188 auto VRegsIt = VMap.findVRegs(Val);
189 if (VRegsIt != VMap.vregs_end())
190 return *VRegsIt->second;
191 auto *Regs = VMap.getVRegs(Val);
192 auto *Offsets = VMap.getOffsets(Val);
193 SmallVector<LLT, 4> SplitTys;
194 computeValueLLTs(*DL, *Val.getType(), SplitTys,
195 Offsets->empty() ? Offsets : nullptr);
196 for (unsigned i = 0; i < SplitTys.size(); ++i)
197 Regs->push_back(0);
198 return *Regs;
199}
200
201ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
202 auto VRegsIt = VMap.findVRegs(Val);
203 if (VRegsIt != VMap.vregs_end())
204 return *VRegsIt->second;
205
206 if (Val.getType()->isVoidTy())
207 return *VMap.getVRegs(Val);
208
209 // Create entry for this type.
210 auto *VRegs = VMap.getVRegs(Val);
211 auto *Offsets = VMap.getOffsets(Val);
212
213 assert(Val.getType()->isSized() &&
214 "Don't know how to create an empty vreg");
215
216 SmallVector<LLT, 4> SplitTys;
217 computeValueLLTs(*DL, *Val.getType(), SplitTys,
218 Offsets->empty() ? Offsets : nullptr);
219
220 if (!isa<Constant>(Val)) {
221 for (auto Ty : SplitTys)
222 VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
223 return *VRegs;
224 }
225
226 if (Val.getType()->isAggregateType()) {
227 // UndefValue, ConstantAggregateZero
228 auto &C = cast<Constant>(Val);
229 unsigned Idx = 0;
230 while (auto Elt = C.getAggregateElement(Idx++)) {
231 auto EltRegs = getOrCreateVRegs(*Elt);
232 llvm::copy(EltRegs, std::back_inserter(*VRegs));
233 }
234 } else {
235 assert(SplitTys.size() == 1 && "unexpectedly split LLT");
236 VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
237 bool Success = translate(cast<Constant>(Val), VRegs->front());
238 if (!Success) {
239 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
241 &MF->getFunction().getEntryBlock());
242 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
243 reportTranslationError(*MF, *TPC, *ORE, R);
244 return *VRegs;
245 }
246 }
247
248 return *VRegs;
249}
250
251int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
252 auto MapEntry = FrameIndices.find(&AI);
253 if (MapEntry != FrameIndices.end())
254 return MapEntry->second;
255
256 uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
257 uint64_t Size =
258 ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
259
260 // Always allocate at least one byte.
261 Size = std::max<uint64_t>(Size, 1u);
262
263 int &FI = FrameIndices[&AI];
264 FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
265 return FI;
266}
267
268Align IRTranslator::getMemOpAlign(const Instruction &I) {
269 if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
270 return SI->getAlign();
271 if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
272 return LI->getAlign();
273 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
274 return AI->getAlign();
275 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
276 return AI->getAlign();
277
278 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
279 R << "unable to translate memop: " << ore::NV("Opcode", &I);
280 reportTranslationError(*MF, *TPC, *ORE, R);
281 return Align(1);
282}
283
284MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
285 MachineBasicBlock *&MBB = BBToMBB[&BB];
286 assert(MBB && "BasicBlock was not encountered before");
287 return *MBB;
288}
289
290void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
291 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
292 MachinePreds[Edge].push_back(NewPred);
293}
294
295bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
296 MachineIRBuilder &MIRBuilder) {
297 // Get or create a virtual register for each value.
298 // Unless the value is a Constant => loadimm cst?
299 // or inline constant each time?
300 // Creation of a virtual register needs to have a size.
301 Register Op0 = getOrCreateVReg(*U.getOperand(0));
302 Register Op1 = getOrCreateVReg(*U.getOperand(1));
303 Register Res = getOrCreateVReg(U);
304 uint32_t Flags = 0;
305 if (isa<Instruction>(U)) {
306 const Instruction &I = cast<Instruction>(U);
308 }
309
310 MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
311 return true;
312}
313
314bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
315 MachineIRBuilder &MIRBuilder) {
316 Register Op0 = getOrCreateVReg(*U.getOperand(0));
317 Register Res = getOrCreateVReg(U);
318 uint32_t Flags = 0;
319 if (isa<Instruction>(U)) {
320 const Instruction &I = cast<Instruction>(U);
322 }
323 MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);
324 return true;
325}
326
327bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
328 return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
329}
330
331bool IRTranslator::translateCompare(const User &U,
332 MachineIRBuilder &MIRBuilder) {
333 auto *CI = dyn_cast<CmpInst>(&U);
334 Register Op0 = getOrCreateVReg(*U.getOperand(0));
335 Register Op1 = getOrCreateVReg(*U.getOperand(1));
336 Register Res = getOrCreateVReg(U);
337 CmpInst::Predicate Pred =
338 CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
339 cast<ConstantExpr>(U).getPredicate());
340 if (CmpInst::isIntPredicate(Pred))
341 MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
342 else if (Pred == CmpInst::FCMP_FALSE)
343 MIRBuilder.buildCopy(
344 Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
345 else if (Pred == CmpInst::FCMP_TRUE)
346 MIRBuilder.buildCopy(
347 Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
348 else {
349 uint32_t Flags = 0;
350 if (CI)
352 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
353 }
354
355 return true;
356}
357
358bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
359 const ReturnInst &RI = cast<ReturnInst>(U);
360 const Value *Ret = RI.getReturnValue();
361 if (Ret && DL->getTypeStoreSize(Ret->getType()) == 0)
362 Ret = nullptr;
363
364 ArrayRef<Register> VRegs;
365 if (Ret)
366 VRegs = getOrCreateVRegs(*Ret);
367
368 Register SwiftErrorVReg = 0;
369 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
370 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
371 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
372 }
373
374 // The target may mess up with the insertion point, but
375 // this is not important as a return is the last instruction
376 // of the block anyway.
377 return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
378}
379
380void IRTranslator::emitBranchForMergedCondition(
382 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
383 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
384 // If the leaf of the tree is a comparison, merge the condition into
385 // the caseblock.
386 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
387 CmpInst::Predicate Condition;
388 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
389 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
390 } else {
391 const FCmpInst *FC = cast<FCmpInst>(Cond);
392 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
393 }
394
395 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),
396 BOp->getOperand(1), nullptr, TBB, FBB, CurBB,
397 CurBuilder->getDebugLoc(), TProb, FProb);
398 SL->SwitchCases.push_back(CB);
399 return;
400 }
401
402 // Create a CaseBlock record representing this branch.
405 Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),
406 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
407 SL->SwitchCases.push_back(CB);
408}
409
410static bool isValInBlock(const Value *V, const BasicBlock *BB) {
411 if (const Instruction *I = dyn_cast<Instruction>(V))
412 return I->getParent() == BB;
413 return true;
414}
415
416void IRTranslator::findMergedConditions(
418 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
420 BranchProbability FProb, bool InvertCond) {
421 using namespace PatternMatch;
422 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
423 "Expected Opc to be AND/OR");
424 // Skip over not part of the tree and remember to invert op and operands at
425 // next level.
426 Value *NotCond;
427 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
428 isValInBlock(NotCond, CurBB->getBasicBlock())) {
429 findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
430 !InvertCond);
431 return;
432 }
433
434 const Instruction *BOp = dyn_cast<Instruction>(Cond);
435 const Value *BOpOp0, *BOpOp1;
436 // Compute the effective opcode for Cond, taking into account whether it needs
437 // to be inverted, e.g.
438 // and (not (or A, B)), C
439 // gets lowered as
440 // and (and (not A, not B), C)
442 if (BOp) {
443 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
444 ? Instruction::And
445 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
446 ? Instruction::Or
448 if (InvertCond) {
449 if (BOpc == Instruction::And)
450 BOpc = Instruction::Or;
451 else if (BOpc == Instruction::Or)
452 BOpc = Instruction::And;
453 }
454 }
455
456 // If this node is not part of the or/and tree, emit it as a branch.
457 // Note that all nodes in the tree should have same opcode.
458 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
459 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
460 !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||
461 !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {
462 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
463 InvertCond);
464 return;
465 }
466
467 // Create TmpBB after CurBB.
468 MachineFunction::iterator BBI(CurBB);
469 MachineBasicBlock *TmpBB =
471 CurBB->getParent()->insert(++BBI, TmpBB);
472
473 if (Opc == Instruction::Or) {
474 // Codegen X | Y as:
475 // BB1:
476 // jmp_if_X TBB
477 // jmp TmpBB
478 // TmpBB:
479 // jmp_if_Y TBB
480 // jmp FBB
481 //
482
483 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
484 // The requirement is that
485 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
486 // = TrueProb for original BB.
487 // Assuming the original probabilities are A and B, one choice is to set
488 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
489 // A/(1+B) and 2B/(1+B). This choice assumes that
490 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
491 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
492 // TmpBB, but the math is more complicated.
493
494 auto NewTrueProb = TProb / 2;
495 auto NewFalseProb = TProb / 2 + FProb;
496 // Emit the LHS condition.
497 findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
498 NewFalseProb, InvertCond);
499
500 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
501 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
502 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
503 // Emit the RHS condition into TmpBB.
504 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
505 Probs[1], InvertCond);
506 } else {
507 assert(Opc == Instruction::And && "Unknown merge op!");
508 // Codegen X & Y as:
509 // BB1:
510 // jmp_if_X TmpBB
511 // jmp FBB
512 // TmpBB:
513 // jmp_if_Y TBB
514 // jmp FBB
515 //
516 // This requires creation of TmpBB after CurBB.
517
518 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
519 // The requirement is that
520 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
521 // = FalseProb for original BB.
522 // Assuming the original probabilities are A and B, one choice is to set
523 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
524 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
525 // TrueProb for BB1 * FalseProb for TmpBB.
526
527 auto NewTrueProb = TProb + FProb / 2;
528 auto NewFalseProb = FProb / 2;
529 // Emit the LHS condition.
530 findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
531 NewFalseProb, InvertCond);
532
533 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
534 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
535 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
536 // Emit the RHS condition into TmpBB.
537 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
538 Probs[1], InvertCond);
539 }
540}
541
542bool IRTranslator::shouldEmitAsBranches(
543 const std::vector<SwitchCG::CaseBlock> &Cases) {
544 // For multiple cases, it's better to emit as branches.
545 if (Cases.size() != 2)
546 return true;
547
548 // If this is two comparisons of the same values or'd or and'd together, they
549 // will get folded into a single comparison, so don't emit two blocks.
550 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
551 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
552 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
553 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
554 return false;
555 }
556
557 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
558 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
559 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
560 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
561 isa<Constant>(Cases[0].CmpRHS) &&
562 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
563 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
564 Cases[0].TrueBB == Cases[1].ThisBB)
565 return false;
566 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
567 Cases[0].FalseBB == Cases[1].ThisBB)
568 return false;
569 }
570
571 return true;
572}
573
574bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
575 const BranchInst &BrInst = cast<BranchInst>(U);
576 auto &CurMBB = MIRBuilder.getMBB();
577 auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));
578
579 if (BrInst.isUnconditional()) {
580 // If the unconditional target is the layout successor, fallthrough.
581 if (OptLevel == CodeGenOptLevel::None ||
582 !CurMBB.isLayoutSuccessor(Succ0MBB))
583 MIRBuilder.buildBr(*Succ0MBB);
584
585 // Link successors.
586 for (const BasicBlock *Succ : successors(&BrInst))
587 CurMBB.addSuccessor(&getMBB(*Succ));
588 return true;
589 }
590
591 // If this condition is one of the special cases we handle, do special stuff
592 // now.
593 const Value *CondVal = BrInst.getCondition();
594 MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));
595
596 const auto &TLI = *MF->getSubtarget().getTargetLowering();
597
598 // If this is a series of conditions that are or'd or and'd together, emit
599 // this as a sequence of branches instead of setcc's with and/or operations.
600 // As long as jumps are not expensive (exceptions for multi-use logic ops,
601 // unpredictable branches, and vector extracts because those jumps are likely
602 // expensive for any target), this should improve performance.
603 // For example, instead of something like:
604 // cmp A, B
605 // C = seteq
606 // cmp D, E
607 // F = setle
608 // or C, F
609 // jnz foo
610 // Emit:
611 // cmp A, B
612 // je foo
613 // cmp D, E
614 // jle foo
615 using namespace PatternMatch;
616 const Instruction *CondI = dyn_cast<Instruction>(CondVal);
617 if (!TLI.isJumpExpensive() && CondI && CondI->hasOneUse() &&
618 !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {
620 Value *Vec;
621 const Value *BOp0, *BOp1;
622 if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
623 Opcode = Instruction::And;
624 else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
625 Opcode = Instruction::Or;
626
627 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
628 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
629 findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
630 getEdgeProbability(&CurMBB, Succ0MBB),
631 getEdgeProbability(&CurMBB, Succ1MBB),
632 /*InvertCond=*/false);
633 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
634
635 // Allow some cases to be rejected.
636 if (shouldEmitAsBranches(SL->SwitchCases)) {
637 // Emit the branch for this block.
638 emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
639 SL->SwitchCases.erase(SL->SwitchCases.begin());
640 return true;
641 }
642
643 // Okay, we decided not to do this, remove any inserted MBB's and clear
644 // SwitchCases.
645 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
646 MF->erase(SL->SwitchCases[I].ThisBB);
647
648 SL->SwitchCases.clear();
649 }
650 }
651
652 // Create a CaseBlock record representing this branch.
653 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
655 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
656 CurBuilder->getDebugLoc());
657
658 // Use emitSwitchCase to actually insert the fast branch sequence for this
659 // cond branch.
660 emitSwitchCase(CB, &CurMBB, *CurBuilder);
661 return true;
662}
663
664void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
666 BranchProbability Prob) {
667 if (!FuncInfo.BPI) {
668 Src->addSuccessorWithoutProb(Dst);
669 return;
670 }
671 if (Prob.isUnknown())
672 Prob = getEdgeProbability(Src, Dst);
673 Src->addSuccessor(Dst, Prob);
674}
675
677IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
678 const MachineBasicBlock *Dst) const {
679 const BasicBlock *SrcBB = Src->getBasicBlock();
680 const BasicBlock *DstBB = Dst->getBasicBlock();
681 if (!FuncInfo.BPI) {
682 // If BPI is not available, set the default probability as 1 / N, where N is
683 // the number of successors.
684 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
685 return BranchProbability(1, SuccSize);
686 }
687 return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
688}
689
690bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
691 using namespace SwitchCG;
692 // Extract cases from the switch.
693 const SwitchInst &SI = cast<SwitchInst>(U);
694 BranchProbabilityInfo *BPI = FuncInfo.BPI;
695 CaseClusterVector Clusters;
696 Clusters.reserve(SI.getNumCases());
697 for (const auto &I : SI.cases()) {
698 MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
699 assert(Succ && "Could not find successor mbb in mapping");
700 const ConstantInt *CaseVal = I.getCaseValue();
701 BranchProbability Prob =
702 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
703 : BranchProbability(1, SI.getNumCases() + 1);
704 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
705 }
706
707 MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
708
709 // Cluster adjacent cases with the same destination. We do this at all
710 // optimization levels because it's cheap to do and will make codegen faster
711 // if there are many clusters.
712 sortAndRangeify(Clusters);
713
714 MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
715
716 // If there is only the default destination, jump there directly.
717 if (Clusters.empty()) {
718 SwitchMBB->addSuccessor(DefaultMBB);
719 if (DefaultMBB != SwitchMBB->getNextNode())
720 MIB.buildBr(*DefaultMBB);
721 return true;
722 }
723
724 SL->findJumpTables(Clusters, &SI, DefaultMBB, nullptr, nullptr);
725 SL->findBitTestClusters(Clusters, &SI);
726
727 LLVM_DEBUG({
728 dbgs() << "Case clusters: ";
729 for (const CaseCluster &C : Clusters) {
730 if (C.Kind == CC_JumpTable)
731 dbgs() << "JT:";
732 if (C.Kind == CC_BitTests)
733 dbgs() << "BT:";
734
735 C.Low->getValue().print(dbgs(), true);
736 if (C.Low != C.High) {
737 dbgs() << '-';
738 C.High->getValue().print(dbgs(), true);
739 }
740 dbgs() << ' ';
741 }
742 dbgs() << '\n';
743 });
744
745 assert(!Clusters.empty());
746 SwitchWorkList WorkList;
747 CaseClusterIt First = Clusters.begin();
748 CaseClusterIt Last = Clusters.end() - 1;
749 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
750 WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
751
752 // FIXME: At the moment we don't do any splitting optimizations here like
753 // SelectionDAG does, so this worklist only has one entry.
754 while (!WorkList.empty()) {
755 SwitchWorkListItem W = WorkList.pop_back_val();
756 if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
757 return false;
758 }
759 return true;
760}
761
762void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
764 // Emit the code for the jump table
765 assert(JT.Reg != -1U && "Should lower JT Header first!");
767 MIB.setMBB(*MBB);
768 MIB.setDebugLoc(CurBuilder->getDebugLoc());
769
770 Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
771 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
772
773 auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
774 MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
775}
776
777bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
779 MachineBasicBlock *HeaderBB) {
780 MachineIRBuilder MIB(*HeaderBB->getParent());
781 MIB.setMBB(*HeaderBB);
782 MIB.setDebugLoc(CurBuilder->getDebugLoc());
783
784 const Value &SValue = *JTH.SValue;
785 // Subtract the lowest switch case value from the value being switched on.
786 const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
787 Register SwitchOpReg = getOrCreateVReg(SValue);
788 auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
789 auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
790
791 // This value may be smaller or larger than the target's pointer type, and
792 // therefore require extension or truncating.
793 Type *PtrIRTy = SValue.getType()->getPointerTo();
794 const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
795 Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
796
797 JT.Reg = Sub.getReg(0);
798
799 if (JTH.FallthroughUnreachable) {
800 if (JT.MBB != HeaderBB->getNextNode())
801 MIB.buildBr(*JT.MBB);
802 return true;
803 }
804
805 // Emit the range check for the jump table, and branch to the default block
806 // for the switch statement if the value being switched on exceeds the
807 // largest case in the switch.
808 auto Cst = getOrCreateVReg(
809 *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
810 Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
811 auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
812
813 auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
814
815 // Avoid emitting unnecessary branches to the next block.
816 if (JT.MBB != HeaderBB->getNextNode())
817 BrCond = MIB.buildBr(*JT.MBB);
818 return true;
819}
820
821void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
822 MachineBasicBlock *SwitchBB,
823 MachineIRBuilder &MIB) {
824 Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
826 DebugLoc OldDbgLoc = MIB.getDebugLoc();
827 MIB.setDebugLoc(CB.DbgLoc);
828 MIB.setMBB(*CB.ThisBB);
829
830 if (CB.PredInfo.NoCmp) {
831 // Branch or fall through to TrueBB.
832 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
833 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
834 CB.ThisBB);
836 if (CB.TrueBB != CB.ThisBB->getNextNode())
837 MIB.buildBr(*CB.TrueBB);
838 MIB.setDebugLoc(OldDbgLoc);
839 return;
840 }
841
842 const LLT i1Ty = LLT::scalar(1);
843 // Build the compare.
844 if (!CB.CmpMHS) {
845 const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);
846 // For conditional branch lowering, we might try to do something silly like
847 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
848 // just re-use the existing condition vreg.
849 if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
851 Cond = CondLHS;
852 } else {
853 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
855 Cond =
856 MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
857 else
858 Cond =
859 MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
860 }
861 } else {
863 "Can only handle SLE ranges");
864
865 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
866 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
867
868 Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
869 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
870 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
871 Cond =
872 MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
873 } else {
874 const LLT CmpTy = MRI->getType(CmpOpReg);
875 auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
876 auto Diff = MIB.buildConstant(CmpTy, High - Low);
877 Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
878 }
879 }
880
881 // Update successor info
882 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
883
884 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
885 CB.ThisBB);
886
887 // TrueBB and FalseBB are always different unless the incoming IR is
888 // degenerate. This only happens when running llc on weird IR.
889 if (CB.TrueBB != CB.FalseBB)
890 addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
892
893 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
894 CB.ThisBB);
895
896 MIB.buildBrCond(Cond, *CB.TrueBB);
897 MIB.buildBr(*CB.FalseBB);
898 MIB.setDebugLoc(OldDbgLoc);
899}
900
901bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
902 MachineBasicBlock *SwitchMBB,
903 MachineBasicBlock *CurMBB,
904 MachineBasicBlock *DefaultMBB,
905 MachineIRBuilder &MIB,
907 BranchProbability UnhandledProbs,
909 MachineBasicBlock *Fallthrough,
910 bool FallthroughUnreachable) {
911 using namespace SwitchCG;
912 MachineFunction *CurMF = SwitchMBB->getParent();
913 // FIXME: Optimize away range check based on pivot comparisons.
914 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
915 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
916 BranchProbability DefaultProb = W.DefaultProb;
917
918 // The jump block hasn't been inserted yet; insert it here.
919 MachineBasicBlock *JumpMBB = JT->MBB;
920 CurMF->insert(BBI, JumpMBB);
921
922 // Since the jump table block is separate from the switch block, we need
923 // to keep track of it as a machine predecessor to the default block,
924 // otherwise we lose the phi edges.
925 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
926 CurMBB);
927 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
928 JumpMBB);
929
930 auto JumpProb = I->Prob;
931 auto FallthroughProb = UnhandledProbs;
932
933 // If the default statement is a target of the jump table, we evenly
934 // distribute the default probability to successors of CurMBB. Also
935 // update the probability on the edge from JumpMBB to Fallthrough.
936 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
937 SE = JumpMBB->succ_end();
938 SI != SE; ++SI) {
939 if (*SI == DefaultMBB) {
940 JumpProb += DefaultProb / 2;
941 FallthroughProb -= DefaultProb / 2;
942 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
943 JumpMBB->normalizeSuccProbs();
944 } else {
945 // Also record edges from the jump table block to it's successors.
946 addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
947 JumpMBB);
948 }
949 }
950
951 if (FallthroughUnreachable)
952 JTH->FallthroughUnreachable = true;
953
954 if (!JTH->FallthroughUnreachable)
955 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
956 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
957 CurMBB->normalizeSuccProbs();
958
959 // The jump table header will be inserted in our current block, do the
960 // range check, and fall through to our fallthrough block.
961 JTH->HeaderBB = CurMBB;
962 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
963
964 // If we're in the right place, emit the jump table header right now.
965 if (CurMBB == SwitchMBB) {
966 if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
967 return false;
968 JTH->Emitted = true;
969 }
970 return true;
971}
972bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
973 Value *Cond,
974 MachineBasicBlock *Fallthrough,
975 bool FallthroughUnreachable,
976 BranchProbability UnhandledProbs,
977 MachineBasicBlock *CurMBB,
978 MachineIRBuilder &MIB,
979 MachineBasicBlock *SwitchMBB) {
980 using namespace SwitchCG;
981 const Value *RHS, *LHS, *MHS;
983 if (I->Low == I->High) {
984 // Check Cond == I->Low.
985 Pred = CmpInst::ICMP_EQ;
986 LHS = Cond;
987 RHS = I->Low;
988 MHS = nullptr;
989 } else {
990 // Check I->Low <= Cond <= I->High.
991 Pred = CmpInst::ICMP_SLE;
992 LHS = I->Low;
993 MHS = Cond;
994 RHS = I->High;
995 }
996
997 // If Fallthrough is unreachable, fold away the comparison.
998 // The false probability is the sum of all unhandled cases.
999 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1000 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1001
1002 emitSwitchCase(CB, SwitchMBB, MIB);
1003 return true;
1004}
1005
1006void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1007 MachineBasicBlock *SwitchBB) {
1008 MachineIRBuilder &MIB = *CurBuilder;
1009 MIB.setMBB(*SwitchBB);
1010
1011 // Subtract the minimum value.
1012 Register SwitchOpReg = getOrCreateVReg(*B.SValue);
1013
1014 LLT SwitchOpTy = MRI->getType(SwitchOpReg);
1015 Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);
1016 auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1017
1018 Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
1019 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1020
1021 LLT MaskTy = SwitchOpTy;
1022 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1023 !llvm::has_single_bit<uint32_t>(MaskTy.getSizeInBits()))
1024 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1025 else {
1026 // Ensure that the type will fit the mask value.
1027 for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1028 if (!isUIntN(SwitchOpTy.getSizeInBits(), B.Cases[I].Mask)) {
1029 // Switch table case range are encoded into series of masks.
1030 // Just use pointer type, it's guaranteed to fit.
1031 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1032 break;
1033 }
1034 }
1035 }
1036 Register SubReg = RangeSub.getReg(0);
1037 if (SwitchOpTy != MaskTy)
1038 SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);
1039
1040 B.RegVT = getMVTForLLT(MaskTy);
1041 B.Reg = SubReg;
1042
1043 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1044
1045 if (!B.FallthroughUnreachable)
1046 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
1047 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
1048
1049 SwitchBB->normalizeSuccProbs();
1050
1051 if (!B.FallthroughUnreachable) {
1052 // Conditional branch to the default block.
1053 auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);
1054 auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),
1055 RangeSub, RangeCst);
1056 MIB.buildBrCond(RangeCmp, *B.Default);
1057 }
1058
1059 // Avoid emitting unnecessary branches to the next block.
1060 if (MBB != SwitchBB->getNextNode())
1061 MIB.buildBr(*MBB);
1062}
1063
1064void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1065 MachineBasicBlock *NextMBB,
1066 BranchProbability BranchProbToNext,
1068 MachineBasicBlock *SwitchBB) {
1069 MachineIRBuilder &MIB = *CurBuilder;
1070 MIB.setMBB(*SwitchBB);
1071
1072 LLT SwitchTy = getLLTForMVT(BB.RegVT);
1073 Register Cmp;
1074 unsigned PopCount = llvm::popcount(B.Mask);
1075 if (PopCount == 1) {
1076 // Testing for a single bit; just compare the shift count with what it
1077 // would need to be to shift a 1 bit in that position.
1078 auto MaskTrailingZeros =
1079 MIB.buildConstant(SwitchTy, llvm::countr_zero(B.Mask));
1080 Cmp =
1081 MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
1082 .getReg(0);
1083 } else if (PopCount == BB.Range) {
1084 // There is only one zero bit in the range, test for it directly.
1085 auto MaskTrailingOnes =
1086 MIB.buildConstant(SwitchTy, llvm::countr_one(B.Mask));
1087 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
1088 .getReg(0);
1089 } else {
1090 // Make desired shift.
1091 auto CstOne = MIB.buildConstant(SwitchTy, 1);
1092 auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);
1093
1094 // Emit bit tests and jumps.
1095 auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);
1096 auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);
1097 auto CstZero = MIB.buildConstant(SwitchTy, 0);
1098 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)
1099 .getReg(0);
1100 }
1101
1102 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1103 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
1104 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1105 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1106 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1107 // one as they are relative probabilities (and thus work more like weights),
1108 // and hence we need to normalize them to let the sum of them become one.
1109 SwitchBB->normalizeSuccProbs();
1110
1111 // Record the fact that the IR edge from the header to the bit test target
1112 // will go through our new block. Neeeded for PHIs to have nodes added.
1113 addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1114 SwitchBB);
1115
1116 MIB.buildBrCond(Cmp, *B.TargetBB);
1117
1118 // Avoid emitting unnecessary branches to the next block.
1119 if (NextMBB != SwitchBB->getNextNode())
1120 MIB.buildBr(*NextMBB);
1121}
1122
1123bool IRTranslator::lowerBitTestWorkItem(
1125 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1127 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1129 bool FallthroughUnreachable) {
1130 using namespace SwitchCG;
1131 MachineFunction *CurMF = SwitchMBB->getParent();
1132 // FIXME: Optimize away range check based on pivot comparisons.
1133 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1134 // The bit test blocks haven't been inserted yet; insert them here.
1135 for (BitTestCase &BTC : BTB->Cases)
1136 CurMF->insert(BBI, BTC.ThisBB);
1137
1138 // Fill in fields of the BitTestBlock.
1139 BTB->Parent = CurMBB;
1140 BTB->Default = Fallthrough;
1141
1142 BTB->DefaultProb = UnhandledProbs;
1143 // If the cases in bit test don't form a contiguous range, we evenly
1144 // distribute the probability on the edge to Fallthrough to two
1145 // successors of CurMBB.
1146 if (!BTB->ContiguousRange) {
1147 BTB->Prob += DefaultProb / 2;
1148 BTB->DefaultProb -= DefaultProb / 2;
1149 }
1150
1151 if (FallthroughUnreachable)
1152 BTB->FallthroughUnreachable = true;
1153
1154 // If we're in the right place, emit the bit test header right now.
1155 if (CurMBB == SwitchMBB) {
1156 emitBitTestHeader(*BTB, SwitchMBB);
1157 BTB->Emitted = true;
1158 }
1159 return true;
1160}
1161
1162bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1163 Value *Cond,
1164 MachineBasicBlock *SwitchMBB,
1165 MachineBasicBlock *DefaultMBB,
1166 MachineIRBuilder &MIB) {
1167 using namespace SwitchCG;
1168 MachineFunction *CurMF = FuncInfo.MF;
1169 MachineBasicBlock *NextMBB = nullptr;
1171 if (++BBI != FuncInfo.MF->end())
1172 NextMBB = &*BBI;
1173
1174 if (EnableOpts) {
1175 // Here, we order cases by probability so the most likely case will be
1176 // checked first. However, two clusters can have the same probability in
1177 // which case their relative ordering is non-deterministic. So we use Low
1178 // as a tie-breaker as clusters are guaranteed to never overlap.
1179 llvm::sort(W.FirstCluster, W.LastCluster + 1,
1180 [](const CaseCluster &a, const CaseCluster &b) {
1181 return a.Prob != b.Prob
1182 ? a.Prob > b.Prob
1183 : a.Low->getValue().slt(b.Low->getValue());
1184 });
1185
1186 // Rearrange the case blocks so that the last one falls through if possible
1187 // without changing the order of probabilities.
1188 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1189 --I;
1190 if (I->Prob > W.LastCluster->Prob)
1191 break;
1192 if (I->Kind == CC_Range && I->MBB == NextMBB) {
1193 std::swap(*I, *W.LastCluster);
1194 break;
1195 }
1196 }
1197 }
1198
1199 // Compute total probability.
1200 BranchProbability DefaultProb = W.DefaultProb;
1201 BranchProbability UnhandledProbs = DefaultProb;
1202 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1203 UnhandledProbs += I->Prob;
1204
1205 MachineBasicBlock *CurMBB = W.MBB;
1206 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1207 bool FallthroughUnreachable = false;
1208 MachineBasicBlock *Fallthrough;
1209 if (I == W.LastCluster) {
1210 // For the last cluster, fall through to the default destination.
1211 Fallthrough = DefaultMBB;
1212 FallthroughUnreachable = isa<UnreachableInst>(
1213 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1214 } else {
1215 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
1216 CurMF->insert(BBI, Fallthrough);
1217 }
1218 UnhandledProbs -= I->Prob;
1219
1220 switch (I->Kind) {
1221 case CC_BitTests: {
1222 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1223 DefaultProb, UnhandledProbs, I, Fallthrough,
1224 FallthroughUnreachable)) {
1225 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1226 return false;
1227 }
1228 break;
1229 }
1230
1231 case CC_JumpTable: {
1232 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1233 UnhandledProbs, I, Fallthrough,
1234 FallthroughUnreachable)) {
1235 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1236 return false;
1237 }
1238 break;
1239 }
1240 case CC_Range: {
1241 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1242 FallthroughUnreachable, UnhandledProbs,
1243 CurMBB, MIB, SwitchMBB)) {
1244 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1245 return false;
1246 }
1247 break;
1248 }
1249 }
1250 CurMBB = Fallthrough;
1251 }
1252
1253 return true;
1254}
1255
1256bool IRTranslator::translateIndirectBr(const User &U,
1257 MachineIRBuilder &MIRBuilder) {
1258 const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
1259
1260 const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
1261 MIRBuilder.buildBrIndirect(Tgt);
1262
1263 // Link successors.
1265 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1266 for (const BasicBlock *Succ : successors(&BrInst)) {
1267 // It's legal for indirectbr instructions to have duplicate blocks in the
1268 // destination list. We don't allow this in MIR. Skip anything that's
1269 // already a successor.
1270 if (!AddedSuccessors.insert(Succ).second)
1271 continue;
1272 CurBB.addSuccessor(&getMBB(*Succ));
1273 }
1274
1275 return true;
1276}
1277
1278static bool isSwiftError(const Value *V) {
1279 if (auto Arg = dyn_cast<Argument>(V))
1280 return Arg->hasSwiftErrorAttr();
1281 if (auto AI = dyn_cast<AllocaInst>(V))
1282 return AI->isSwiftError();
1283 return false;
1284}
1285
1286bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1287 const LoadInst &LI = cast<LoadInst>(U);
1288
1289 unsigned StoreSize = DL->getTypeStoreSize(LI.getType());
1290 if (StoreSize == 0)
1291 return true;
1292
1293 ArrayRef<Register> Regs = getOrCreateVRegs(LI);
1294 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
1295 Register Base = getOrCreateVReg(*LI.getPointerOperand());
1296 AAMDNodes AAInfo = LI.getAAMetadata();
1297
1298 const Value *Ptr = LI.getPointerOperand();
1299 Type *OffsetIRTy = DL->getIndexType(Ptr->getType());
1300 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1301
1302 if (CLI->supportSwiftError() && isSwiftError(Ptr)) {
1303 assert(Regs.size() == 1 && "swifterror should be single pointer");
1304 Register VReg =
1305 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1306 MIRBuilder.buildCopy(Regs[0], VReg);
1307 return true;
1308 }
1309
1310 auto &TLI = *MF->getSubtarget().getTargetLowering();
1312 TLI.getLoadMemOperandFlags(LI, *DL, AC, LibInfo);
1313 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1314 if (AA->pointsToConstantMemory(
1315 MemoryLocation(Ptr, LocationSize::precise(StoreSize), AAInfo))) {
1317 }
1318 }
1319
1320 const MDNode *Ranges =
1321 Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
1322 for (unsigned i = 0; i < Regs.size(); ++i) {
1323 Register Addr;
1324 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1325
1326 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
1327 Align BaseAlign = getMemOpAlign(LI);
1328 auto MMO = MF->getMachineMemOperand(
1329 Ptr, Flags, MRI->getType(Regs[i]),
1330 commonAlignment(BaseAlign, Offsets[i] / 8), AAInfo, Ranges,
1331 LI.getSyncScopeID(), LI.getOrdering());
1332 MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
1333 }
1334
1335 return true;
1336}
1337
1338bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1339 const StoreInst &SI = cast<StoreInst>(U);
1340 if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
1341 return true;
1342
1343 ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
1344 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
1345 Register Base = getOrCreateVReg(*SI.getPointerOperand());
1346
1347 Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());
1348 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1349
1350 if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
1351 assert(Vals.size() == 1 && "swifterror should be single pointer");
1352
1353 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1354 SI.getPointerOperand());
1355 MIRBuilder.buildCopy(VReg, Vals[0]);
1356 return true;
1357 }
1358
1359 auto &TLI = *MF->getSubtarget().getTargetLowering();
1360 MachineMemOperand::Flags Flags = TLI.getStoreMemOperandFlags(SI, *DL);
1361
1362 for (unsigned i = 0; i < Vals.size(); ++i) {
1363 Register Addr;
1364 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1365
1366 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1367 Align BaseAlign = getMemOpAlign(SI);
1368 auto MMO = MF->getMachineMemOperand(
1369 Ptr, Flags, MRI->getType(Vals[i]),
1370 commonAlignment(BaseAlign, Offsets[i] / 8), SI.getAAMetadata(), nullptr,
1371 SI.getSyncScopeID(), SI.getOrdering());
1372 MIRBuilder.buildStore(Vals[i], Addr, *MMO);
1373 }
1374 return true;
1375}
1376
1378 const Value *Src = U.getOperand(0);
1379 Type *Int32Ty = Type::getInt32Ty(U.getContext());
1380
1381 // getIndexedOffsetInType is designed for GEPs, so the first index is the
1382 // usual array element rather than looking into the actual aggregate.
1384 Indices.push_back(ConstantInt::get(Int32Ty, 0));
1385
1386 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
1387 for (auto Idx : EVI->indices())
1389 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1390 for (auto Idx : IVI->indices())
1392 } else {
1393 for (unsigned i = 1; i < U.getNumOperands(); ++i)
1394 Indices.push_back(U.getOperand(i));
1395 }
1396
1397 return 8 * static_cast<uint64_t>(
1398 DL.getIndexedOffsetInType(Src->getType(), Indices));
1399}
1400
1401bool IRTranslator::translateExtractValue(const User &U,
1402 MachineIRBuilder &MIRBuilder) {
1403 const Value *Src = U.getOperand(0);
1405 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1406 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
1407 unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
1408 auto &DstRegs = allocateVRegs(U);
1409
1410 for (unsigned i = 0; i < DstRegs.size(); ++i)
1411 DstRegs[i] = SrcRegs[Idx++];
1412
1413 return true;
1414}
1415
1416bool IRTranslator::translateInsertValue(const User &U,
1417 MachineIRBuilder &MIRBuilder) {
1418 const Value *Src = U.getOperand(0);
1420 auto &DstRegs = allocateVRegs(U);
1421 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
1422 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1423 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
1424 auto *InsertedIt = InsertedRegs.begin();
1425
1426 for (unsigned i = 0; i < DstRegs.size(); ++i) {
1427 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1428 DstRegs[i] = *InsertedIt++;
1429 else
1430 DstRegs[i] = SrcRegs[i];
1431 }
1432
1433 return true;
1434}
1435
1436bool IRTranslator::translateSelect(const User &U,
1437 MachineIRBuilder &MIRBuilder) {
1438 Register Tst = getOrCreateVReg(*U.getOperand(0));
1439 ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1440 ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1441 ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1442
1443 uint32_t Flags = 0;
1444 if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
1446
1447 for (unsigned i = 0; i < ResRegs.size(); ++i) {
1448 MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
1449 }
1450
1451 return true;
1452}
1453
1454bool IRTranslator::translateCopy(const User &U, const Value &V,
1455 MachineIRBuilder &MIRBuilder) {
1456 Register Src = getOrCreateVReg(V);
1457 auto &Regs = *VMap.getVRegs(U);
1458 if (Regs.empty()) {
1459 Regs.push_back(Src);
1460 VMap.getOffsets(U)->push_back(0);
1461 } else {
1462 // If we already assigned a vreg for this instruction, we can't change that.
1463 // Emit a copy to satisfy the users we already emitted.
1464 MIRBuilder.buildCopy(Regs[0], Src);
1465 }
1466 return true;
1467}
1468
1469bool IRTranslator::translateBitCast(const User &U,
1470 MachineIRBuilder &MIRBuilder) {
1471 // If we're bitcasting to the source type, we can reuse the source vreg.
1472 if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1473 getLLTForType(*U.getType(), *DL)) {
1474 // If the source is a ConstantInt then it was probably created by
1475 // ConstantHoisting and we should leave it alone.
1476 if (isa<ConstantInt>(U.getOperand(0)))
1477 return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1478 MIRBuilder);
1479 return translateCopy(U, *U.getOperand(0), MIRBuilder);
1480 }
1481
1482 return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1483}
1484
1485bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1486 MachineIRBuilder &MIRBuilder) {
1487 Register Op = getOrCreateVReg(*U.getOperand(0));
1488 Register Res = getOrCreateVReg(U);
1489 MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1490 return true;
1491}
1492
1493bool IRTranslator::translateGetElementPtr(const User &U,
1494 MachineIRBuilder &MIRBuilder) {
1495 Value &Op0 = *U.getOperand(0);
1496 Register BaseReg = getOrCreateVReg(Op0);
1497 Type *PtrIRTy = Op0.getType();
1498 LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1499 Type *OffsetIRTy = DL->getIndexType(PtrIRTy);
1500 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1501
1502 // Normalize Vector GEP - all scalar operands should be converted to the
1503 // splat vector.
1504 unsigned VectorWidth = 0;
1505
1506 // True if we should use a splat vector; using VectorWidth alone is not
1507 // sufficient.
1508 bool WantSplatVector = false;
1509 if (auto *VT = dyn_cast<VectorType>(U.getType())) {
1510 VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1511 // We don't produce 1 x N vectors; those are treated as scalars.
1512 WantSplatVector = VectorWidth > 1;
1513 }
1514
1515 // We might need to splat the base pointer into a vector if the offsets
1516 // are vectors.
1517 if (WantSplatVector && !PtrTy.isVector()) {
1518 BaseReg =
1519 MIRBuilder
1520 .buildSplatVector(LLT::fixed_vector(VectorWidth, PtrTy), BaseReg)
1521 .getReg(0);
1522 PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
1523 PtrTy = getLLTForType(*PtrIRTy, *DL);
1524 OffsetIRTy = DL->getIndexType(PtrIRTy);
1525 OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1526 }
1527
1528 int64_t Offset = 0;
1529 for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1530 GTI != E; ++GTI) {
1531 const Value *Idx = GTI.getOperand();
1532 if (StructType *StTy = GTI.getStructTypeOrNull()) {
1533 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1535 continue;
1536 } else {
1537 uint64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
1538
1539 // If this is a scalar constant or a splat vector of constants,
1540 // handle it quickly.
1541 if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1542 Offset += ElementSize * CI->getSExtValue();
1543 continue;
1544 }
1545
1546 if (Offset != 0) {
1547 auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1548 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1549 .getReg(0);
1550 Offset = 0;
1551 }
1552
1553 Register IdxReg = getOrCreateVReg(*Idx);
1554 LLT IdxTy = MRI->getType(IdxReg);
1555 if (IdxTy != OffsetTy) {
1556 if (!IdxTy.isVector() && WantSplatVector) {
1557 IdxReg = MIRBuilder.buildSplatVector(
1558 OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0);
1559 }
1560
1561 IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1562 }
1563
1564 // N = N + Idx * ElementSize;
1565 // Avoid doing it for ElementSize of 1.
1566 Register GepOffsetReg;
1567 if (ElementSize != 1) {
1568 auto ElementSizeMIB = MIRBuilder.buildConstant(
1569 getLLTForType(*OffsetIRTy, *DL), ElementSize);
1570 GepOffsetReg =
1571 MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
1572 } else
1573 GepOffsetReg = IdxReg;
1574
1575 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1576 }
1577 }
1578
1579 if (Offset != 0) {
1580 auto OffsetMIB =
1581 MIRBuilder.buildConstant(OffsetTy, Offset);
1582 MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
1583 return true;
1584 }
1585
1586 MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1587 return true;
1588}
1589
1590bool IRTranslator::translateMemFunc(const CallInst &CI,
1591 MachineIRBuilder &MIRBuilder,
1592 unsigned Opcode) {
1593 const Value *SrcPtr = CI.getArgOperand(1);
1594 // If the source is undef, then just emit a nop.
1595 if (isa<UndefValue>(SrcPtr))
1596 return true;
1597
1599
1600 unsigned MinPtrSize = UINT_MAX;
1601 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {
1602 Register SrcReg = getOrCreateVReg(**AI);
1603 LLT SrcTy = MRI->getType(SrcReg);
1604 if (SrcTy.isPointer())
1605 MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);
1606 SrcRegs.push_back(SrcReg);
1607 }
1608
1609 LLT SizeTy = LLT::scalar(MinPtrSize);
1610
1611 // The size operand should be the minimum of the pointer sizes.
1612 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1613 if (MRI->getType(SizeOpReg) != SizeTy)
1614 SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);
1615
1616 auto ICall = MIRBuilder.buildInstr(Opcode);
1617 for (Register SrcReg : SrcRegs)
1618 ICall.addUse(SrcReg);
1619
1620 Align DstAlign;
1621 Align SrcAlign;
1622 unsigned IsVol =
1623 cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();
1624
1625 ConstantInt *CopySize = nullptr;
1626
1627 if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1628 DstAlign = MCI->getDestAlign().valueOrOne();
1629 SrcAlign = MCI->getSourceAlign().valueOrOne();
1630 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1631 } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
1632 DstAlign = MCI->getDestAlign().valueOrOne();
1633 SrcAlign = MCI->getSourceAlign().valueOrOne();
1634 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1635 } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1636 DstAlign = MMI->getDestAlign().valueOrOne();
1637 SrcAlign = MMI->getSourceAlign().valueOrOne();
1638 CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2));
1639 } else {
1640 auto *MSI = cast<MemSetInst>(&CI);
1641 DstAlign = MSI->getDestAlign().valueOrOne();
1642 }
1643
1644 if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1645 // We need to propagate the tail call flag from the IR inst as an argument.
1646 // Otherwise, we have to pessimize and assume later that we cannot tail call
1647 // any memory intrinsics.
1648 ICall.addImm(CI.isTailCall() ? 1 : 0);
1649 }
1650
1651 // Create mem operands to store the alignment and volatile info.
1654 if (IsVol) {
1655 LoadFlags |= MachineMemOperand::MOVolatile;
1656 StoreFlags |= MachineMemOperand::MOVolatile;
1657 }
1658
1659 AAMDNodes AAInfo = CI.getAAMetadata();
1660 if (AA && CopySize &&
1662 SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) {
1663 LoadFlags |= MachineMemOperand::MOInvariant;
1664
1665 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1666 // but the previous usage implied it did. Probably should check
1667 // isDereferenceableAndAlignedPointer.
1669 }
1670
1671 ICall.addMemOperand(
1673 StoreFlags, 1, DstAlign, AAInfo));
1674 if (Opcode != TargetOpcode::G_MEMSET)
1675 ICall.addMemOperand(MF->getMachineMemOperand(
1676 MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
1677
1678 return true;
1679}
1680
1681void IRTranslator::getStackGuard(Register DstReg,
1682 MachineIRBuilder &MIRBuilder) {
1684 MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1685 auto MIB =
1686 MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1687
1688 auto &TLI = *MF->getSubtarget().getTargetLowering();
1689 Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
1690 if (!Global)
1691 return;
1692
1693 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1694 LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
1695
1696 MachinePointerInfo MPInfo(Global);
1700 MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
1701 MIB.setMemRefs({MemRef});
1702}
1703
1704bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1705 MachineIRBuilder &MIRBuilder) {
1706 ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1707 MIRBuilder.buildInstr(
1708 Op, {ResRegs[0], ResRegs[1]},
1709 {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});
1710
1711 return true;
1712}
1713
1714bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1715 MachineIRBuilder &MIRBuilder) {
1716 Register Dst = getOrCreateVReg(CI);
1717 Register Src0 = getOrCreateVReg(*CI.getOperand(0));
1718 Register Src1 = getOrCreateVReg(*CI.getOperand(1));
1719 uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
1720 MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
1721 return true;
1722}
1723
1724unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1725 switch (ID) {
1726 default:
1727 break;
1728 case Intrinsic::bswap:
1729 return TargetOpcode::G_BSWAP;
1730 case Intrinsic::bitreverse:
1731 return TargetOpcode::G_BITREVERSE;
1732 case Intrinsic::fshl:
1733 return TargetOpcode::G_FSHL;
1734 case Intrinsic::fshr:
1735 return TargetOpcode::G_FSHR;
1736 case Intrinsic::ceil:
1737 return TargetOpcode::G_FCEIL;
1738 case Intrinsic::cos:
1739 return TargetOpcode::G_FCOS;
1740 case Intrinsic::ctpop:
1741 return TargetOpcode::G_CTPOP;
1742 case Intrinsic::exp:
1743 return TargetOpcode::G_FEXP;
1744 case Intrinsic::exp2:
1745 return TargetOpcode::G_FEXP2;
1746 case Intrinsic::exp10:
1747 return TargetOpcode::G_FEXP10;
1748 case Intrinsic::fabs:
1749 return TargetOpcode::G_FABS;
1750 case Intrinsic::copysign:
1751 return TargetOpcode::G_FCOPYSIGN;
1752 case Intrinsic::minnum:
1753 return TargetOpcode::G_FMINNUM;
1754 case Intrinsic::maxnum:
1755 return TargetOpcode::G_FMAXNUM;
1756 case Intrinsic::minimum:
1757 return TargetOpcode::G_FMINIMUM;
1758 case Intrinsic::maximum:
1759 return TargetOpcode::G_FMAXIMUM;
1760 case Intrinsic::canonicalize:
1761 return TargetOpcode::G_FCANONICALIZE;
1762 case Intrinsic::floor:
1763 return TargetOpcode::G_FFLOOR;
1764 case Intrinsic::fma:
1765 return TargetOpcode::G_FMA;
1766 case Intrinsic::log:
1767 return TargetOpcode::G_FLOG;
1768 case Intrinsic::log2:
1769 return TargetOpcode::G_FLOG2;
1770 case Intrinsic::log10:
1771 return TargetOpcode::G_FLOG10;
1772 case Intrinsic::ldexp:
1773 return TargetOpcode::G_FLDEXP;
1774 case Intrinsic::nearbyint:
1775 return TargetOpcode::G_FNEARBYINT;
1776 case Intrinsic::pow:
1777 return TargetOpcode::G_FPOW;
1778 case Intrinsic::powi:
1779 return TargetOpcode::G_FPOWI;
1780 case Intrinsic::rint:
1781 return TargetOpcode::G_FRINT;
1782 case Intrinsic::round:
1783 return TargetOpcode::G_INTRINSIC_ROUND;
1784 case Intrinsic::roundeven:
1785 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1786 case Intrinsic::sin:
1787 return TargetOpcode::G_FSIN;
1788 case Intrinsic::sqrt:
1789 return TargetOpcode::G_FSQRT;
1790 case Intrinsic::trunc:
1791 return TargetOpcode::G_INTRINSIC_TRUNC;
1792 case Intrinsic::readcyclecounter:
1793 return TargetOpcode::G_READCYCLECOUNTER;
1794 case Intrinsic::ptrmask:
1795 return TargetOpcode::G_PTRMASK;
1796 case Intrinsic::lrint:
1797 return TargetOpcode::G_INTRINSIC_LRINT;
1798 // FADD/FMUL require checking the FMF, so are handled elsewhere.
1799 case Intrinsic::vector_reduce_fmin:
1800 return TargetOpcode::G_VECREDUCE_FMIN;
1801 case Intrinsic::vector_reduce_fmax:
1802 return TargetOpcode::G_VECREDUCE_FMAX;
1803 case Intrinsic::vector_reduce_fminimum:
1804 return TargetOpcode::G_VECREDUCE_FMINIMUM;
1805 case Intrinsic::vector_reduce_fmaximum:
1806 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
1807 case Intrinsic::vector_reduce_add:
1808 return TargetOpcode::G_VECREDUCE_ADD;
1809 case Intrinsic::vector_reduce_mul:
1810 return TargetOpcode::G_VECREDUCE_MUL;
1811 case Intrinsic::vector_reduce_and:
1812 return TargetOpcode::G_VECREDUCE_AND;
1813 case Intrinsic::vector_reduce_or:
1814 return TargetOpcode::G_VECREDUCE_OR;
1815 case Intrinsic::vector_reduce_xor:
1816 return TargetOpcode::G_VECREDUCE_XOR;
1817 case Intrinsic::vector_reduce_smax:
1818 return TargetOpcode::G_VECREDUCE_SMAX;
1819 case Intrinsic::vector_reduce_smin:
1820 return TargetOpcode::G_VECREDUCE_SMIN;
1821 case Intrinsic::vector_reduce_umax:
1822 return TargetOpcode::G_VECREDUCE_UMAX;
1823 case Intrinsic::vector_reduce_umin:
1824 return TargetOpcode::G_VECREDUCE_UMIN;
1825 case Intrinsic::lround:
1826 return TargetOpcode::G_LROUND;
1827 case Intrinsic::llround:
1828 return TargetOpcode::G_LLROUND;
1829 }
1831}
1832
1833bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1835 MachineIRBuilder &MIRBuilder) {
1836
1837 unsigned Op = getSimpleIntrinsicOpcode(ID);
1838
1839 // Is this a simple intrinsic?
1841 return false;
1842
1843 // Yes. Let's translate it.
1845 for (const auto &Arg : CI.args())
1846 VRegs.push_back(getOrCreateVReg(*Arg));
1847
1848 MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1850 return true;
1851}
1852
1853// TODO: Include ConstainedOps.def when all strict instructions are defined.
1855 switch (ID) {
1856 case Intrinsic::experimental_constrained_fadd:
1857 return TargetOpcode::G_STRICT_FADD;
1858 case Intrinsic::experimental_constrained_fsub:
1859 return TargetOpcode::G_STRICT_FSUB;
1860 case Intrinsic::experimental_constrained_fmul:
1861 return TargetOpcode::G_STRICT_FMUL;
1862 case Intrinsic::experimental_constrained_fdiv:
1863 return TargetOpcode::G_STRICT_FDIV;
1864 case Intrinsic::experimental_constrained_frem:
1865 return TargetOpcode::G_STRICT_FREM;
1866 case Intrinsic::experimental_constrained_fma:
1867 return TargetOpcode::G_STRICT_FMA;
1868 case Intrinsic::experimental_constrained_sqrt:
1869 return TargetOpcode::G_STRICT_FSQRT;
1870 case Intrinsic::experimental_constrained_ldexp:
1871 return TargetOpcode::G_STRICT_FLDEXP;
1872 default:
1873 return 0;
1874 }
1875}
1876
1877bool IRTranslator::translateConstrainedFPIntrinsic(
1878 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
1880
1881 unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
1882 if (!Opcode)
1883 return false;
1884
1888
1890 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(0)));
1891 if (!FPI.isUnaryOp())
1892 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(1)));
1893 if (FPI.isTernaryOp())
1894 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(2)));
1895
1896 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
1897 return true;
1898}
1899
1900std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
1901 auto VRegs = getOrCreateVRegs(Arg);
1902 if (VRegs.size() != 1)
1903 return std::nullopt;
1904
1905 // Arguments are lowered as a copy of a livein physical register.
1906 auto *VRegDef = MF->getRegInfo().getVRegDef(VRegs[0]);
1907 if (!VRegDef || !VRegDef->isCopy())
1908 return std::nullopt;
1909 return VRegDef->getOperand(1).getReg().asMCReg();
1910}
1911
1912bool IRTranslator::translateIfEntryValueArgument(const DbgValueInst &DebugInst,
1913 MachineIRBuilder &MIRBuilder) {
1914 auto *Arg = dyn_cast<Argument>(DebugInst.getValue());
1915 if (!Arg)
1916 return false;
1917
1918 const DIExpression *Expr = DebugInst.getExpression();
1919 if (!Expr->isEntryValue())
1920 return false;
1921
1922 std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
1923 if (!PhysReg) {
1924 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
1925 "couldn't find a physical register\n"
1926 << DebugInst << "\n");
1927 return true;
1928 }
1929
1930 MIRBuilder.buildDirectDbgValue(*PhysReg, DebugInst.getVariable(),
1931 DebugInst.getExpression());
1932 return true;
1933}
1934
1935bool IRTranslator::translateIfEntryValueArgument(
1936 const DbgDeclareInst &DebugInst) {
1937 auto *Arg = dyn_cast<Argument>(DebugInst.getAddress());
1938 if (!Arg)
1939 return false;
1940
1941 const DIExpression *Expr = DebugInst.getExpression();
1942 if (!Expr->isEntryValue())
1943 return false;
1944
1945 std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
1946 if (!PhysReg)
1947 return false;
1948
1949 // Append an op deref to account for the fact that this is a dbg_declare.
1950 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
1951 MF->setVariableDbgInfo(DebugInst.getVariable(), Expr, *PhysReg,
1952 DebugInst.getDebugLoc());
1953 return true;
1954}
1955
1956bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
1957 MachineIRBuilder &MIRBuilder) {
1958 if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
1959 if (ORE->enabled()) {
1960 if (MemoryOpRemark::canHandle(MI, *LibInfo)) {
1961 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
1962 R.visit(MI);
1963 }
1964 }
1965 }
1966
1967 // If this is a simple intrinsic (that is, we just need to add a def of
1968 // a vreg, and uses for each arg operand, then translate it.
1969 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
1970 return true;
1971
1972 switch (ID) {
1973 default:
1974 break;
1975 case Intrinsic::lifetime_start:
1976 case Intrinsic::lifetime_end: {
1977 // No stack colouring in O0, discard region information.
1979 return true;
1980
1981 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
1982 : TargetOpcode::LIFETIME_END;
1983
1984 // Get the underlying objects for the location passed on the lifetime
1985 // marker.
1987 getUnderlyingObjects(CI.getArgOperand(1), Allocas);
1988
1989 // Iterate over each underlying object, creating lifetime markers for each
1990 // static alloca. Quit if we find a non-static alloca.
1991 for (const Value *V : Allocas) {
1992 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
1993 if (!AI)
1994 continue;
1995
1996 if (!AI->isStaticAlloca())
1997 return true;
1998
1999 MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
2000 }
2001 return true;
2002 }
2003 case Intrinsic::dbg_declare: {
2004 const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
2005 assert(DI.getVariable() && "Missing variable");
2006
2007 const Value *Address = DI.getAddress();
2008 if (!Address || isa<UndefValue>(Address)) {
2009 LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
2010 return true;
2011 }
2012
2014 MIRBuilder.getDebugLoc()) &&
2015 "Expected inlined-at fields to agree");
2016 auto AI = dyn_cast<AllocaInst>(Address);
2017 if (AI && AI->isStaticAlloca()) {
2018 // Static allocas are tracked at the MF level, no need for DBG_VALUE
2019 // instructions (in fact, they get ignored if they *do* exist).
2021 getOrCreateFrameIndex(*AI), DI.getDebugLoc());
2022 return true;
2023 }
2024
2025 if (translateIfEntryValueArgument(DI))
2026 return true;
2027
2028 // A dbg.declare describes the address of a source variable, so lower it
2029 // into an indirect DBG_VALUE.
2030 MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
2031 DI.getVariable(), DI.getExpression());
2032 return true;
2033 }
2034 case Intrinsic::dbg_label: {
2035 const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
2036 assert(DI.getLabel() && "Missing label");
2037
2039 MIRBuilder.getDebugLoc()) &&
2040 "Expected inlined-at fields to agree");
2041
2042 MIRBuilder.buildDbgLabel(DI.getLabel());
2043 return true;
2044 }
2045 case Intrinsic::vaend:
2046 // No target I know of cares about va_end. Certainly no in-tree target
2047 // does. Simplest intrinsic ever!
2048 return true;
2049 case Intrinsic::vastart: {
2050 auto &TLI = *MF->getSubtarget().getTargetLowering();
2051 Value *Ptr = CI.getArgOperand(0);
2052 unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
2053
2054 // FIXME: Get alignment
2055 MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
2056 .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),
2058 ListSize, Align(1)));
2059 return true;
2060 }
2061 case Intrinsic::dbg_value: {
2062 // This form of DBG_VALUE is target-independent.
2063 const DbgValueInst &DI = cast<DbgValueInst>(CI);
2064 const Value *V = DI.getValue();
2066 MIRBuilder.getDebugLoc()) &&
2067 "Expected inlined-at fields to agree");
2068 if (!V || DI.hasArgList()) {
2069 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
2070 // terminate any prior location.
2071 MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
2072 return true;
2073 }
2074 if (const auto *CI = dyn_cast<Constant>(V)) {
2075 MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
2076 return true;
2077 }
2078 if (auto *AI = dyn_cast<AllocaInst>(V);
2079 AI && AI->isStaticAlloca() && DI.getExpression()->startsWithDeref()) {
2080 // If the value is an alloca and the expression starts with a
2081 // dereference, track a stack slot instead of a register, as registers
2082 // may be clobbered.
2083 auto ExprOperands = DI.getExpression()->getElements();
2084 auto *ExprDerefRemoved =
2085 DIExpression::get(AI->getContext(), ExprOperands.drop_front());
2086 MIRBuilder.buildFIDbgValue(getOrCreateFrameIndex(*AI), DI.getVariable(),
2087 ExprDerefRemoved);
2088 return true;
2089 }
2090 if (translateIfEntryValueArgument(DI, MIRBuilder))
2091 return true;
2092 for (Register Reg : getOrCreateVRegs(*V)) {
2093 // FIXME: This does not handle register-indirect values at offset 0. The
2094 // direct/indirect thing shouldn't really be handled by something as
2095 // implicit as reg+noreg vs reg+imm in the first place, but it seems
2096 // pretty baked in right now.
2097 MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
2098 }
2099 return true;
2100 }
2101 case Intrinsic::uadd_with_overflow:
2102 return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
2103 case Intrinsic::sadd_with_overflow:
2104 return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
2105 case Intrinsic::usub_with_overflow:
2106 return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
2107 case Intrinsic::ssub_with_overflow:
2108 return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
2109 case Intrinsic::umul_with_overflow:
2110 return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
2111 case Intrinsic::smul_with_overflow:
2112 return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
2113 case Intrinsic::uadd_sat:
2114 return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
2115 case Intrinsic::sadd_sat:
2116 return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
2117 case Intrinsic::usub_sat:
2118 return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
2119 case Intrinsic::ssub_sat:
2120 return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
2121 case Intrinsic::ushl_sat:
2122 return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
2123 case Intrinsic::sshl_sat:
2124 return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
2125 case Intrinsic::umin:
2126 return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
2127 case Intrinsic::umax:
2128 return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
2129 case Intrinsic::smin:
2130 return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
2131 case Intrinsic::smax:
2132 return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
2133 case Intrinsic::abs:
2134 // TODO: Preserve "int min is poison" arg in GMIR?
2135 return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
2136 case Intrinsic::smul_fix:
2137 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2138 case Intrinsic::umul_fix:
2139 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2140 case Intrinsic::smul_fix_sat:
2141 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2142 case Intrinsic::umul_fix_sat:
2143 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2144 case Intrinsic::sdiv_fix:
2145 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2146 case Intrinsic::udiv_fix:
2147 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2148 case Intrinsic::sdiv_fix_sat:
2149 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2150 case Intrinsic::udiv_fix_sat:
2151 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2152 case Intrinsic::fmuladd: {
2153 const TargetMachine &TM = MF->getTarget();
2154 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2155 Register Dst = getOrCreateVReg(CI);
2156 Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
2157 Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
2158 Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
2159 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2161 TLI.getValueType(*DL, CI.getType()))) {
2162 // TODO: Revisit this to see if we should move this part of the
2163 // lowering to the combiner.
2164 MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,
2166 } else {
2167 LLT Ty = getLLTForType(*CI.getType(), *DL);
2168 auto FMul = MIRBuilder.buildFMul(
2169 Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));
2170 MIRBuilder.buildFAdd(Dst, FMul, Op2,
2172 }
2173 return true;
2174 }
2175 case Intrinsic::convert_from_fp16:
2176 // FIXME: This intrinsic should probably be removed from the IR.
2177 MIRBuilder.buildFPExt(getOrCreateVReg(CI),
2178 getOrCreateVReg(*CI.getArgOperand(0)),
2180 return true;
2181 case Intrinsic::convert_to_fp16:
2182 // FIXME: This intrinsic should probably be removed from the IR.
2183 MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
2184 getOrCreateVReg(*CI.getArgOperand(0)),
2186 return true;
2187 case Intrinsic::frexp: {
2188 ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
2189 MIRBuilder.buildFFrexp(VRegs[0], VRegs[1],
2190 getOrCreateVReg(*CI.getArgOperand(0)),
2192 return true;
2193 }
2194 case Intrinsic::memcpy_inline:
2195 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
2196 case Intrinsic::memcpy:
2197 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2198 case Intrinsic::memmove:
2199 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2200 case Intrinsic::memset:
2201 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2202 case Intrinsic::eh_typeid_for: {
2204 Register Reg = getOrCreateVReg(CI);
2205 unsigned TypeID = MF->getTypeIDFor(GV);
2206 MIRBuilder.buildConstant(Reg, TypeID);
2207 return true;
2208 }
2209 case Intrinsic::objectsize:
2210 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2211
2212 case Intrinsic::is_constant:
2213 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2214
2215 case Intrinsic::stackguard:
2216 getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2217 return true;
2218 case Intrinsic::stackprotector: {
2219 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2220 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2221 Register GuardVal;
2222 if (TLI.useLoadStackGuardNode()) {
2223 GuardVal = MRI->createGenericVirtualRegister(PtrTy);
2224 getStackGuard(GuardVal, MIRBuilder);
2225 } else
2226 GuardVal = getOrCreateVReg(*CI.getArgOperand(0)); // The guard's value.
2227
2228 AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
2229 int FI = getOrCreateFrameIndex(*Slot);
2231
2232 MIRBuilder.buildStore(
2233 GuardVal, getOrCreateVReg(*Slot),
2237 PtrTy, Align(8)));
2238 return true;
2239 }
2240 case Intrinsic::stacksave: {
2241 MIRBuilder.buildInstr(TargetOpcode::G_STACKSAVE, {getOrCreateVReg(CI)}, {});
2242 return true;
2243 }
2244 case Intrinsic::stackrestore: {
2245 MIRBuilder.buildInstr(TargetOpcode::G_STACKRESTORE, {},
2246 {getOrCreateVReg(*CI.getArgOperand(0))});
2247 return true;
2248 }
2249 case Intrinsic::cttz:
2250 case Intrinsic::ctlz: {
2251 ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
2252 bool isTrailing = ID == Intrinsic::cttz;
2253 unsigned Opcode = isTrailing
2254 ? Cst->isZero() ? TargetOpcode::G_CTTZ
2255 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2256 : Cst->isZero() ? TargetOpcode::G_CTLZ
2257 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2258 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
2259 {getOrCreateVReg(*CI.getArgOperand(0))});
2260 return true;
2261 }
2262 case Intrinsic::invariant_start: {
2263 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2265 MIRBuilder.buildUndef(Undef);
2266 return true;
2267 }
2268 case Intrinsic::invariant_end:
2269 return true;
2270 case Intrinsic::expect:
2271 case Intrinsic::annotation:
2272 case Intrinsic::ptr_annotation:
2273 case Intrinsic::launder_invariant_group:
2274 case Intrinsic::strip_invariant_group: {
2275 // Drop the intrinsic, but forward the value.
2276 MIRBuilder.buildCopy(getOrCreateVReg(CI),
2277 getOrCreateVReg(*CI.getArgOperand(0)));
2278 return true;
2279 }
2280 case Intrinsic::assume:
2281 case Intrinsic::experimental_noalias_scope_decl:
2282 case Intrinsic::var_annotation:
2283 case Intrinsic::sideeffect:
2284 // Discard annotate attributes, assumptions, and artificial side-effects.
2285 return true;
2286 case Intrinsic::read_volatile_register:
2287 case Intrinsic::read_register: {
2288 Value *Arg = CI.getArgOperand(0);
2289 MIRBuilder
2290 .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2291 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
2292 return true;
2293 }
2294 case Intrinsic::write_register: {
2295 Value *Arg = CI.getArgOperand(0);
2296 MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
2297 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
2298 .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
2299 return true;
2300 }
2301 case Intrinsic::localescape: {
2302 MachineBasicBlock &EntryMBB = MF->front();
2304
2305 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2306 // is the same on all targets.
2307 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2309 if (isa<ConstantPointerNull>(Arg))
2310 continue; // Skip null pointers. They represent a hole in index space.
2311
2312 int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
2313 MCSymbol *FrameAllocSym =
2314 MF->getMMI().getContext().getOrCreateFrameAllocSymbol(EscapedName,
2315 Idx);
2316
2317 // This should be inserted at the start of the entry block.
2318 auto LocalEscape =
2319 MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)
2320 .addSym(FrameAllocSym)
2321 .addFrameIndex(FI);
2322
2323 EntryMBB.insert(EntryMBB.begin(), LocalEscape);
2324 }
2325
2326 return true;
2327 }
2328 case Intrinsic::vector_reduce_fadd:
2329 case Intrinsic::vector_reduce_fmul: {
2330 // Need to check for the reassoc flag to decide whether we want a
2331 // sequential reduction opcode or not.
2332 Register Dst = getOrCreateVReg(CI);
2333 Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));
2334 Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));
2335 unsigned Opc = 0;
2336 if (!CI.hasAllowReassoc()) {
2337 // The sequential ordering case.
2338 Opc = ID == Intrinsic::vector_reduce_fadd
2339 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2340 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2341 MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2343 return true;
2344 }
2345 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2346 // since the associativity doesn't matter.
2347 unsigned ScalarOpc;
2348 if (ID == Intrinsic::vector_reduce_fadd) {
2349 Opc = TargetOpcode::G_VECREDUCE_FADD;
2350 ScalarOpc = TargetOpcode::G_FADD;
2351 } else {
2352 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2353 ScalarOpc = TargetOpcode::G_FMUL;
2354 }
2355 LLT DstTy = MRI->getType(Dst);
2356 auto Rdx = MIRBuilder.buildInstr(
2357 Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));
2358 MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2360
2361 return true;
2362 }
2363 case Intrinsic::trap:
2364 case Intrinsic::debugtrap:
2365 case Intrinsic::ubsantrap: {
2366 StringRef TrapFuncName =
2367 CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
2368 if (TrapFuncName.empty())
2369 break; // Use the default handling.
2371 if (ID == Intrinsic::ubsantrap) {
2372 Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
2373 CI.getArgOperand(0)->getType(), 0});
2374 }
2375 Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
2376 Info.CB = &CI;
2377 Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
2378 return CLI->lowerCall(MIRBuilder, Info);
2379 }
2380 case Intrinsic::fptrunc_round: {
2382
2383 // Convert the metadata argument to a constant integer
2384 Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata();
2385 std::optional<RoundingMode> RoundMode =
2386 convertStrToRoundingMode(cast<MDString>(MD)->getString());
2387
2388 // Add the Rounding mode as an integer
2389 MIRBuilder
2390 .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2391 {getOrCreateVReg(CI)},
2392 {getOrCreateVReg(*CI.getArgOperand(0))}, Flags)
2393 .addImm((int)*RoundMode);
2394
2395 return true;
2396 }
2397 case Intrinsic::is_fpclass: {
2398 Value *FpValue = CI.getOperand(0);
2399 ConstantInt *TestMaskValue = cast<ConstantInt>(CI.getOperand(1));
2400
2401 MIRBuilder
2402 .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},
2403 {getOrCreateVReg(*FpValue)})
2404 .addImm(TestMaskValue->getZExtValue());
2405
2406 return true;
2407 }
2408#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2409 case Intrinsic::INTRINSIC:
2410#include "llvm/IR/ConstrainedOps.def"
2411 return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2412 MIRBuilder);
2413
2414 }
2415 return false;
2416}
2417
2418bool IRTranslator::translateInlineAsm(const CallBase &CB,
2419 MachineIRBuilder &MIRBuilder) {
2420
2422
2423 if (!ALI) {
2424 LLVM_DEBUG(
2425 dbgs() << "Inline asm lowering is not supported for this target yet\n");
2426 return false;
2427 }
2428
2429 return ALI->lowerInlineAsm(
2430 MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });
2431}
2432
2433bool IRTranslator::translateCallBase(const CallBase &CB,
2434 MachineIRBuilder &MIRBuilder) {
2435 ArrayRef<Register> Res = getOrCreateVRegs(CB);
2436
2438 Register SwiftInVReg = 0;
2439 Register SwiftErrorVReg = 0;
2440 for (const auto &Arg : CB.args()) {
2441 if (CLI->supportSwiftError() && isSwiftError(Arg)) {
2442 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2443 LLT Ty = getLLTForType(*Arg->getType(), *DL);
2444 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2445 MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
2446 &CB, &MIRBuilder.getMBB(), Arg));
2447 Args.emplace_back(ArrayRef(SwiftInVReg));
2448 SwiftErrorVReg =
2449 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2450 continue;
2451 }
2452 Args.push_back(getOrCreateVRegs(*Arg));
2453 }
2454
2455 if (auto *CI = dyn_cast<CallInst>(&CB)) {
2456 if (ORE->enabled()) {
2457 if (MemoryOpRemark::canHandle(CI, *LibInfo)) {
2458 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2459 R.visit(CI);
2460 }
2461 }
2462 }
2463
2464 // We don't set HasCalls on MFI here yet because call lowering may decide to
2465 // optimize into tail calls. Instead, we defer that to selection where a final
2466 // scan is done to check if any instructions are calls.
2467 bool Success =
2468 CLI->lowerCall(MIRBuilder, CB, Res, Args, SwiftErrorVReg,
2469 [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
2470
2471 // Check if we just inserted a tail call.
2472 if (Success) {
2473 assert(!HasTailCall && "Can't tail call return twice from block?");
2475 HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
2476 }
2477
2478 return Success;
2479}
2480
2481bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2482 const CallInst &CI = cast<CallInst>(U);
2483 auto TII = MF->getTarget().getIntrinsicInfo();
2484 const Function *F = CI.getCalledFunction();
2485
2486 // FIXME: support Windows dllimport function calls and calls through
2487 // weak symbols.
2488 if (F && (F->hasDLLImportStorageClass() ||
2490 F->hasExternalWeakLinkage())))
2491 return false;
2492
2493 // FIXME: support control flow guard targets.
2495 return false;
2496
2497 // FIXME: support statepoints and related.
2498 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
2499 return false;
2500
2501 if (CI.isInlineAsm())
2502 return translateInlineAsm(CI, MIRBuilder);
2503
2504 diagnoseDontCall(CI);
2505
2507 if (F && F->isIntrinsic()) {
2508 ID = F->getIntrinsicID();
2510 ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2511 }
2512
2513 if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2514 return translateCallBase(CI, MIRBuilder);
2515
2516 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2517
2518 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2519 return true;
2520
2521 ArrayRef<Register> ResultRegs;
2522 if (!CI.getType()->isVoidTy())
2523 ResultRegs = getOrCreateVRegs(CI);
2524
2525 // Ignore the callsite attributes. Backend code is most likely not expecting
2526 // an intrinsic to sometimes have side effects and sometimes not.
2527 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);
2528 if (isa<FPMathOperator>(CI))
2529 MIB->copyIRFlags(CI);
2530
2531 for (const auto &Arg : enumerate(CI.args())) {
2532 // If this is required to be an immediate, don't materialize it in a
2533 // register.
2534 if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2535 if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
2536 // imm arguments are more convenient than cimm (and realistically
2537 // probably sufficient), so use them.
2538 assert(CI->getBitWidth() <= 64 &&
2539 "large intrinsic immediates not handled");
2540 MIB.addImm(CI->getSExtValue());
2541 } else {
2542 MIB.addFPImm(cast<ConstantFP>(Arg.value()));
2543 }
2544 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {
2545 auto *MD = MDVal->getMetadata();
2546 auto *MDN = dyn_cast<MDNode>(MD);
2547 if (!MDN) {
2548 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))
2549 MDN = MDNode::get(MF->getFunction().getContext(), ConstMD);
2550 else // This was probably an MDString.
2551 return false;
2552 }
2553 MIB.addMetadata(MDN);
2554 } else {
2555 ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
2556 if (VRegs.size() > 1)
2557 return false;
2558 MIB.addUse(VRegs[0]);
2559 }
2560 }
2561
2562 // Add a MachineMemOperand if it is a target mem intrinsic.
2563 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
2565 // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2566 if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2567 Align Alignment = Info.align.value_or(
2568 DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
2569 LLT MemTy = Info.memVT.isSimple()
2570 ? getLLTForMVT(Info.memVT.getSimpleVT())
2571 : LLT::scalar(Info.memVT.getStoreSizeInBits());
2572
2573 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
2574 // didn't yield anything useful.
2576 if (Info.ptrVal)
2577 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2578 else if (Info.fallbackAddressSpace)
2579 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2580 MIB.addMemOperand(
2581 MF->getMachineMemOperand(MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata()));
2582 }
2583
2584 return true;
2585}
2586
2587bool IRTranslator::findUnwindDestinations(
2588 const BasicBlock *EHPadBB,
2589 BranchProbability Prob,
2590 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2591 &UnwindDests) {
2593 EHPadBB->getParent()->getFunction().getPersonalityFn());
2594 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2595 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2596 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2597 bool IsSEH = isAsynchronousEHPersonality(Personality);
2598
2599 if (IsWasmCXX) {
2600 // Ignore this for now.
2601 return false;
2602 }
2603
2604 while (EHPadBB) {
2605 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2606 BasicBlock *NewEHPadBB = nullptr;
2607 if (isa<LandingPadInst>(Pad)) {
2608 // Stop on landingpads. They are not funclets.
2609 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2610 break;
2611 }
2612 if (isa<CleanupPadInst>(Pad)) {
2613 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2614 // personalities.
2615 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2616 UnwindDests.back().first->setIsEHScopeEntry();
2617 UnwindDests.back().first->setIsEHFuncletEntry();
2618 break;
2619 }
2620 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2621 // Add the catchpad handlers to the possible destinations.
2622 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2623 UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2624 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2625 if (IsMSVCCXX || IsCoreCLR)
2626 UnwindDests.back().first->setIsEHFuncletEntry();
2627 if (!IsSEH)
2628 UnwindDests.back().first->setIsEHScopeEntry();
2629 }
2630 NewEHPadBB = CatchSwitch->getUnwindDest();
2631 } else {
2632 continue;
2633 }
2634
2635 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2636 if (BPI && NewEHPadBB)
2637 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2638 EHPadBB = NewEHPadBB;
2639 }
2640 return true;
2641}
2642
2643bool IRTranslator::translateInvoke(const User &U,
2644 MachineIRBuilder &MIRBuilder) {
2645 const InvokeInst &I = cast<InvokeInst>(U);
2646 MCContext &Context = MF->getContext();
2647
2648 const BasicBlock *ReturnBB = I.getSuccessor(0);
2649 const BasicBlock *EHPadBB = I.getSuccessor(1);
2650
2651 const Function *Fn = I.getCalledFunction();
2652
2653 // FIXME: support invoking patchpoint and statepoint intrinsics.
2654 if (Fn && Fn->isIntrinsic())
2655 return false;
2656
2657 // FIXME: support whatever these are.
2658 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
2659 return false;
2660
2661 // FIXME: support control flow guard targets.
2662 if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
2663 return false;
2664
2665 // FIXME: support Windows exception handling.
2666 if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHI()))
2667 return false;
2668
2669 // FIXME: support Windows dllimport function calls and calls through
2670 // weak symbols.
2671 if (Fn && (Fn->hasDLLImportStorageClass() ||
2673 Fn->hasExternalWeakLinkage())))
2674 return false;
2675
2676 bool LowerInlineAsm = I.isInlineAsm();
2677 bool NeedEHLabel = true;
2678
2679 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2680 // the region covered by the try.
2681 MCSymbol *BeginSymbol = nullptr;
2682 if (NeedEHLabel) {
2683 MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START);
2684 BeginSymbol = Context.createTempSymbol();
2685 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
2686 }
2687
2688 if (LowerInlineAsm) {
2689 if (!translateInlineAsm(I, MIRBuilder))
2690 return false;
2691 } else if (!translateCallBase(I, MIRBuilder))
2692 return false;
2693
2694 MCSymbol *EndSymbol = nullptr;
2695 if (NeedEHLabel) {
2696 EndSymbol = Context.createTempSymbol();
2697 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
2698 }
2699
2701 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2702 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2703 BranchProbability EHPadBBProb =
2704 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2706
2707 if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2708 return false;
2709
2710 MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
2711 &ReturnMBB = getMBB(*ReturnBB);
2712 // Update successor info.
2713 addSuccessorWithProb(InvokeMBB, &ReturnMBB);
2714 for (auto &UnwindDest : UnwindDests) {
2715 UnwindDest.first->setIsEHPad();
2716 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2717 }
2718 InvokeMBB->normalizeSuccProbs();
2719
2720 if (NeedEHLabel) {
2721 assert(BeginSymbol && "Expected a begin symbol!");
2722 assert(EndSymbol && "Expected an end symbol!");
2723 MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
2724 }
2725
2726 MIRBuilder.buildBr(ReturnMBB);
2727 return true;
2728}
2729
2730bool IRTranslator::translateCallBr(const User &U,
2731 MachineIRBuilder &MIRBuilder) {
2732 // FIXME: Implement this.
2733 return false;
2734}
2735
2736bool IRTranslator::translateLandingPad(const User &U,
2737 MachineIRBuilder &MIRBuilder) {
2738 const LandingPadInst &LP = cast<LandingPadInst>(U);
2739
2740 MachineBasicBlock &MBB = MIRBuilder.getMBB();
2741
2742 MBB.setIsEHPad();
2743
2744 // If there aren't registers to copy the values into (e.g., during SjLj
2745 // exceptions), then don't bother.
2746 auto &TLI = *MF->getSubtarget().getTargetLowering();
2747 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
2748 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2749 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2750 return true;
2751
2752 // If landingpad's return type is token type, we don't create DAG nodes
2753 // for its exception pointer and selector value. The extraction of exception
2754 // pointer or selector value from token type landingpads is not currently
2755 // supported.
2756 if (LP.getType()->isTokenTy())
2757 return true;
2758
2759 // Add a label to mark the beginning of the landing pad. Deletion of the
2760 // landing pad can thus be detected via the MachineModuleInfo.
2761 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
2762 .addSym(MF->addLandingPad(&MBB));
2763
2764 // If the unwinder does not preserve all registers, ensure that the
2765 // function marks the clobbered registers as used.
2767 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
2769
2770 LLT Ty = getLLTForType(*LP.getType(), *DL);
2772 MIRBuilder.buildUndef(Undef);
2773
2775 for (Type *Ty : cast<StructType>(LP.getType())->elements())
2776 Tys.push_back(getLLTForType(*Ty, *DL));
2777 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
2778
2779 // Mark exception register as live in.
2780 Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
2781 if (!ExceptionReg)
2782 return false;
2783
2784 MBB.addLiveIn(ExceptionReg);
2785 ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
2786 MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
2787
2788 Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
2789 if (!SelectorReg)
2790 return false;
2791
2792 MBB.addLiveIn(SelectorReg);
2793 Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
2794 MIRBuilder.buildCopy(PtrVReg, SelectorReg);
2795 MIRBuilder.buildCast(ResRegs[1], PtrVReg);
2796
2797 return true;
2798}
2799
2800bool IRTranslator::translateAlloca(const User &U,
2801 MachineIRBuilder &MIRBuilder) {
2802 auto &AI = cast<AllocaInst>(U);
2803
2804 if (AI.isSwiftError())
2805 return true;
2806
2807 if (AI.isStaticAlloca()) {
2808 Register Res = getOrCreateVReg(AI);
2809 int FI = getOrCreateFrameIndex(AI);
2810 MIRBuilder.buildFrameIndex(Res, FI);
2811 return true;
2812 }
2813
2814 // FIXME: support stack probing for Windows.
2816 return false;
2817
2818 // Now we're in the harder dynamic case.
2819 Register NumElts = getOrCreateVReg(*AI.getArraySize());
2820 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
2821 LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
2822 if (MRI->getType(NumElts) != IntPtrTy) {
2823 Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
2824 MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
2825 NumElts = ExtElts;
2826 }
2827
2828 Type *Ty = AI.getAllocatedType();
2829
2830 Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
2831 Register TySize =
2832 getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
2833 MIRBuilder.buildMul(AllocSize, NumElts, TySize);
2834
2835 // Round the size of the allocation up to the stack alignment size
2836 // by add SA-1 to the size. This doesn't overflow because we're computing
2837 // an address inside an alloca.
2839 auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
2840 auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
2842 auto AlignCst =
2843 MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
2844 auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
2845
2846 Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
2847 if (Alignment <= StackAlign)
2848 Alignment = Align(1);
2849 MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
2850
2851 MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
2853 return true;
2854}
2855
2856bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
2857 // FIXME: We may need more info about the type. Because of how LLT works,
2858 // we're completely discarding the i64/double distinction here (amongst
2859 // others). Fortunately the ABIs I know of where that matters don't use va_arg
2860 // anyway but that's not guaranteed.
2861 MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
2862 {getOrCreateVReg(*U.getOperand(0)),
2863 DL->getABITypeAlign(U.getType()).value()});
2864 return true;
2865}
2866
2867bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
2869 return true;
2870
2871 auto &UI = cast<UnreachableInst>(U);
2872 // We may be able to ignore unreachable behind a noreturn call.
2874 const BasicBlock &BB = *UI.getParent();
2875 if (&UI != &BB.front()) {
2877 std::prev(BasicBlock::const_iterator(UI));
2878 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2879 if (Call->doesNotReturn())
2880 return true;
2881 }
2882 }
2883 }
2884
2885 MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>());
2886 return true;
2887}
2888
2889bool IRTranslator::translateInsertElement(const User &U,
2890 MachineIRBuilder &MIRBuilder) {
2891 // If it is a <1 x Ty> vector, use the scalar as it is
2892 // not a legal vector type in LLT.
2893 if (cast<FixedVectorType>(U.getType())->getNumElements() == 1)
2894 return translateCopy(U, *U.getOperand(1), MIRBuilder);
2895
2896 Register Res = getOrCreateVReg(U);
2897 Register Val = getOrCreateVReg(*U.getOperand(0));
2898 Register Elt = getOrCreateVReg(*U.getOperand(1));
2899 Register Idx = getOrCreateVReg(*U.getOperand(2));
2900 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
2901 return true;
2902}
2903
2904bool IRTranslator::translateExtractElement(const User &U,
2905 MachineIRBuilder &MIRBuilder) {
2906 // If it is a <1 x Ty> vector, use the scalar as it is
2907 // not a legal vector type in LLT.
2908 if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
2909 return translateCopy(U, *U.getOperand(0), MIRBuilder);
2910
2911 Register Res = getOrCreateVReg(U);
2912 Register Val = getOrCreateVReg(*U.getOperand(0));
2913 const auto &TLI = *MF->getSubtarget().getTargetLowering();
2914 unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
2915 Register Idx;
2916 if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
2917 if (CI->getBitWidth() != PreferredVecIdxWidth) {
2918 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
2919 auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
2920 Idx = getOrCreateVReg(*NewIdxCI);
2921 }
2922 }
2923 if (!Idx)
2924 Idx = getOrCreateVReg(*U.getOperand(1));
2925 if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
2926 const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
2927 Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
2928 }
2929 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
2930 return true;
2931}
2932
2933bool IRTranslator::translateShuffleVector(const User &U,
2934 MachineIRBuilder &MIRBuilder) {
2936 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
2937 Mask = SVI->getShuffleMask();
2938 else
2939 Mask = cast<ConstantExpr>(U).getShuffleMask();
2940 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
2941 MIRBuilder
2942 .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
2943 {getOrCreateVReg(*U.getOperand(0)),
2944 getOrCreateVReg(*U.getOperand(1))})
2945 .addShuffleMask(MaskAlloc);
2946 return true;
2947}
2948
2949bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
2950 const PHINode &PI = cast<PHINode>(U);
2951
2953 for (auto Reg : getOrCreateVRegs(PI)) {
2954 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
2955 Insts.push_back(MIB.getInstr());
2956 }
2957
2958 PendingPHIs.emplace_back(&PI, std::move(Insts));
2959 return true;
2960}
2961
2962bool IRTranslator::translateAtomicCmpXchg(const User &U,
2963 MachineIRBuilder &MIRBuilder) {
2964 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
2965
2966 auto &TLI = *MF->getSubtarget().getTargetLowering();
2967 auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
2968
2969 auto Res = getOrCreateVRegs(I);
2970 Register OldValRes = Res[0];
2971 Register SuccessRes = Res[1];
2972 Register Addr = getOrCreateVReg(*I.getPointerOperand());
2973 Register Cmp = getOrCreateVReg(*I.getCompareOperand());
2974 Register NewVal = getOrCreateVReg(*I.getNewValOperand());
2975
2977 OldValRes, SuccessRes, Addr, Cmp, NewVal,
2979 MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
2980 getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),
2981 I.getSuccessOrdering(), I.getFailureOrdering()));
2982 return true;
2983}
2984
2985bool IRTranslator::translateAtomicRMW(const User &U,
2986 MachineIRBuilder &MIRBuilder) {
2987 const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
2988 auto &TLI = *MF->getSubtarget().getTargetLowering();
2989 auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
2990
2991 Register Res = getOrCreateVReg(I);
2992 Register Addr = getOrCreateVReg(*I.getPointerOperand());
2993 Register Val = getOrCreateVReg(*I.getValOperand());
2994
2995 unsigned Opcode = 0;
2996 switch (I.getOperation()) {
2997 default:
2998 return false;
3000 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3001 break;
3002 case AtomicRMWInst::Add:
3003 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3004 break;
3005 case AtomicRMWInst::Sub:
3006 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3007 break;
3008 case AtomicRMWInst::And:
3009 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3010 break;
3012 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3013 break;
3014 case AtomicRMWInst::Or:
3015 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3016 break;
3017 case AtomicRMWInst::Xor:
3018 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3019 break;
3020 case AtomicRMWInst::Max:
3021 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3022 break;
3023 case AtomicRMWInst::Min:
3024 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3025 break;
3027 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3028 break;
3030 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3031 break;
3033 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3034 break;
3036 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3037 break;
3039 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3040 break;
3042 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3043 break;
3045 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3046 break;
3048 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3049 break;
3050 }
3051
3052 MIRBuilder.buildAtomicRMW(
3053 Opcode, Res, Addr, Val,
3054 *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3055 Flags, MRI->getType(Val), getMemOpAlign(I),
3056 I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3057 I.getOrdering()));
3058 return true;
3059}
3060
3061bool IRTranslator::translateFence(const User &U,
3062 MachineIRBuilder &MIRBuilder) {
3063 const FenceInst &Fence = cast<FenceInst>(U);
3064 MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
3065 Fence.getSyncScopeID());
3066 return true;
3067}
3068
3069bool IRTranslator::translateFreeze(const User &U,
3070 MachineIRBuilder &MIRBuilder) {
3071 const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);
3072 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));
3073
3074 assert(DstRegs.size() == SrcRegs.size() &&
3075 "Freeze with different source and destination type?");
3076
3077 for (unsigned I = 0; I < DstRegs.size(); ++I) {
3078 MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);
3079 }
3080
3081 return true;
3082}
3083
3084void IRTranslator::finishPendingPhis() {
3085#ifndef NDEBUG
3086 DILocationVerifier Verifier;
3087 GISelObserverWrapper WrapperObserver(&Verifier);
3088 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3089#endif // ifndef NDEBUG
3090 for (auto &Phi : PendingPHIs) {
3091 const PHINode *PI = Phi.first;
3092 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3093 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3094 EntryBuilder->setDebugLoc(PI->getDebugLoc());
3095#ifndef NDEBUG
3096 Verifier.setCurrentInst(PI);
3097#endif // ifndef NDEBUG
3098
3100 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3101 auto IRPred = PI->getIncomingBlock(i);
3102 ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
3103 for (auto *Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
3104 if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
3105 continue;
3106 SeenPreds.insert(Pred);
3107 for (unsigned j = 0; j < ValRegs.size(); ++j) {
3108 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3109 MIB.addUse(ValRegs[j]);
3110 MIB.addMBB(Pred);
3111 }
3112 }
3113 }
3114 }
3115}
3116
3117bool IRTranslator::translate(const Instruction &Inst) {
3118 CurBuilder->setDebugLoc(Inst.getDebugLoc());
3119 CurBuilder->setPCSections(Inst.getMetadata(LLVMContext::MD_pcsections));
3120
3121 auto &TLI = *MF->getSubtarget().getTargetLowering();
3122 if (TLI.fallBackToDAGISel(Inst))
3123 return false;
3124
3125 switch (Inst.getOpcode()) {
3126#define HANDLE_INST(NUM, OPCODE, CLASS) \
3127 case Instruction::OPCODE: \
3128 return translate##OPCODE(Inst, *CurBuilder.get());
3129#include "llvm/IR/Instruction.def"
3130 default:
3131 return false;
3132 }
3133}
3134
3135bool IRTranslator::translate(const Constant &C, Register Reg) {
3136 // We only emit constants into the entry block from here. To prevent jumpy
3137 // debug behaviour remove debug line.
3138 if (auto CurrInstDL = CurBuilder->getDL())
3139 EntryBuilder->setDebugLoc(DebugLoc());
3140
3141 if (auto CI = dyn_cast<ConstantInt>(&C))
3142 EntryBuilder->buildConstant(Reg, *CI);
3143 else if (auto CF = dyn_cast<ConstantFP>(&C))
3144 EntryBuilder->buildFConstant(Reg, *CF);
3145 else if (isa<UndefValue>(C))
3146 EntryBuilder->buildUndef(Reg);
3147 else if (isa<ConstantPointerNull>(C))
3148 EntryBuilder->buildConstant(Reg, 0);
3149 else if (auto GV = dyn_cast<GlobalValue>(&C))
3150 EntryBuilder->buildGlobalValue(Reg, GV);
3151 else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
3152 if (!isa<FixedVectorType>(CAZ->getType()))
3153 return false;
3154 // Return the scalar if it is a <1 x Ty> vector.
3155 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3156 if (NumElts == 1)
3157 return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder);
3159 for (unsigned I = 0; I < NumElts; ++I) {
3160 Constant &Elt = *CAZ->getElementValue(I);
3161 Ops.push_back(getOrCreateVReg(Elt));
3162 }
3163 EntryBuilder->buildBuildVector(Reg, Ops);
3164 } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
3165 // Return the scalar if it is a <1 x Ty> vector.
3166 if (CV->getNumElements() == 1)
3167 return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder);
3169 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3170 Constant &Elt = *CV->getElementAsConstant(i);
3171 Ops.push_back(getOrCreateVReg(Elt));
3172 }
3173 EntryBuilder->buildBuildVector(Reg, Ops);
3174 } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
3175 switch(CE->getOpcode()) {
3176#define HANDLE_INST(NUM, OPCODE, CLASS) \
3177 case Instruction::OPCODE: \
3178 return translate##OPCODE(*CE, *EntryBuilder.get());
3179#include "llvm/IR/Instruction.def"
3180 default:
3181 return false;
3182 }
3183 } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
3184 if (CV->getNumOperands() == 1)
3185 return translateCopy(C, *CV->getOperand(0), *EntryBuilder);
3187 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3188 Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
3189 }
3190 EntryBuilder->buildBuildVector(Reg, Ops);
3191 } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
3192 EntryBuilder->buildBlockAddress(Reg, BA);
3193 } else
3194 return false;
3195
3196 return true;
3197}
3198
3199bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3201 for (auto &BTB : SL->BitTestCases) {
3202 // Emit header first, if it wasn't already emitted.
3203 if (!BTB.Emitted)
3204 emitBitTestHeader(BTB, BTB.Parent);
3205
3206 BranchProbability UnhandledProb = BTB.Prob;
3207 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3208 UnhandledProb -= BTB.Cases[j].ExtraProb;
3209 // Set the current basic block to the mbb we wish to insert the code into
3210 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3211 // If all cases cover a contiguous range, it is not necessary to jump to
3212 // the default block after the last bit test fails. This is because the
3213 // range check during bit test header creation has guaranteed that every
3214 // case here doesn't go outside the range. In this case, there is no need
3215 // to perform the last bit test, as it will always be true. Instead, make
3216 // the second-to-last bit-test fall through to the target of the last bit
3217 // test, and delete the last bit test.
3218
3219 MachineBasicBlock *NextMBB;
3220 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3221 // Second-to-last bit-test with contiguous range: fall through to the
3222 // target of the final bit test.
3223 NextMBB = BTB.Cases[j + 1].TargetBB;
3224 } else if (j + 1 == ej) {
3225 // For the last bit test, fall through to Default.
3226 NextMBB = BTB.Default;
3227 } else {
3228 // Otherwise, fall through to the next bit test.
3229 NextMBB = BTB.Cases[j + 1].ThisBB;
3230 }
3231
3232 emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
3233
3234 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3235 // We need to record the replacement phi edge here that normally
3236 // happens in emitBitTestCase before we delete the case, otherwise the
3237 // phi edge will be lost.
3238 addMachineCFGPred({BTB.Parent->getBasicBlock(),
3239 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3240 MBB);
3241 // Since we're not going to use the final bit test, remove it.
3242 BTB.Cases.pop_back();
3243 break;
3244 }
3245 }
3246 // This is "default" BB. We have two jumps to it. From "header" BB and from
3247 // last "case" BB, unless the latter was skipped.
3248 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3249 BTB.Default->getBasicBlock()};
3250 addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
3251 if (!BTB.ContiguousRange) {
3252 addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
3253 }
3254 }
3255 SL->BitTestCases.clear();
3256
3257 for (auto &JTCase : SL->JTCases) {
3258 // Emit header first, if it wasn't already emitted.
3259 if (!JTCase.first.Emitted)
3260 emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
3261
3262 emitJumpTable(JTCase.second, JTCase.second.MBB);
3263 }
3264 SL->JTCases.clear();
3265
3266 for (auto &SwCase : SL->SwitchCases)
3267 emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
3268 SL->SwitchCases.clear();
3269
3270 // Check if we need to generate stack-protector guard checks.
3271 StackProtector &SP = getAnalysis<StackProtector>();
3272 if (SP.shouldEmitSDCheck(BB)) {
3273 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3274 bool FunctionBasedInstrumentation =
3276 SPDescriptor.initialize(&BB, &MBB, FunctionBasedInstrumentation);
3277 }
3278 // Handle stack protector.
3279 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3280 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3281 return false;
3282 } else if (SPDescriptor.shouldEmitStackProtector()) {
3283 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3284 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3285
3286 // Find the split point to split the parent mbb. At the same time copy all
3287 // physical registers used in the tail of parent mbb into virtual registers
3288 // before the split point and back into physical registers after the split
3289 // point. This prevents us needing to deal with Live-ins and many other
3290 // register allocation issues caused by us splitting the parent mbb. The
3291 // register allocator will clean up said virtual copies later on.
3293 ParentMBB, *MF->getSubtarget().getInstrInfo());
3294
3295 // Splice the terminator of ParentMBB into SuccessMBB.
3296 SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
3297 ParentMBB->end());
3298
3299 // Add compare/jump on neq/jump to the parent BB.
3300 if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))
3301 return false;
3302
3303 // CodeGen Failure MBB if we have not codegened it yet.
3304 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3305 if (FailureMBB->empty()) {
3306 if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))
3307 return false;
3308 }
3309
3310 // Clear the Per-BB State.
3311 SPDescriptor.resetPerBBState();
3312 }
3313 return true;
3314}
3315
3316bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3317 MachineBasicBlock *ParentBB) {
3318 CurBuilder->setInsertPt(*ParentBB, ParentBB->end());
3319 // First create the loads to the guard/stack slot for the comparison.
3320 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3321 Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
3322 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
3323 LLT PtrMemTy = getLLTForMVT(TLI.getPointerMemTy(*DL));
3324
3325 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3326 int FI = MFI.getStackProtectorIndex();
3327
3328 Register Guard;
3329 Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);
3330 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3331 Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
3332
3333 // Generate code to load the content of the guard slot.
3334 Register GuardVal =
3335 CurBuilder
3336 ->buildLoad(PtrMemTy, StackSlotPtr,
3339 .getReg(0);
3340
3341 if (TLI.useStackGuardXorFP()) {
3342 LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3343 return false;
3344 }
3345
3346 // Retrieve guard check function, nullptr if instrumentation is inlined.
3347 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3348 // This path is currently untestable on GlobalISel, since the only platform
3349 // that needs this seems to be Windows, and we fall back on that currently.
3350 // The code still lives here in case that changes.
3351 // Silence warning about unused variable until the code below that uses
3352 // 'GuardCheckFn' is enabled.
3353 (void)GuardCheckFn;
3354 return false;
3355#if 0
3356 // The target provides a guard check function to validate the guard value.
3357 // Generate a call to that function with the content of the guard slot as
3358 // argument.
3359 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3360 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3362 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3363 Flags.setInReg();
3364 CallLowering::ArgInfo GuardArgInfo(
3365 {GuardVal, FnTy->getParamType(0), {Flags}});
3366
3368 Info.OrigArgs.push_back(GuardArgInfo);
3369 Info.CallConv = GuardCheckFn->getCallingConv();
3370 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
3371 Info.OrigRet = {Register(), FnTy->getReturnType()};
3372 if (!CLI->lowerCall(MIRBuilder, Info)) {
3373 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
3374 return false;
3375 }
3376 return true;
3377#endif
3378 }
3379
3380 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3381 // Otherwise, emit a volatile load to retrieve the stack guard value.
3382 if (TLI.useLoadStackGuardNode()) {
3383 Guard =
3385 getStackGuard(Guard, *CurBuilder);
3386 } else {
3387 // TODO: test using android subtarget when we support @llvm.thread.pointer.
3388 const Value *IRGuard = TLI.getSDagStackGuard(M);
3389 Register GuardPtr = getOrCreateVReg(*IRGuard);
3390
3391 Guard = CurBuilder
3392 ->buildLoad(PtrMemTy, GuardPtr,
3396 .getReg(0);
3397 }
3398
3399 // Perform the comparison.
3400 auto Cmp =
3401 CurBuilder->buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Guard, GuardVal);
3402 // If the guard/stackslot do not equal, branch to failure MBB.
3403 CurBuilder->buildBrCond(Cmp, *SPD.getFailureMBB());
3404 // Otherwise branch to success MBB.
3405 CurBuilder->buildBr(*SPD.getSuccessMBB());
3406 return true;
3407}
3408
3409bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
3410 MachineBasicBlock *FailureBB) {
3411 CurBuilder->setInsertPt(*FailureBB, FailureBB->end());
3412 const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
3413
3414 const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
3415 const char *Name = TLI.getLibcallName(Libcall);
3416
3418 Info.CallConv = TLI.getLibcallCallingConv(Libcall);
3420 Info.OrigRet = {Register(), Type::getVoidTy(MF->getFunction().getContext()),
3421 0};
3422 if (!CLI->lowerCall(*CurBuilder, Info)) {
3423 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
3424 return false;
3425 }
3426
3427 // On PS4/PS5, the "return address" must still be within the calling
3428 // function, even if it's at the very end, so emit an explicit TRAP here.
3429 // WebAssembly needs an unreachable instruction after a non-returning call,
3430 // because the function return type can be different from __stack_chk_fail's
3431 // return type (void).
3432 const TargetMachine &TM = MF->getTarget();
3433 if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) {
3434 LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
3435 return false;
3436 }
3437 return true;
3438}
3439
3440void IRTranslator::finalizeFunction() {
3441 // Release the memory used by the different maps we
3442 // needed during the translation.
3443 PendingPHIs.clear();
3444 VMap.reset();
3445 FrameIndices.clear();
3446 MachinePreds.clear();
3447 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3448 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3449 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
3450 EntryBuilder.reset();
3451 CurBuilder.reset();
3452 FuncInfo.clear();
3453 SPDescriptor.resetPerFunctionState();
3454}
3455
3456/// Returns true if a BasicBlock \p BB within a variadic function contains a
3457/// variadic musttail call.
3458static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3459 if (!IsVarArg)
3460 return false;
3461
3462 // Walk the block backwards, because tail calls usually only appear at the end
3463 // of a block.
3464 return llvm::any_of(llvm::reverse(BB), [](const Instruction &I) {
3465 const auto *CI = dyn_cast<CallInst>(&I);
3466 return CI && CI->isMustTailCall();
3467 });
3468}
3469
3471 MF = &CurMF;
3472 const Function &F = MF->getFunction();
3474 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3475 // Set the CSEConfig and run the analysis.
3476 GISelCSEInfo *CSEInfo = nullptr;
3477 TPC = &getAnalysis<TargetPassConfig>();
3478 bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
3480 : TPC->isGISelCSEEnabled();
3481
3482 if (EnableCSE) {
3483 EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3484 CSEInfo = &Wrapper.get(TPC->getCSEConfig());
3485 EntryBuilder->setCSEInfo(CSEInfo);
3486 CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
3487 CurBuilder->setCSEInfo(CSEInfo);
3488 } else {
3489 EntryBuilder = std::make_unique<MachineIRBuilder>();
3490 CurBuilder = std::make_unique<MachineIRBuilder>();
3491 }
3492 CLI = MF->getSubtarget().getCallLowering();
3493 CurBuilder->setMF(*MF);
3494 EntryBuilder->setMF(*MF);
3495 MRI = &MF->getRegInfo();
3496 DL = &F.getParent()->getDataLayout();
3497 ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
3498 const TargetMachine &TM = MF->getTarget();
3499 TM.resetTargetOptions(F);
3500 EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
3501 FuncInfo.MF = MF;
3502 if (EnableOpts) {
3503 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3504 FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
3505 } else {
3506 AA = nullptr;
3507 FuncInfo.BPI = nullptr;
3508 }
3509
3510 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
3511 MF->getFunction());
3512 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
3513 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
3514
3515 const auto &TLI = *MF->getSubtarget().getTargetLowering();
3516
3517 SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
3518 SL->init(TLI, TM, *DL);
3519
3520
3521
3522 assert(PendingPHIs.empty() && "stale PHIs");
3523
3524 // Targets which want to use big endian can enable it using
3525 // enableBigEndian()
3526 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
3527 // Currently we don't properly handle big endian code.
3528 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3529 F.getSubprogram(), &F.getEntryBlock());
3530 R << "unable to translate in big endian mode";
3531 reportTranslationError(*MF, *TPC, *ORE, R);
3532 }
3533
3534 // Release the per-function state when we return, whether we succeeded or not.
3535 auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
3536
3537 // Setup a separate basic-block for the arguments and constants
3539 MF->push_back(EntryBB);
3540 EntryBuilder->setMBB(*EntryBB);
3541
3542 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
3543 SwiftError.setFunction(CurMF);
3544 SwiftError.createEntriesInEntryBlock(DbgLoc);
3545
3546 bool IsVarArg = F.isVarArg();
3547 bool HasMustTailInVarArgFn = false;
3548
3549 // Create all blocks, in IR order, to preserve the layout.
3550 for (const BasicBlock &BB: F) {
3551 auto *&MBB = BBToMBB[&BB];
3552
3553 MBB = MF->CreateMachineBasicBlock(&BB);
3554 MF->push_back(MBB);
3555
3556 if (BB.hasAddressTaken())
3557 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
3558
3559 if (!HasMustTailInVarArgFn)
3560 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
3561 }
3562
3563 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
3564
3565 // Make our arguments/constants entry block fallthrough to the IR entry block.
3566 EntryBB->addSuccessor(&getMBB(F.front()));
3567
3568 if (CLI->fallBackToDAGISel(*MF)) {
3569 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3570 F.getSubprogram(), &F.getEntryBlock());
3571 R << "unable to lower function: " << ore::NV("Prototype", F.getType());
3572 reportTranslationError(*MF, *TPC, *ORE, R);
3573 return false;
3574 }
3575
3576 // Lower the actual args into this basic block.
3577 SmallVector<ArrayRef<Register>, 8> VRegArgs;
3578 for (const Argument &Arg: F.args()) {
3579 if (DL->getTypeStoreSize(Arg.getType()).isZero())
3580 continue; // Don't handle zero sized types.
3581 ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
3582 VRegArgs.push_back(VRegs);
3583
3584 if (Arg.hasSwiftErrorAttr()) {
3585 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
3586 SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
3587 }
3588 }
3589
3590 if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) {
3591 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3592 F.getSubprogram(), &F.getEntryBlock());
3593 R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
3594 reportTranslationError(*MF, *TPC, *ORE, R);
3595 return false;
3596 }
3597
3598 // Need to visit defs before uses when translating instructions.
3599 GISelObserverWrapper WrapperObserver;
3600 if (EnableCSE && CSEInfo)
3601 WrapperObserver.addObserver(CSEInfo);
3602 {
3604#ifndef NDEBUG
3605 DILocationVerifier Verifier;
3606 WrapperObserver.addObserver(&Verifier);
3607#endif // ifndef NDEBUG
3608 RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
3609 RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
3610 for (const BasicBlock *BB : RPOT) {
3611 MachineBasicBlock &MBB = getMBB(*BB);
3612 // Set the insertion point of all the following translations to
3613 // the end of this basic block.
3614 CurBuilder->setMBB(MBB);
3615 HasTailCall = false;
3616 for (const Instruction &Inst : *BB) {
3617 // If we translated a tail call in the last step, then we know
3618 // everything after the call is either a return, or something that is
3619 // handled by the call itself. (E.g. a lifetime marker or assume
3620 // intrinsic.) In this case, we should stop translating the block and
3621 // move on.
3622 if (HasTailCall)
3623 break;
3624#ifndef NDEBUG
3625 Verifier.setCurrentInst(&Inst);
3626#endif // ifndef NDEBUG
3627 if (translate(Inst))
3628 continue;
3629
3630 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3631 Inst.getDebugLoc(), BB);
3632 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
3633
3634 if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
3635 std::string InstStrStorage;
3636 raw_string_ostream InstStr(InstStrStorage);
3637 InstStr << Inst;
3638
3639 R << ": '" << InstStr.str() << "'";
3640 }
3641
3642 reportTranslationError(*MF, *TPC, *ORE, R);
3643 return false;
3644 }
3645
3646 if (!finalizeBasicBlock(*BB, MBB)) {
3647 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
3648 BB->getTerminator()->getDebugLoc(), BB);
3649 R << "unable to translate basic block";
3650 reportTranslationError(*MF, *TPC, *ORE, R);
3651 return false;
3652 }
3653 }
3654#ifndef NDEBUG
3655 WrapperObserver.removeObserver(&Verifier);
3656#endif
3657 }
3658
3659 finishPendingPhis();
3660
3661 SwiftError.propagateVRegs();
3662
3663 // Merge the argument lowering and constants block with its single
3664 // successor, the LLVM-IR entry block. We want the basic block to
3665 // be maximal.
3666 assert(EntryBB->succ_size() == 1 &&
3667 "Custom BB used for lowering should have only one successor");
3668 // Get the successor of the current entry block.
3669 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
3670 assert(NewEntryBB.pred_size() == 1 &&
3671 "LLVM-IR entry block has a predecessor!?");
3672 // Move all the instruction from the current entry block to the
3673 // new entry block.
3674 NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
3675 EntryBB->end());
3676
3677 // Update the live-in information for the new entry block.
3678 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
3679 NewEntryBB.addLiveIn(LiveIn);
3680 NewEntryBB.sortUniqueLiveIns();
3681
3682 // Get rid of the now empty basic block.
3683 EntryBB->removeSuccessor(&NewEntryBB);
3684 MF->remove(EntryBB);
3685 MF->deleteMachineBasicBlock(EntryBB);
3686
3687 assert(&MF->front() == &NewEntryBB &&
3688 "New entry wasn't next in the list of basic block!");
3689
3690 // Initialize stack protector information.
3691 StackProtector &SP = getAnalysis<StackProtector>();
3693
3694 return false;
3695}
unsigned SubReg
#define Success
aarch64 promote const
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Provides analysis for continuously CSEing during GISel passes.
This file implements a version of MachineIRBuilder which CSEs insts within a MachineBasicBlock.
This file describes how to lower LLVM calls to machine code calls.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
std::string Name
uint64_t Size
This contains common code to allow clients to notify changes to machine instr.
const HexagonInstrInfo * TII
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB)
Returns true if a BasicBlock BB within a variadic function contains a variadic musttail call.
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
static unsigned getConstrainedOpcode(Intrinsic::ID ID)
IRTranslator LLVM IR MI
#define DEBUG_TYPE
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
static bool isValInBlock(const Value *V, const BasicBlock *BB)
static bool isSwiftError(const Value *V)
This file declares the IRTranslator pass.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file describes how to lower LLVM inline asm to machine code INLINEASM.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:81
Implement a low-level type suitable for MachineInstr level instruction selection.
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
This file contains the declarations for metadata subclasses.
uint64_t High
IntegerType * Int32Ty
LLVMContext & Context
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
verify safepoint Safepoint IR Verifier
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
Value * RHS
Value * LHS
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:76
an instruction to allocate memory on the stack
Definition: Instructions.h:58
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:150
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:125
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:100
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:118
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:96
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
An immutable pass that tracks lazily created AssumptionCache objects.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:513
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:718
@ Add
*p = old + v
Definition: Instructions.h:734
@ FAdd
*p = old + v
Definition: Instructions.h:755
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:748
@ Or
*p = old | v
Definition: Instructions.h:742
@ Sub
*p = old - v
Definition: Instructions.h:736
@ And
*p = old & v
Definition: Instructions.h:738
@ Xor
*p = old ^ v
Definition: Instructions.h:744
@ FSub
*p = old - v
Definition: Instructions.h:758
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:770
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:746
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:752
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:766
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:750
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:762
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:774
@ Nand
*p = ~(old & v)
Definition: Instructions.h:740
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
Definition: Attributes.h:826
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:318
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
Definition: BasicBlock.h:516
InstListType::const_iterator const_iterator
Definition: BasicBlock.h:88
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:216
const Instruction & front() const
Definition: BasicBlock.h:347
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:227
const Instruction & back() const
Definition: BasicBlock.h:349
Legacy analysis pass which computes BlockFrequencyInfo.
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Legacy analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1190
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1479
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1412
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1332
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2030
Value * getCalledOperand() const
Definition: InstrTypes.h:1405
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1357
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1338
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1348
unsigned arg_size() const
Definition: InstrTypes.h:1355
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1489
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
bool isMustTailCall() const
bool checkReturnTypeForCallConv(MachineFunction &MF) const
Toplevel function to check the return type based on the target calling convention.
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
Definition: CallLowering.h:542
virtual bool enableBigEndian() const
For targets which want to use big-endian can enable it with enableBigEndian() hook.
Definition: CallLowering.h:590
virtual bool supportSwiftError() const
Definition: CallLowering.h:445
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
Definition: CallLowering.h:510
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const
This hook must be implemented to lower the given call instruction, including argument and return valu...
Definition: CallLowering.h:554
virtual bool fallBackToDAGISel(const MachineFunction &MF) const
Definition: CallLowering.h:528
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:701
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:711
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:728
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:741
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:734
@ ICMP_EQ
equal
Definition: InstrTypes.h:732
@ ICMP_NE
not equal
Definition: InstrTypes.h:733
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:737
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:713
bool isFPPredicate() const
Definition: InstrTypes.h:818
bool isIntPredicate() const
Definition: InstrTypes.h:819
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:833
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:197
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:145
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:403
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:356
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
bool startsWithDeref() const
Return whether the first element a DW_OP_deref.
ArrayRef< uint64_t > getElements() const
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:410
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:238
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:718
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:876
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:863
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:903
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:504
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:672
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:472
Align getPointerABIAlignment(unsigned AS) const
Layout pointer alignment.
Definition: DataLayout.cpp:740
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:872
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
This represents the llvm.dbg.value instruction.
Value * getValue(unsigned OpIdx=0) const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
A debug info location.
Definition: DebugLoc.h:33
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
An instruction for ordering other memory operations.
Definition: Instructions.h:436
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:472
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:461
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:693
BranchProbabilityInfo * BPI
void clear()
clear - Clear out all the function-specific state.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:178
const BasicBlock & getEntryBlock() const
Definition: Function.h:747
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1727
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1845
const Function & getFunction() const
Definition: Function.h:136
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:211
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:320
The actual analysis pass wrapper.
Definition: CSEInfo.h:222
Simple wrapper that does the following.
Definition: CSEInfo.h:204
The CSE Analysis object.
Definition: CSEInfo.h:69
Abstract class that contains various methods for clients to notify about changes.
Simple wrapper observer that takes several observers, and calls each one for each event.
void removeObserver(GISelChangeObserver *O)
void addObserver(GISelChangeObserver *O)
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:562
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:524
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:274
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
bool isTailCall(const MachineInstr &MI) const override
This instruction compares its operands according to the predicate given to the constructor.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
IRTranslator(CodeGenOptLevel OptLevel=CodeGenOptLevel::None)
static char ID
Definition: IRTranslator.h:68
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Indirect Branch Instruction.
bool lowerInlineAsm(MachineIRBuilder &MIRBuilder, const CallBase &CB, std::function< ArrayRef< Register >(const Value &Val)> GetOrCreateVRegs) const
Lower the given inline asm call instruction GetOrCreateVRegs is a callback to materialize a register ...
This instruction inserts a struct field of array element value into an aggregate value.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:392
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:284
const BasicBlock * getParent() const
Definition: Instruction.h:90
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:302
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1596
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:195
bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:54
Invoke instruction.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
Definition: LowLevelType.h:196
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr bool isVector() const
Definition: LowLevelType.h:145
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelType.h:49
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:175
constexpr bool isPointer() const
Definition: LowLevelType.h:141
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelType.h:92
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:177
Value * getPointerOperand()
Definition: Instructions.h:264
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:229
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:239
static LocationSize precise(uint64_t Value)
Context object for machine code objects.
Definition: MCContext.h:76
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:214
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
Metadata node.
Definition: Metadata.h:950
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1416
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
unsigned pred_size() const
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
void setAddressTakenIRBlock(BasicBlock *BB)
Set this block to reflect that it corresponds to an IR-level basic block with a BlockAddress.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
std::vector< MachineBasicBlock * >::iterator succ_iterator
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
void setHasMustTailInVarArgFunc(bool B)
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
ArrayRef< int > allocateShuffleMask(ArrayRef< int > Mask)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
void push_back(MachineBasicBlock *MBB)
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
void deleteMachineBasicBlock(MachineBasicBlock *MBB)
DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineModuleInfo & getMMI() const
void remove(iterator MBBI)
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
Helper class to build MachineInstr.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPEXT Op.
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, const SrcOp &Src2, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FMA Op0, Op1, Op2.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr,...
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
MachineInstrBuilder buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, Align Alignment)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
MachineInstrBuilder buildFIDbgValue(int FI, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in th...
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
const DebugLoc & getDebugLoc()
Get the current instruction's debug location.
MachineInstrBuilder buildFFrexp(const DstOp &Fract, const DstOp &Exp, const SrcOp &Src, std::optional< unsigned > Flags=std::nullopt)
Build and insert Fract, Exp = G_FFREXP Src.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
Definition: MachineInstr.h:68
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
static uint32_t copyFlagsFromInstruction(const Instruction &I)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:553
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MCContext & getContext() const
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
Representation for a specific memory location.
Root of the metadata hierarchy.
Definition: Metadata.h:61
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
The optimization diagnostic interface.
Diagnostic information for missed-optimization remarks.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
A simple RAII based Delegate installer.
A simple RAII based Observer installer.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Definition: Register.h:110
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents the LLVM 'select' instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:366
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:451
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
void initialize(const BasicBlock *BB, MachineBasicBlock *MBB, bool FunctionBasedInstrumentation)
Initialize the stack protector descriptor structure for a new basic block.
MachineBasicBlock * getSuccessMBB()
void resetPerBBState()
Reset state that changes when we handle different basic blocks.
void resetPerFunctionState()
Reset state that only changes when we switch functions.
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitStackProtector() const
Returns true if all fields of the stack protector descriptor are initialized implying that we should/...
bool shouldEmitFunctionBasedCheckStackProtector() const
bool shouldEmitSDCheck(const BasicBlock &BB) const
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
An instruction for storing to memory.
Definition: Instructions.h:301
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:651
Class to represent struct types.
Definition: DerivedTypes.h:213
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Multiway switch.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool fallBackToDAGISel(const Instruction &Inst) const
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
const Triple & getTargetTriple() const
TargetOptions Options
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
Target-Independent Code Generator Pass Configuration Options.
virtual std::unique_ptr< CSEConfigBase > getCSEConfig() const
Returns the CSEConfig object to use for the current optimization level.
virtual bool isGISelCSEEnabled() const
Check whether continuous CSE should be enabled in GISel passes.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const InlineAsmLowering * getInlineAsmLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const CallLowering * getCallLowering() const
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetLowering * getTargetLowering() const
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:584
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:302
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:295
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static IntegerType * getInt32Ty(LLVMContext &C)
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:225
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Value * getOperand(unsigned i) const
Definition: User.h:169
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:688
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1069
constexpr bool isZero() const
Definition: TypeSize.h:151
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:289
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:660
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:81
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49