LLVM 20.0.0git
IRTranslator.cpp
Go to the documentation of this file.
1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the IRTranslator class.
10//===----------------------------------------------------------------------===//
11
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/ScopeExit.h"
16#include "llvm/ADT/SmallSet.h"
21#include "llvm/Analysis/Loads.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/CFG.h"
53#include "llvm/IR/Constant.h"
54#include "llvm/IR/Constants.h"
55#include "llvm/IR/DataLayout.h"
58#include "llvm/IR/Function.h"
60#include "llvm/IR/InlineAsm.h"
61#include "llvm/IR/InstrTypes.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/IntrinsicsAMDGPU.h"
66#include "llvm/IR/LLVMContext.h"
67#include "llvm/IR/Metadata.h"
69#include "llvm/IR/Statepoint.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/User.h"
72#include "llvm/IR/Value.h"
74#include "llvm/MC/MCContext.h"
75#include "llvm/Pass.h"
78#include "llvm/Support/Debug.h"
86#include <algorithm>
87#include <cassert>
88#include <cstdint>
89#include <iterator>
90#include <optional>
91#include <string>
92#include <utility>
93#include <vector>
94
95#define DEBUG_TYPE "irtranslator"
96
97using namespace llvm;
98
99static cl::opt<bool>
100 EnableCSEInIRTranslator("enable-cse-in-irtranslator",
101 cl::desc("Should enable CSE in irtranslator"),
102 cl::Optional, cl::init(false));
103char IRTranslator::ID = 0;
104
105INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
106 false, false)
114
119 MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
120
121 // Print the function name explicitly if we don't have a debug location (which
122 // makes the diagnostic less useful) or if we're going to emit a raw error.
123 if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
124 R << (" (in function: " + MF.getName() + ")").str();
125
126 if (TPC.isGlobalISelAbortEnabled())
127 report_fatal_error(Twine(R.getMsg()));
128 else
129 ORE.emit(R);
130}
131
133 : MachineFunctionPass(ID), OptLevel(optlevel) {}
134
135#ifndef NDEBUG
136namespace {
137/// Verify that every instruction created has the same DILocation as the
138/// instruction being translated.
139class DILocationVerifier : public GISelChangeObserver {
140 const Instruction *CurrInst = nullptr;
141
142public:
143 DILocationVerifier() = default;
144 ~DILocationVerifier() = default;
145
146 const Instruction *getCurrentInst() const { return CurrInst; }
147 void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
148
149 void erasingInstr(MachineInstr &MI) override {}
150 void changingInstr(MachineInstr &MI) override {}
151 void changedInstr(MachineInstr &MI) override {}
152
153 void createdInstr(MachineInstr &MI) override {
154 assert(getCurrentInst() && "Inserted instruction without a current MI");
155
156 // Only print the check message if we're actually checking it.
157#ifndef NDEBUG
158 LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
159 << " was copied to " << MI);
160#endif
161 // We allow insts in the entry block to have no debug loc because
162 // they could have originated from constants, and we don't want a jumpy
163 // debug experience.
164 assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
165 (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||
166 (MI.isDebugInstr())) &&
167 "Line info was not transferred to all instructions");
168 }
169};
170} // namespace
171#endif // ifndef NDEBUG
172
173
179 if (OptLevel != CodeGenOptLevel::None) {
182 }
187}
188
190IRTranslator::allocateVRegs(const Value &Val) {
191 auto VRegsIt = VMap.findVRegs(Val);
192 if (VRegsIt != VMap.vregs_end())
193 return *VRegsIt->second;
194 auto *Regs = VMap.getVRegs(Val);
195 auto *Offsets = VMap.getOffsets(Val);
196 SmallVector<LLT, 4> SplitTys;
197 computeValueLLTs(*DL, *Val.getType(), SplitTys,
198 Offsets->empty() ? Offsets : nullptr);
199 for (unsigned i = 0; i < SplitTys.size(); ++i)
200 Regs->push_back(0);
201 return *Regs;
202}
203
204ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
205 auto VRegsIt = VMap.findVRegs(Val);
206 if (VRegsIt != VMap.vregs_end())
207 return *VRegsIt->second;
208
209 if (Val.getType()->isVoidTy())
210 return *VMap.getVRegs(Val);
211
212 // Create entry for this type.
213 auto *VRegs = VMap.getVRegs(Val);
214 auto *Offsets = VMap.getOffsets(Val);
215
216 if (!Val.getType()->isTokenTy())
217 assert(Val.getType()->isSized() &&
218 "Don't know how to create an empty vreg");
219
220 SmallVector<LLT, 4> SplitTys;
221 computeValueLLTs(*DL, *Val.getType(), SplitTys,
222 Offsets->empty() ? Offsets : nullptr);
223
224 if (!isa<Constant>(Val)) {
225 for (auto Ty : SplitTys)
226 VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
227 return *VRegs;
228 }
229
230 if (Val.getType()->isAggregateType()) {
231 // UndefValue, ConstantAggregateZero
232 auto &C = cast<Constant>(Val);
233 unsigned Idx = 0;
234 while (auto Elt = C.getAggregateElement(Idx++)) {
235 auto EltRegs = getOrCreateVRegs(*Elt);
236 llvm::copy(EltRegs, std::back_inserter(*VRegs));
237 }
238 } else {
239 assert(SplitTys.size() == 1 && "unexpectedly split LLT");
240 VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
241 bool Success = translate(cast<Constant>(Val), VRegs->front());
242 if (!Success) {
243 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
245 &MF->getFunction().getEntryBlock());
246 R << "unable to translate constant: " << ore::NV("Type", Val.getType());
247 reportTranslationError(*MF, *TPC, *ORE, R);
248 return *VRegs;
249 }
250 }
251
252 return *VRegs;
253}
254
255int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
256 auto MapEntry = FrameIndices.find(&AI);
257 if (MapEntry != FrameIndices.end())
258 return MapEntry->second;
259
260 uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
261 uint64_t Size =
262 ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
263
264 // Always allocate at least one byte.
265 Size = std::max<uint64_t>(Size, 1u);
266
267 int &FI = FrameIndices[&AI];
268 FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
269 return FI;
270}
271
272Align IRTranslator::getMemOpAlign(const Instruction &I) {
273 if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
274 return SI->getAlign();
275 if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
276 return LI->getAlign();
277 if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
278 return AI->getAlign();
279 if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
280 return AI->getAlign();
281
282 OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
283 R << "unable to translate memop: " << ore::NV("Opcode", &I);
284 reportTranslationError(*MF, *TPC, *ORE, R);
285 return Align(1);
286}
287
288MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
289 MachineBasicBlock *MBB = FuncInfo.getMBB(&BB);
290 assert(MBB && "BasicBlock was not encountered before");
291 return *MBB;
292}
293
294void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
295 assert(NewPred && "new predecessor must be a real MachineBasicBlock");
296 MachinePreds[Edge].push_back(NewPred);
297}
298
299static bool containsBF16Type(const User &U) {
300 // BF16 cannot currently be represented by LLT, to avoid miscompiles we
301 // prevent any instructions using them. FIXME: This can be removed once LLT
302 // supports bfloat.
303 return U.getType()->getScalarType()->isBFloatTy() ||
304 any_of(U.operands(), [](Value *V) {
305 return V->getType()->getScalarType()->isBFloatTy();
306 });
307}
308
309bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
310 MachineIRBuilder &MIRBuilder) {
311 if (containsBF16Type(U))
312 return false;
313
314 // Get or create a virtual register for each value.
315 // Unless the value is a Constant => loadimm cst?
316 // or inline constant each time?
317 // Creation of a virtual register needs to have a size.
318 Register Op0 = getOrCreateVReg(*U.getOperand(0));
319 Register Op1 = getOrCreateVReg(*U.getOperand(1));
320 Register Res = getOrCreateVReg(U);
321 uint32_t Flags = 0;
322 if (isa<Instruction>(U)) {
323 const Instruction &I = cast<Instruction>(U);
325 }
326
327 MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
328 return true;
329}
330
331bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
332 MachineIRBuilder &MIRBuilder) {
333 if (containsBF16Type(U))
334 return false;
335
336 Register Op0 = getOrCreateVReg(*U.getOperand(0));
337 Register Res = getOrCreateVReg(U);
338 uint32_t Flags = 0;
339 if (isa<Instruction>(U)) {
340 const Instruction &I = cast<Instruction>(U);
342 }
343 MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);
344 return true;
345}
346
347bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
348 return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
349}
350
351bool IRTranslator::translateCompare(const User &U,
352 MachineIRBuilder &MIRBuilder) {
353 if (containsBF16Type(U))
354 return false;
355
356 auto *CI = cast<CmpInst>(&U);
357 Register Op0 = getOrCreateVReg(*U.getOperand(0));
358 Register Op1 = getOrCreateVReg(*U.getOperand(1));
359 Register Res = getOrCreateVReg(U);
360 CmpInst::Predicate Pred = CI->getPredicate();
362 if (CmpInst::isIntPredicate(Pred))
363 MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);
364 else if (Pred == CmpInst::FCMP_FALSE)
365 MIRBuilder.buildCopy(
366 Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
367 else if (Pred == CmpInst::FCMP_TRUE)
368 MIRBuilder.buildCopy(
369 Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
370 else
371 MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
372
373 return true;
374}
375
376bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
377 const ReturnInst &RI = cast<ReturnInst>(U);
378 const Value *Ret = RI.getReturnValue();
379 if (Ret && DL->getTypeStoreSize(Ret->getType()).isZero())
380 Ret = nullptr;
381
382 ArrayRef<Register> VRegs;
383 if (Ret)
384 VRegs = getOrCreateVRegs(*Ret);
385
386 Register SwiftErrorVReg = 0;
387 if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
388 SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
389 &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
390 }
391
392 // The target may mess up with the insertion point, but
393 // this is not important as a return is the last instruction
394 // of the block anyway.
395 return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
396}
397
398void IRTranslator::emitBranchForMergedCondition(
400 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
401 BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
402 // If the leaf of the tree is a comparison, merge the condition into
403 // the caseblock.
404 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
405 CmpInst::Predicate Condition;
406 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
407 Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
408 } else {
409 const FCmpInst *FC = cast<FCmpInst>(Cond);
410 Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
411 }
412
413 SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),
414 BOp->getOperand(1), nullptr, TBB, FBB, CurBB,
415 CurBuilder->getDebugLoc(), TProb, FProb);
416 SL->SwitchCases.push_back(CB);
417 return;
418 }
419
420 // Create a CaseBlock record representing this branch.
423 Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),
424 nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
425 SL->SwitchCases.push_back(CB);
426}
427
428static bool isValInBlock(const Value *V, const BasicBlock *BB) {
429 if (const Instruction *I = dyn_cast<Instruction>(V))
430 return I->getParent() == BB;
431 return true;
432}
433
434void IRTranslator::findMergedConditions(
436 MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
438 BranchProbability FProb, bool InvertCond) {
439 using namespace PatternMatch;
440 assert((Opc == Instruction::And || Opc == Instruction::Or) &&
441 "Expected Opc to be AND/OR");
442 // Skip over not part of the tree and remember to invert op and operands at
443 // next level.
444 Value *NotCond;
445 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
446 isValInBlock(NotCond, CurBB->getBasicBlock())) {
447 findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
448 !InvertCond);
449 return;
450 }
451
452 const Instruction *BOp = dyn_cast<Instruction>(Cond);
453 const Value *BOpOp0, *BOpOp1;
454 // Compute the effective opcode for Cond, taking into account whether it needs
455 // to be inverted, e.g.
456 // and (not (or A, B)), C
457 // gets lowered as
458 // and (and (not A, not B), C)
460 if (BOp) {
461 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
462 ? Instruction::And
463 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
464 ? Instruction::Or
466 if (InvertCond) {
467 if (BOpc == Instruction::And)
468 BOpc = Instruction::Or;
469 else if (BOpc == Instruction::Or)
470 BOpc = Instruction::And;
471 }
472 }
473
474 // If this node is not part of the or/and tree, emit it as a branch.
475 // Note that all nodes in the tree should have same opcode.
476 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
477 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
478 !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||
479 !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {
480 emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
481 InvertCond);
482 return;
483 }
484
485 // Create TmpBB after CurBB.
486 MachineFunction::iterator BBI(CurBB);
487 MachineBasicBlock *TmpBB =
489 CurBB->getParent()->insert(++BBI, TmpBB);
490
491 if (Opc == Instruction::Or) {
492 // Codegen X | Y as:
493 // BB1:
494 // jmp_if_X TBB
495 // jmp TmpBB
496 // TmpBB:
497 // jmp_if_Y TBB
498 // jmp FBB
499 //
500
501 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
502 // The requirement is that
503 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
504 // = TrueProb for original BB.
505 // Assuming the original probabilities are A and B, one choice is to set
506 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
507 // A/(1+B) and 2B/(1+B). This choice assumes that
508 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
509 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
510 // TmpBB, but the math is more complicated.
511
512 auto NewTrueProb = TProb / 2;
513 auto NewFalseProb = TProb / 2 + FProb;
514 // Emit the LHS condition.
515 findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
516 NewFalseProb, InvertCond);
517
518 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
519 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
520 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
521 // Emit the RHS condition into TmpBB.
522 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
523 Probs[1], InvertCond);
524 } else {
525 assert(Opc == Instruction::And && "Unknown merge op!");
526 // Codegen X & Y as:
527 // BB1:
528 // jmp_if_X TmpBB
529 // jmp FBB
530 // TmpBB:
531 // jmp_if_Y TBB
532 // jmp FBB
533 //
534 // This requires creation of TmpBB after CurBB.
535
536 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
537 // The requirement is that
538 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
539 // = FalseProb for original BB.
540 // Assuming the original probabilities are A and B, one choice is to set
541 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
542 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
543 // TrueProb for BB1 * FalseProb for TmpBB.
544
545 auto NewTrueProb = TProb + FProb / 2;
546 auto NewFalseProb = FProb / 2;
547 // Emit the LHS condition.
548 findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
549 NewFalseProb, InvertCond);
550
551 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
552 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
553 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
554 // Emit the RHS condition into TmpBB.
555 findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
556 Probs[1], InvertCond);
557 }
558}
559
560bool IRTranslator::shouldEmitAsBranches(
561 const std::vector<SwitchCG::CaseBlock> &Cases) {
562 // For multiple cases, it's better to emit as branches.
563 if (Cases.size() != 2)
564 return true;
565
566 // If this is two comparisons of the same values or'd or and'd together, they
567 // will get folded into a single comparison, so don't emit two blocks.
568 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
569 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
570 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
571 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
572 return false;
573 }
574
575 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
576 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
577 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
578 Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
579 isa<Constant>(Cases[0].CmpRHS) &&
580 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
581 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
582 Cases[0].TrueBB == Cases[1].ThisBB)
583 return false;
584 if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
585 Cases[0].FalseBB == Cases[1].ThisBB)
586 return false;
587 }
588
589 return true;
590}
591
592bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
593 const BranchInst &BrInst = cast<BranchInst>(U);
594 auto &CurMBB = MIRBuilder.getMBB();
595 auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));
596
597 if (BrInst.isUnconditional()) {
598 // If the unconditional target is the layout successor, fallthrough.
599 if (OptLevel == CodeGenOptLevel::None ||
600 !CurMBB.isLayoutSuccessor(Succ0MBB))
601 MIRBuilder.buildBr(*Succ0MBB);
602
603 // Link successors.
604 for (const BasicBlock *Succ : successors(&BrInst))
605 CurMBB.addSuccessor(&getMBB(*Succ));
606 return true;
607 }
608
609 // If this condition is one of the special cases we handle, do special stuff
610 // now.
611 const Value *CondVal = BrInst.getCondition();
612 MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));
613
614 // If this is a series of conditions that are or'd or and'd together, emit
615 // this as a sequence of branches instead of setcc's with and/or operations.
616 // As long as jumps are not expensive (exceptions for multi-use logic ops,
617 // unpredictable branches, and vector extracts because those jumps are likely
618 // expensive for any target), this should improve performance.
619 // For example, instead of something like:
620 // cmp A, B
621 // C = seteq
622 // cmp D, E
623 // F = setle
624 // or C, F
625 // jnz foo
626 // Emit:
627 // cmp A, B
628 // je foo
629 // cmp D, E
630 // jle foo
631 using namespace PatternMatch;
632 const Instruction *CondI = dyn_cast<Instruction>(CondVal);
633 if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&
634 !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {
636 Value *Vec;
637 const Value *BOp0, *BOp1;
638 if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
639 Opcode = Instruction::And;
640 else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
641 Opcode = Instruction::Or;
642
643 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
644 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
645 findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
646 getEdgeProbability(&CurMBB, Succ0MBB),
647 getEdgeProbability(&CurMBB, Succ1MBB),
648 /*InvertCond=*/false);
649 assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
650
651 // Allow some cases to be rejected.
652 if (shouldEmitAsBranches(SL->SwitchCases)) {
653 // Emit the branch for this block.
654 emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
655 SL->SwitchCases.erase(SL->SwitchCases.begin());
656 return true;
657 }
658
659 // Okay, we decided not to do this, remove any inserted MBB's and clear
660 // SwitchCases.
661 for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
662 MF->erase(SL->SwitchCases[I].ThisBB);
663
664 SL->SwitchCases.clear();
665 }
666 }
667
668 // Create a CaseBlock record representing this branch.
669 SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
671 nullptr, Succ0MBB, Succ1MBB, &CurMBB,
672 CurBuilder->getDebugLoc());
673
674 // Use emitSwitchCase to actually insert the fast branch sequence for this
675 // cond branch.
676 emitSwitchCase(CB, &CurMBB, *CurBuilder);
677 return true;
678}
679
680void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
682 BranchProbability Prob) {
683 if (!FuncInfo.BPI) {
684 Src->addSuccessorWithoutProb(Dst);
685 return;
686 }
687 if (Prob.isUnknown())
688 Prob = getEdgeProbability(Src, Dst);
689 Src->addSuccessor(Dst, Prob);
690}
691
693IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
694 const MachineBasicBlock *Dst) const {
695 const BasicBlock *SrcBB = Src->getBasicBlock();
696 const BasicBlock *DstBB = Dst->getBasicBlock();
697 if (!FuncInfo.BPI) {
698 // If BPI is not available, set the default probability as 1 / N, where N is
699 // the number of successors.
700 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
701 return BranchProbability(1, SuccSize);
702 }
703 return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
704}
705
706bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
707 using namespace SwitchCG;
708 // Extract cases from the switch.
709 const SwitchInst &SI = cast<SwitchInst>(U);
710 BranchProbabilityInfo *BPI = FuncInfo.BPI;
711 CaseClusterVector Clusters;
712 Clusters.reserve(SI.getNumCases());
713 for (const auto &I : SI.cases()) {
714 MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
715 assert(Succ && "Could not find successor mbb in mapping");
716 const ConstantInt *CaseVal = I.getCaseValue();
717 BranchProbability Prob =
718 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
719 : BranchProbability(1, SI.getNumCases() + 1);
720 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
721 }
722
723 MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
724
725 // Cluster adjacent cases with the same destination. We do this at all
726 // optimization levels because it's cheap to do and will make codegen faster
727 // if there are many clusters.
728 sortAndRangeify(Clusters);
729
730 MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
731
732 // If there is only the default destination, jump there directly.
733 if (Clusters.empty()) {
734 SwitchMBB->addSuccessor(DefaultMBB);
735 if (DefaultMBB != SwitchMBB->getNextNode())
736 MIB.buildBr(*DefaultMBB);
737 return true;
738 }
739
740 SL->findJumpTables(Clusters, &SI, std::nullopt, DefaultMBB, nullptr, nullptr);
741 SL->findBitTestClusters(Clusters, &SI);
742
743 LLVM_DEBUG({
744 dbgs() << "Case clusters: ";
745 for (const CaseCluster &C : Clusters) {
746 if (C.Kind == CC_JumpTable)
747 dbgs() << "JT:";
748 if (C.Kind == CC_BitTests)
749 dbgs() << "BT:";
750
751 C.Low->getValue().print(dbgs(), true);
752 if (C.Low != C.High) {
753 dbgs() << '-';
754 C.High->getValue().print(dbgs(), true);
755 }
756 dbgs() << ' ';
757 }
758 dbgs() << '\n';
759 });
760
761 assert(!Clusters.empty());
762 SwitchWorkList WorkList;
763 CaseClusterIt First = Clusters.begin();
764 CaseClusterIt Last = Clusters.end() - 1;
765 auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
766 WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
767
768 while (!WorkList.empty()) {
769 SwitchWorkListItem W = WorkList.pop_back_val();
770
771 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
772 // For optimized builds, lower large range as a balanced binary tree.
773 if (NumClusters > 3 &&
775 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
776 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB, MIB);
777 continue;
778 }
779
780 if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
781 return false;
782 }
783 return true;
784}
785
786void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,
788 Value *Cond, MachineBasicBlock *SwitchMBB,
789 MachineIRBuilder &MIB) {
790 using namespace SwitchCG;
791 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
792 "Clusters not sorted?");
793 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
794
795 auto [LastLeft, FirstRight, LeftProb, RightProb] =
796 SL->computeSplitWorkItemInfo(W);
797
798 // Use the first element on the right as pivot since we will make less-than
799 // comparisons against it.
800 CaseClusterIt PivotCluster = FirstRight;
801 assert(PivotCluster > W.FirstCluster);
802 assert(PivotCluster <= W.LastCluster);
803
804 CaseClusterIt FirstLeft = W.FirstCluster;
805 CaseClusterIt LastRight = W.LastCluster;
806
807 const ConstantInt *Pivot = PivotCluster->Low;
808
809 // New blocks will be inserted immediately after the current one.
811 ++BBI;
812
813 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
814 // we can branch to its destination directly if it's squeezed exactly in
815 // between the known lower bound and Pivot - 1.
816 MachineBasicBlock *LeftMBB;
817 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
818 FirstLeft->Low == W.GE &&
819 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
820 LeftMBB = FirstLeft->MBB;
821 } else {
822 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
823 FuncInfo.MF->insert(BBI, LeftMBB);
824 WorkList.push_back(
825 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
826 }
827
828 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
829 // single cluster, RHS.Low == Pivot, and we can branch to its destination
830 // directly if RHS.High equals the current upper bound.
831 MachineBasicBlock *RightMBB;
832 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&
833 (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
834 RightMBB = FirstRight->MBB;
835 } else {
836 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
837 FuncInfo.MF->insert(BBI, RightMBB);
838 WorkList.push_back(
839 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
840 }
841
842 // Create the CaseBlock record that will be used to lower the branch.
843 CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,
844 LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,
845 RightProb);
846
847 if (W.MBB == SwitchMBB)
848 emitSwitchCase(CB, SwitchMBB, MIB);
849 else
850 SL->SwitchCases.push_back(CB);
851}
852
853void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
855 // Emit the code for the jump table
856 assert(JT.Reg && "Should lower JT Header first!");
858 MIB.setMBB(*MBB);
859 MIB.setDebugLoc(CurBuilder->getDebugLoc());
860
862 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
863
864 auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
865 MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
866}
867
868bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
870 MachineBasicBlock *HeaderBB) {
871 MachineIRBuilder MIB(*HeaderBB->getParent());
872 MIB.setMBB(*HeaderBB);
873 MIB.setDebugLoc(CurBuilder->getDebugLoc());
874
875 const Value &SValue = *JTH.SValue;
876 // Subtract the lowest switch case value from the value being switched on.
877 const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
878 Register SwitchOpReg = getOrCreateVReg(SValue);
879 auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
880 auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
881
882 // This value may be smaller or larger than the target's pointer type, and
883 // therefore require extension or truncating.
884 auto *PtrIRTy = PointerType::getUnqual(SValue.getContext());
885 const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
886 Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
887
888 JT.Reg = Sub.getReg(0);
889
890 if (JTH.FallthroughUnreachable) {
891 if (JT.MBB != HeaderBB->getNextNode())
892 MIB.buildBr(*JT.MBB);
893 return true;
894 }
895
896 // Emit the range check for the jump table, and branch to the default block
897 // for the switch statement if the value being switched on exceeds the
898 // largest case in the switch.
899 auto Cst = getOrCreateVReg(
900 *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
901 Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
902 auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
903
904 auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
905
906 // Avoid emitting unnecessary branches to the next block.
907 if (JT.MBB != HeaderBB->getNextNode())
908 BrCond = MIB.buildBr(*JT.MBB);
909 return true;
910}
911
912void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
913 MachineBasicBlock *SwitchBB,
914 MachineIRBuilder &MIB) {
915 Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
917 DebugLoc OldDbgLoc = MIB.getDebugLoc();
918 MIB.setDebugLoc(CB.DbgLoc);
919 MIB.setMBB(*CB.ThisBB);
920
921 if (CB.PredInfo.NoCmp) {
922 // Branch or fall through to TrueBB.
923 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
924 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
925 CB.ThisBB);
927 if (CB.TrueBB != CB.ThisBB->getNextNode())
928 MIB.buildBr(*CB.TrueBB);
929 MIB.setDebugLoc(OldDbgLoc);
930 return;
931 }
932
933 const LLT i1Ty = LLT::scalar(1);
934 // Build the compare.
935 if (!CB.CmpMHS) {
936 const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);
937 // For conditional branch lowering, we might try to do something silly like
938 // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
939 // just re-use the existing condition vreg.
940 if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
942 Cond = CondLHS;
943 } else {
944 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
946 Cond =
947 MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
948 else
949 Cond =
950 MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
951 }
952 } else {
954 "Can only handle SLE ranges");
955
956 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
957 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
958
959 Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
960 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
961 Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
962 Cond =
963 MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
964 } else {
965 const LLT CmpTy = MRI->getType(CmpOpReg);
966 auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
967 auto Diff = MIB.buildConstant(CmpTy, High - Low);
968 Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
969 }
970 }
971
972 // Update successor info
973 addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
974
975 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
976 CB.ThisBB);
977
978 // TrueBB and FalseBB are always different unless the incoming IR is
979 // degenerate. This only happens when running llc on weird IR.
980 if (CB.TrueBB != CB.FalseBB)
981 addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
983
984 addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
985 CB.ThisBB);
986
987 MIB.buildBrCond(Cond, *CB.TrueBB);
988 MIB.buildBr(*CB.FalseBB);
989 MIB.setDebugLoc(OldDbgLoc);
990}
991
992bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
993 MachineBasicBlock *SwitchMBB,
994 MachineBasicBlock *CurMBB,
995 MachineBasicBlock *DefaultMBB,
996 MachineIRBuilder &MIB,
998 BranchProbability UnhandledProbs,
1000 MachineBasicBlock *Fallthrough,
1001 bool FallthroughUnreachable) {
1002 using namespace SwitchCG;
1003 MachineFunction *CurMF = SwitchMBB->getParent();
1004 // FIXME: Optimize away range check based on pivot comparisons.
1005 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
1006 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
1007 BranchProbability DefaultProb = W.DefaultProb;
1008
1009 // The jump block hasn't been inserted yet; insert it here.
1010 MachineBasicBlock *JumpMBB = JT->MBB;
1011 CurMF->insert(BBI, JumpMBB);
1012
1013 // Since the jump table block is separate from the switch block, we need
1014 // to keep track of it as a machine predecessor to the default block,
1015 // otherwise we lose the phi edges.
1016 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1017 CurMBB);
1018 addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
1019 JumpMBB);
1020
1021 auto JumpProb = I->Prob;
1022 auto FallthroughProb = UnhandledProbs;
1023
1024 // If the default statement is a target of the jump table, we evenly
1025 // distribute the default probability to successors of CurMBB. Also
1026 // update the probability on the edge from JumpMBB to Fallthrough.
1027 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
1028 SE = JumpMBB->succ_end();
1029 SI != SE; ++SI) {
1030 if (*SI == DefaultMBB) {
1031 JumpProb += DefaultProb / 2;
1032 FallthroughProb -= DefaultProb / 2;
1033 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
1034 JumpMBB->normalizeSuccProbs();
1035 } else {
1036 // Also record edges from the jump table block to it's successors.
1037 addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
1038 JumpMBB);
1039 }
1040 }
1041
1042 if (FallthroughUnreachable)
1043 JTH->FallthroughUnreachable = true;
1044
1045 if (!JTH->FallthroughUnreachable)
1046 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
1047 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
1048 CurMBB->normalizeSuccProbs();
1049
1050 // The jump table header will be inserted in our current block, do the
1051 // range check, and fall through to our fallthrough block.
1052 JTH->HeaderBB = CurMBB;
1053 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
1054
1055 // If we're in the right place, emit the jump table header right now.
1056 if (CurMBB == SwitchMBB) {
1057 if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
1058 return false;
1059 JTH->Emitted = true;
1060 }
1061 return true;
1062}
1063bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
1064 Value *Cond,
1065 MachineBasicBlock *Fallthrough,
1066 bool FallthroughUnreachable,
1067 BranchProbability UnhandledProbs,
1068 MachineBasicBlock *CurMBB,
1069 MachineIRBuilder &MIB,
1070 MachineBasicBlock *SwitchMBB) {
1071 using namespace SwitchCG;
1072 const Value *RHS, *LHS, *MHS;
1073 CmpInst::Predicate Pred;
1074 if (I->Low == I->High) {
1075 // Check Cond == I->Low.
1076 Pred = CmpInst::ICMP_EQ;
1077 LHS = Cond;
1078 RHS = I->Low;
1079 MHS = nullptr;
1080 } else {
1081 // Check I->Low <= Cond <= I->High.
1082 Pred = CmpInst::ICMP_SLE;
1083 LHS = I->Low;
1084 MHS = Cond;
1085 RHS = I->High;
1086 }
1087
1088 // If Fallthrough is unreachable, fold away the comparison.
1089 // The false probability is the sum of all unhandled cases.
1090 CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
1091 CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
1092
1093 emitSwitchCase(CB, SwitchMBB, MIB);
1094 return true;
1095}
1096
1097void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
1098 MachineBasicBlock *SwitchBB) {
1099 MachineIRBuilder &MIB = *CurBuilder;
1100 MIB.setMBB(*SwitchBB);
1101
1102 // Subtract the minimum value.
1103 Register SwitchOpReg = getOrCreateVReg(*B.SValue);
1104
1105 LLT SwitchOpTy = MRI->getType(SwitchOpReg);
1106 Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);
1107 auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
1108
1110 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1111
1112 LLT MaskTy = SwitchOpTy;
1113 if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
1114 !llvm::has_single_bit<uint32_t>(MaskTy.getSizeInBits()))
1115 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1116 else {
1117 // Ensure that the type will fit the mask value.
1118 for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
1119 if (!isUIntN(SwitchOpTy.getSizeInBits(), B.Cases[I].Mask)) {
1120 // Switch table case range are encoded into series of masks.
1121 // Just use pointer type, it's guaranteed to fit.
1122 MaskTy = LLT::scalar(PtrTy.getSizeInBits());
1123 break;
1124 }
1125 }
1126 }
1127 Register SubReg = RangeSub.getReg(0);
1128 if (SwitchOpTy != MaskTy)
1129 SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);
1130
1131 B.RegVT = getMVTForLLT(MaskTy);
1132 B.Reg = SubReg;
1133
1134 MachineBasicBlock *MBB = B.Cases[0].ThisBB;
1135
1136 if (!B.FallthroughUnreachable)
1137 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
1138 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
1139
1140 SwitchBB->normalizeSuccProbs();
1141
1142 if (!B.FallthroughUnreachable) {
1143 // Conditional branch to the default block.
1144 auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);
1145 auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),
1146 RangeSub, RangeCst);
1147 MIB.buildBrCond(RangeCmp, *B.Default);
1148 }
1149
1150 // Avoid emitting unnecessary branches to the next block.
1151 if (MBB != SwitchBB->getNextNode())
1152 MIB.buildBr(*MBB);
1153}
1154
1155void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
1156 MachineBasicBlock *NextMBB,
1157 BranchProbability BranchProbToNext,
1159 MachineBasicBlock *SwitchBB) {
1160 MachineIRBuilder &MIB = *CurBuilder;
1161 MIB.setMBB(*SwitchBB);
1162
1163 LLT SwitchTy = getLLTForMVT(BB.RegVT);
1164 Register Cmp;
1165 unsigned PopCount = llvm::popcount(B.Mask);
1166 if (PopCount == 1) {
1167 // Testing for a single bit; just compare the shift count with what it
1168 // would need to be to shift a 1 bit in that position.
1169 auto MaskTrailingZeros =
1170 MIB.buildConstant(SwitchTy, llvm::countr_zero(B.Mask));
1171 Cmp =
1172 MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
1173 .getReg(0);
1174 } else if (PopCount == BB.Range) {
1175 // There is only one zero bit in the range, test for it directly.
1176 auto MaskTrailingOnes =
1177 MIB.buildConstant(SwitchTy, llvm::countr_one(B.Mask));
1178 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
1179 .getReg(0);
1180 } else {
1181 // Make desired shift.
1182 auto CstOne = MIB.buildConstant(SwitchTy, 1);
1183 auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);
1184
1185 // Emit bit tests and jumps.
1186 auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);
1187 auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);
1188 auto CstZero = MIB.buildConstant(SwitchTy, 0);
1189 Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)
1190 .getReg(0);
1191 }
1192
1193 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
1194 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
1195 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
1196 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
1197 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
1198 // one as they are relative probabilities (and thus work more like weights),
1199 // and hence we need to normalize them to let the sum of them become one.
1200 SwitchBB->normalizeSuccProbs();
1201
1202 // Record the fact that the IR edge from the header to the bit test target
1203 // will go through our new block. Neeeded for PHIs to have nodes added.
1204 addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
1205 SwitchBB);
1206
1207 MIB.buildBrCond(Cmp, *B.TargetBB);
1208
1209 // Avoid emitting unnecessary branches to the next block.
1210 if (NextMBB != SwitchBB->getNextNode())
1211 MIB.buildBr(*NextMBB);
1212}
1213
1214bool IRTranslator::lowerBitTestWorkItem(
1216 MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
1218 BranchProbability DefaultProb, BranchProbability UnhandledProbs,
1220 bool FallthroughUnreachable) {
1221 using namespace SwitchCG;
1222 MachineFunction *CurMF = SwitchMBB->getParent();
1223 // FIXME: Optimize away range check based on pivot comparisons.
1224 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
1225 // The bit test blocks haven't been inserted yet; insert them here.
1226 for (BitTestCase &BTC : BTB->Cases)
1227 CurMF->insert(BBI, BTC.ThisBB);
1228
1229 // Fill in fields of the BitTestBlock.
1230 BTB->Parent = CurMBB;
1231 BTB->Default = Fallthrough;
1232
1233 BTB->DefaultProb = UnhandledProbs;
1234 // If the cases in bit test don't form a contiguous range, we evenly
1235 // distribute the probability on the edge to Fallthrough to two
1236 // successors of CurMBB.
1237 if (!BTB->ContiguousRange) {
1238 BTB->Prob += DefaultProb / 2;
1239 BTB->DefaultProb -= DefaultProb / 2;
1240 }
1241
1242 if (FallthroughUnreachable)
1243 BTB->FallthroughUnreachable = true;
1244
1245 // If we're in the right place, emit the bit test header right now.
1246 if (CurMBB == SwitchMBB) {
1247 emitBitTestHeader(*BTB, SwitchMBB);
1248 BTB->Emitted = true;
1249 }
1250 return true;
1251}
1252
1253bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
1254 Value *Cond,
1255 MachineBasicBlock *SwitchMBB,
1256 MachineBasicBlock *DefaultMBB,
1257 MachineIRBuilder &MIB) {
1258 using namespace SwitchCG;
1259 MachineFunction *CurMF = FuncInfo.MF;
1260 MachineBasicBlock *NextMBB = nullptr;
1262 if (++BBI != FuncInfo.MF->end())
1263 NextMBB = &*BBI;
1264
1265 if (EnableOpts) {
1266 // Here, we order cases by probability so the most likely case will be
1267 // checked first. However, two clusters can have the same probability in
1268 // which case their relative ordering is non-deterministic. So we use Low
1269 // as a tie-breaker as clusters are guaranteed to never overlap.
1270 llvm::sort(W.FirstCluster, W.LastCluster + 1,
1271 [](const CaseCluster &a, const CaseCluster &b) {
1272 return a.Prob != b.Prob
1273 ? a.Prob > b.Prob
1274 : a.Low->getValue().slt(b.Low->getValue());
1275 });
1276
1277 // Rearrange the case blocks so that the last one falls through if possible
1278 // without changing the order of probabilities.
1279 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
1280 --I;
1281 if (I->Prob > W.LastCluster->Prob)
1282 break;
1283 if (I->Kind == CC_Range && I->MBB == NextMBB) {
1284 std::swap(*I, *W.LastCluster);
1285 break;
1286 }
1287 }
1288 }
1289
1290 // Compute total probability.
1291 BranchProbability DefaultProb = W.DefaultProb;
1292 BranchProbability UnhandledProbs = DefaultProb;
1293 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
1294 UnhandledProbs += I->Prob;
1295
1296 MachineBasicBlock *CurMBB = W.MBB;
1297 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
1298 bool FallthroughUnreachable = false;
1299 MachineBasicBlock *Fallthrough;
1300 if (I == W.LastCluster) {
1301 // For the last cluster, fall through to the default destination.
1302 Fallthrough = DefaultMBB;
1303 FallthroughUnreachable = isa<UnreachableInst>(
1304 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
1305 } else {
1306 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
1307 CurMF->insert(BBI, Fallthrough);
1308 }
1309 UnhandledProbs -= I->Prob;
1310
1311 switch (I->Kind) {
1312 case CC_BitTests: {
1313 if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1314 DefaultProb, UnhandledProbs, I, Fallthrough,
1315 FallthroughUnreachable)) {
1316 LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
1317 return false;
1318 }
1319 break;
1320 }
1321
1322 case CC_JumpTable: {
1323 if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
1324 UnhandledProbs, I, Fallthrough,
1325 FallthroughUnreachable)) {
1326 LLVM_DEBUG(dbgs() << "Failed to lower jump table");
1327 return false;
1328 }
1329 break;
1330 }
1331 case CC_Range: {
1332 if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
1333 FallthroughUnreachable, UnhandledProbs,
1334 CurMBB, MIB, SwitchMBB)) {
1335 LLVM_DEBUG(dbgs() << "Failed to lower switch range");
1336 return false;
1337 }
1338 break;
1339 }
1340 }
1341 CurMBB = Fallthrough;
1342 }
1343
1344 return true;
1345}
1346
1347bool IRTranslator::translateIndirectBr(const User &U,
1348 MachineIRBuilder &MIRBuilder) {
1349 const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
1350
1351 const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
1352 MIRBuilder.buildBrIndirect(Tgt);
1353
1354 // Link successors.
1356 MachineBasicBlock &CurBB = MIRBuilder.getMBB();
1357 for (const BasicBlock *Succ : successors(&BrInst)) {
1358 // It's legal for indirectbr instructions to have duplicate blocks in the
1359 // destination list. We don't allow this in MIR. Skip anything that's
1360 // already a successor.
1361 if (!AddedSuccessors.insert(Succ).second)
1362 continue;
1363 CurBB.addSuccessor(&getMBB(*Succ));
1364 }
1365
1366 return true;
1367}
1368
1369static bool isSwiftError(const Value *V) {
1370 if (auto Arg = dyn_cast<Argument>(V))
1371 return Arg->hasSwiftErrorAttr();
1372 if (auto AI = dyn_cast<AllocaInst>(V))
1373 return AI->isSwiftError();
1374 return false;
1375}
1376
1377bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
1378 const LoadInst &LI = cast<LoadInst>(U);
1379 TypeSize StoreSize = DL->getTypeStoreSize(LI.getType());
1380 if (StoreSize.isZero())
1381 return true;
1382
1383 ArrayRef<Register> Regs = getOrCreateVRegs(LI);
1384 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
1385 Register Base = getOrCreateVReg(*LI.getPointerOperand());
1386 AAMDNodes AAInfo = LI.getAAMetadata();
1387
1388 const Value *Ptr = LI.getPointerOperand();
1389 Type *OffsetIRTy = DL->getIndexType(Ptr->getType());
1390 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1391
1392 if (CLI->supportSwiftError() && isSwiftError(Ptr)) {
1393 assert(Regs.size() == 1 && "swifterror should be single pointer");
1394 Register VReg =
1395 SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);
1396 MIRBuilder.buildCopy(Regs[0], VReg);
1397 return true;
1398 }
1399
1401 TLI->getLoadMemOperandFlags(LI, *DL, AC, LibInfo);
1402 if (AA && !(Flags & MachineMemOperand::MOInvariant)) {
1403 if (AA->pointsToConstantMemory(
1404 MemoryLocation(Ptr, LocationSize::precise(StoreSize), AAInfo))) {
1406 }
1407 }
1408
1409 const MDNode *Ranges =
1410 Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
1411 for (unsigned i = 0; i < Regs.size(); ++i) {
1412 Register Addr;
1413 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1414
1415 MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
1416 Align BaseAlign = getMemOpAlign(LI);
1417 auto MMO = MF->getMachineMemOperand(
1418 Ptr, Flags, MRI->getType(Regs[i]),
1419 commonAlignment(BaseAlign, Offsets[i] / 8), AAInfo, Ranges,
1420 LI.getSyncScopeID(), LI.getOrdering());
1421 MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
1422 }
1423
1424 return true;
1425}
1426
1427bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
1428 const StoreInst &SI = cast<StoreInst>(U);
1429 if (DL->getTypeStoreSize(SI.getValueOperand()->getType()).isZero())
1430 return true;
1431
1432 ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
1433 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
1434 Register Base = getOrCreateVReg(*SI.getPointerOperand());
1435
1436 Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());
1437 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1438
1439 if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
1440 assert(Vals.size() == 1 && "swifterror should be single pointer");
1441
1442 Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
1443 SI.getPointerOperand());
1444 MIRBuilder.buildCopy(VReg, Vals[0]);
1445 return true;
1446 }
1447
1449
1450 for (unsigned i = 0; i < Vals.size(); ++i) {
1451 Register Addr;
1452 MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
1453
1454 MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
1455 Align BaseAlign = getMemOpAlign(SI);
1456 auto MMO = MF->getMachineMemOperand(
1457 Ptr, Flags, MRI->getType(Vals[i]),
1458 commonAlignment(BaseAlign, Offsets[i] / 8), SI.getAAMetadata(), nullptr,
1459 SI.getSyncScopeID(), SI.getOrdering());
1460 MIRBuilder.buildStore(Vals[i], Addr, *MMO);
1461 }
1462 return true;
1463}
1464
1466 const Value *Src = U.getOperand(0);
1467 Type *Int32Ty = Type::getInt32Ty(U.getContext());
1468
1469 // getIndexedOffsetInType is designed for GEPs, so the first index is the
1470 // usual array element rather than looking into the actual aggregate.
1472 Indices.push_back(ConstantInt::get(Int32Ty, 0));
1473
1474 if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
1475 for (auto Idx : EVI->indices())
1476 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1477 } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
1478 for (auto Idx : IVI->indices())
1479 Indices.push_back(ConstantInt::get(Int32Ty, Idx));
1480 } else {
1481 for (Value *Op : drop_begin(U.operands()))
1482 Indices.push_back(Op);
1483 }
1484
1485 return 8 * static_cast<uint64_t>(
1486 DL.getIndexedOffsetInType(Src->getType(), Indices));
1487}
1488
1489bool IRTranslator::translateExtractValue(const User &U,
1490 MachineIRBuilder &MIRBuilder) {
1491 const Value *Src = U.getOperand(0);
1493 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1494 ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
1495 unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
1496 auto &DstRegs = allocateVRegs(U);
1497
1498 for (unsigned i = 0; i < DstRegs.size(); ++i)
1499 DstRegs[i] = SrcRegs[Idx++];
1500
1501 return true;
1502}
1503
1504bool IRTranslator::translateInsertValue(const User &U,
1505 MachineIRBuilder &MIRBuilder) {
1506 const Value *Src = U.getOperand(0);
1508 auto &DstRegs = allocateVRegs(U);
1509 ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
1510 ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
1511 ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
1512 auto *InsertedIt = InsertedRegs.begin();
1513
1514 for (unsigned i = 0; i < DstRegs.size(); ++i) {
1515 if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
1516 DstRegs[i] = *InsertedIt++;
1517 else
1518 DstRegs[i] = SrcRegs[i];
1519 }
1520
1521 return true;
1522}
1523
1524bool IRTranslator::translateSelect(const User &U,
1525 MachineIRBuilder &MIRBuilder) {
1526 Register Tst = getOrCreateVReg(*U.getOperand(0));
1527 ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1528 ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1529 ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1530
1531 uint32_t Flags = 0;
1532 if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
1534
1535 for (unsigned i = 0; i < ResRegs.size(); ++i) {
1536 MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
1537 }
1538
1539 return true;
1540}
1541
1542bool IRTranslator::translateCopy(const User &U, const Value &V,
1543 MachineIRBuilder &MIRBuilder) {
1544 Register Src = getOrCreateVReg(V);
1545 auto &Regs = *VMap.getVRegs(U);
1546 if (Regs.empty()) {
1547 Regs.push_back(Src);
1548 VMap.getOffsets(U)->push_back(0);
1549 } else {
1550 // If we already assigned a vreg for this instruction, we can't change that.
1551 // Emit a copy to satisfy the users we already emitted.
1552 MIRBuilder.buildCopy(Regs[0], Src);
1553 }
1554 return true;
1555}
1556
1557bool IRTranslator::translateBitCast(const User &U,
1558 MachineIRBuilder &MIRBuilder) {
1559 // If we're bitcasting to the source type, we can reuse the source vreg.
1560 if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1561 getLLTForType(*U.getType(), *DL)) {
1562 // If the source is a ConstantInt then it was probably created by
1563 // ConstantHoisting and we should leave it alone.
1564 if (isa<ConstantInt>(U.getOperand(0)))
1565 return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,
1566 MIRBuilder);
1567 return translateCopy(U, *U.getOperand(0), MIRBuilder);
1568 }
1569
1570 return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1571}
1572
1573bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1574 MachineIRBuilder &MIRBuilder) {
1575 if (containsBF16Type(U))
1576 return false;
1577
1578 uint32_t Flags = 0;
1579 if (const Instruction *I = dyn_cast<Instruction>(&U))
1581
1582 Register Op = getOrCreateVReg(*U.getOperand(0));
1583 Register Res = getOrCreateVReg(U);
1584 MIRBuilder.buildInstr(Opcode, {Res}, {Op}, Flags);
1585 return true;
1586}
1587
1588bool IRTranslator::translateGetElementPtr(const User &U,
1589 MachineIRBuilder &MIRBuilder) {
1590 Value &Op0 = *U.getOperand(0);
1591 Register BaseReg = getOrCreateVReg(Op0);
1592 Type *PtrIRTy = Op0.getType();
1593 LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1594 Type *OffsetIRTy = DL->getIndexType(PtrIRTy);
1595 LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1596
1597 uint32_t Flags = 0;
1598 if (const Instruction *I = dyn_cast<Instruction>(&U))
1600
1601 // Normalize Vector GEP - all scalar operands should be converted to the
1602 // splat vector.
1603 unsigned VectorWidth = 0;
1604
1605 // True if we should use a splat vector; using VectorWidth alone is not
1606 // sufficient.
1607 bool WantSplatVector = false;
1608 if (auto *VT = dyn_cast<VectorType>(U.getType())) {
1609 VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
1610 // We don't produce 1 x N vectors; those are treated as scalars.
1611 WantSplatVector = VectorWidth > 1;
1612 }
1613
1614 // We might need to splat the base pointer into a vector if the offsets
1615 // are vectors.
1616 if (WantSplatVector && !PtrTy.isVector()) {
1617 BaseReg = MIRBuilder
1618 .buildSplatBuildVector(LLT::fixed_vector(VectorWidth, PtrTy),
1619 BaseReg)
1620 .getReg(0);
1621 PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
1622 PtrTy = getLLTForType(*PtrIRTy, *DL);
1623 OffsetIRTy = DL->getIndexType(PtrIRTy);
1624 OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1625 }
1626
1627 int64_t Offset = 0;
1628 for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1629 GTI != E; ++GTI) {
1630 const Value *Idx = GTI.getOperand();
1631 if (StructType *StTy = GTI.getStructTypeOrNull()) {
1632 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1634 continue;
1635 } else {
1636 uint64_t ElementSize = GTI.getSequentialElementStride(*DL);
1637
1638 // If this is a scalar constant or a splat vector of constants,
1639 // handle it quickly.
1640 if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1641 if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
1642 Offset += ElementSize * *Val;
1643 continue;
1644 }
1645 }
1646
1647 if (Offset != 0) {
1648 auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1649 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
1650 .getReg(0);
1651 Offset = 0;
1652 }
1653
1654 Register IdxReg = getOrCreateVReg(*Idx);
1655 LLT IdxTy = MRI->getType(IdxReg);
1656 if (IdxTy != OffsetTy) {
1657 if (!IdxTy.isVector() && WantSplatVector) {
1658 IdxReg = MIRBuilder
1660 IdxReg)
1661 .getReg(0);
1662 }
1663
1664 IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1665 }
1666
1667 // N = N + Idx * ElementSize;
1668 // Avoid doing it for ElementSize of 1.
1669 Register GepOffsetReg;
1670 if (ElementSize != 1) {
1671 auto ElementSizeMIB = MIRBuilder.buildConstant(
1672 getLLTForType(*OffsetIRTy, *DL), ElementSize);
1673 GepOffsetReg =
1674 MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
1675 } else
1676 GepOffsetReg = IdxReg;
1677
1678 BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1679 }
1680 }
1681
1682 if (Offset != 0) {
1683 auto OffsetMIB =
1684 MIRBuilder.buildConstant(OffsetTy, Offset);
1685
1686 if (int64_t(Offset) >= 0 && cast<GEPOperator>(U).isInBounds())
1688
1689 MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0),
1690 Flags);
1691 return true;
1692 }
1693
1694 MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1695 return true;
1696}
1697
1698bool IRTranslator::translateMemFunc(const CallInst &CI,
1699 MachineIRBuilder &MIRBuilder,
1700 unsigned Opcode) {
1701 const Value *SrcPtr = CI.getArgOperand(1);
1702 // If the source is undef, then just emit a nop.
1703 if (isa<UndefValue>(SrcPtr))
1704 return true;
1705
1707
1708 unsigned MinPtrSize = UINT_MAX;
1709 for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {
1710 Register SrcReg = getOrCreateVReg(**AI);
1711 LLT SrcTy = MRI->getType(SrcReg);
1712 if (SrcTy.isPointer())
1713 MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);
1714 SrcRegs.push_back(SrcReg);
1715 }
1716
1717 LLT SizeTy = LLT::scalar(MinPtrSize);
1718
1719 // The size operand should be the minimum of the pointer sizes.
1720 Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
1721 if (MRI->getType(SizeOpReg) != SizeTy)
1722 SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);
1723
1724 auto ICall = MIRBuilder.buildInstr(Opcode);
1725 for (Register SrcReg : SrcRegs)
1726 ICall.addUse(SrcReg);
1727
1728 Align DstAlign;
1729 Align SrcAlign;
1730 unsigned IsVol =
1731 cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();
1732
1733 ConstantInt *CopySize = nullptr;
1734
1735 if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1736 DstAlign = MCI->getDestAlign().valueOrOne();
1737 SrcAlign = MCI->getSourceAlign().valueOrOne();
1738 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1739 } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
1740 DstAlign = MCI->getDestAlign().valueOrOne();
1741 SrcAlign = MCI->getSourceAlign().valueOrOne();
1742 CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));
1743 } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1744 DstAlign = MMI->getDestAlign().valueOrOne();
1745 SrcAlign = MMI->getSourceAlign().valueOrOne();
1746 CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2));
1747 } else {
1748 auto *MSI = cast<MemSetInst>(&CI);
1749 DstAlign = MSI->getDestAlign().valueOrOne();
1750 }
1751
1752 if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
1753 // We need to propagate the tail call flag from the IR inst as an argument.
1754 // Otherwise, we have to pessimize and assume later that we cannot tail call
1755 // any memory intrinsics.
1756 ICall.addImm(CI.isTailCall() ? 1 : 0);
1757 }
1758
1759 // Create mem operands to store the alignment and volatile info.
1762 if (IsVol) {
1763 LoadFlags |= MachineMemOperand::MOVolatile;
1764 StoreFlags |= MachineMemOperand::MOVolatile;
1765 }
1766
1767 AAMDNodes AAInfo = CI.getAAMetadata();
1768 if (AA && CopySize &&
1770 SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) {
1771 LoadFlags |= MachineMemOperand::MOInvariant;
1772
1773 // FIXME: pointsToConstantMemory probably does not imply dereferenceable,
1774 // but the previous usage implied it did. Probably should check
1775 // isDereferenceableAndAlignedPointer.
1777 }
1778
1779 ICall.addMemOperand(
1781 StoreFlags, 1, DstAlign, AAInfo));
1782 if (Opcode != TargetOpcode::G_MEMSET)
1783 ICall.addMemOperand(MF->getMachineMemOperand(
1784 MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));
1785
1786 return true;
1787}
1788
1789bool IRTranslator::translateTrap(const CallInst &CI,
1790 MachineIRBuilder &MIRBuilder,
1791 unsigned Opcode) {
1792 StringRef TrapFuncName =
1793 CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
1794 if (TrapFuncName.empty()) {
1795 if (Opcode == TargetOpcode::G_UBSANTRAP) {
1796 uint64_t Code = cast<ConstantInt>(CI.getOperand(0))->getZExtValue();
1797 MIRBuilder.buildInstr(Opcode, {}, ArrayRef<llvm::SrcOp>{Code});
1798 } else {
1799 MIRBuilder.buildInstr(Opcode);
1800 }
1801 return true;
1802 }
1803
1805 if (Opcode == TargetOpcode::G_UBSANTRAP)
1806 Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
1807 CI.getArgOperand(0)->getType(), 0});
1808
1809 Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
1810 Info.CB = &CI;
1811 Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
1812 return CLI->lowerCall(MIRBuilder, Info);
1813}
1814
1815bool IRTranslator::translateVectorInterleave2Intrinsic(
1816 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1817 assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&
1818 "This function can only be called on the interleave2 intrinsic!");
1819 // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).
1820 Register Op0 = getOrCreateVReg(*CI.getOperand(0));
1821 Register Op1 = getOrCreateVReg(*CI.getOperand(1));
1822 Register Res = getOrCreateVReg(CI);
1823
1824 LLT OpTy = MRI->getType(Op0);
1825 MIRBuilder.buildShuffleVector(Res, Op0, Op1,
1827
1828 return true;
1829}
1830
1831bool IRTranslator::translateVectorDeinterleave2Intrinsic(
1832 const CallInst &CI, MachineIRBuilder &MIRBuilder) {
1833 assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&
1834 "This function can only be called on the deinterleave2 intrinsic!");
1835 // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to
1836 // SelectionDAG).
1837 Register Op = getOrCreateVReg(*CI.getOperand(0));
1838 auto Undef = MIRBuilder.buildUndef(MRI->getType(Op));
1839 ArrayRef<Register> Res = getOrCreateVRegs(CI);
1840
1841 LLT ResTy = MRI->getType(Res[0]);
1842 MIRBuilder.buildShuffleVector(Res[0], Op, Undef,
1843 createStrideMask(0, 2, ResTy.getNumElements()));
1844 MIRBuilder.buildShuffleVector(Res[1], Op, Undef,
1845 createStrideMask(1, 2, ResTy.getNumElements()));
1846
1847 return true;
1848}
1849
1850void IRTranslator::getStackGuard(Register DstReg,
1851 MachineIRBuilder &MIRBuilder) {
1853 MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1854 auto MIB =
1855 MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
1856
1858 if (!Global)
1859 return;
1860
1861 unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
1862 LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
1863
1864 MachinePointerInfo MPInfo(Global);
1868 MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
1869 MIB.setMemRefs({MemRef});
1870}
1871
1872bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1873 MachineIRBuilder &MIRBuilder) {
1874 ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1875 MIRBuilder.buildInstr(
1876 Op, {ResRegs[0], ResRegs[1]},
1877 {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});
1878
1879 return true;
1880}
1881
1882bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
1883 MachineIRBuilder &MIRBuilder) {
1884 Register Dst = getOrCreateVReg(CI);
1885 Register Src0 = getOrCreateVReg(*CI.getOperand(0));
1886 Register Src1 = getOrCreateVReg(*CI.getOperand(1));
1887 uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
1888 MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
1889 return true;
1890}
1891
1892unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1893 switch (ID) {
1894 default:
1895 break;
1896 case Intrinsic::acos:
1897 return TargetOpcode::G_FACOS;
1898 case Intrinsic::asin:
1899 return TargetOpcode::G_FASIN;
1900 case Intrinsic::atan:
1901 return TargetOpcode::G_FATAN;
1902 case Intrinsic::atan2:
1903 return TargetOpcode::G_FATAN2;
1904 case Intrinsic::bswap:
1905 return TargetOpcode::G_BSWAP;
1906 case Intrinsic::bitreverse:
1907 return TargetOpcode::G_BITREVERSE;
1908 case Intrinsic::fshl:
1909 return TargetOpcode::G_FSHL;
1910 case Intrinsic::fshr:
1911 return TargetOpcode::G_FSHR;
1912 case Intrinsic::ceil:
1913 return TargetOpcode::G_FCEIL;
1914 case Intrinsic::cos:
1915 return TargetOpcode::G_FCOS;
1916 case Intrinsic::cosh:
1917 return TargetOpcode::G_FCOSH;
1918 case Intrinsic::ctpop:
1919 return TargetOpcode::G_CTPOP;
1920 case Intrinsic::exp:
1921 return TargetOpcode::G_FEXP;
1922 case Intrinsic::exp2:
1923 return TargetOpcode::G_FEXP2;
1924 case Intrinsic::exp10:
1925 return TargetOpcode::G_FEXP10;
1926 case Intrinsic::fabs:
1927 return TargetOpcode::G_FABS;
1928 case Intrinsic::copysign:
1929 return TargetOpcode::G_FCOPYSIGN;
1930 case Intrinsic::minnum:
1931 return TargetOpcode::G_FMINNUM;
1932 case Intrinsic::maxnum:
1933 return TargetOpcode::G_FMAXNUM;
1934 case Intrinsic::minimum:
1935 return TargetOpcode::G_FMINIMUM;
1936 case Intrinsic::maximum:
1937 return TargetOpcode::G_FMAXIMUM;
1938 case Intrinsic::canonicalize:
1939 return TargetOpcode::G_FCANONICALIZE;
1940 case Intrinsic::floor:
1941 return TargetOpcode::G_FFLOOR;
1942 case Intrinsic::fma:
1943 return TargetOpcode::G_FMA;
1944 case Intrinsic::log:
1945 return TargetOpcode::G_FLOG;
1946 case Intrinsic::log2:
1947 return TargetOpcode::G_FLOG2;
1948 case Intrinsic::log10:
1949 return TargetOpcode::G_FLOG10;
1950 case Intrinsic::ldexp:
1951 return TargetOpcode::G_FLDEXP;
1952 case Intrinsic::nearbyint:
1953 return TargetOpcode::G_FNEARBYINT;
1954 case Intrinsic::pow:
1955 return TargetOpcode::G_FPOW;
1956 case Intrinsic::powi:
1957 return TargetOpcode::G_FPOWI;
1958 case Intrinsic::rint:
1959 return TargetOpcode::G_FRINT;
1960 case Intrinsic::round:
1961 return TargetOpcode::G_INTRINSIC_ROUND;
1962 case Intrinsic::roundeven:
1963 return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
1964 case Intrinsic::sin:
1965 return TargetOpcode::G_FSIN;
1966 case Intrinsic::sinh:
1967 return TargetOpcode::G_FSINH;
1968 case Intrinsic::sqrt:
1969 return TargetOpcode::G_FSQRT;
1970 case Intrinsic::tan:
1971 return TargetOpcode::G_FTAN;
1972 case Intrinsic::tanh:
1973 return TargetOpcode::G_FTANH;
1974 case Intrinsic::trunc:
1975 return TargetOpcode::G_INTRINSIC_TRUNC;
1976 case Intrinsic::readcyclecounter:
1977 return TargetOpcode::G_READCYCLECOUNTER;
1978 case Intrinsic::readsteadycounter:
1979 return TargetOpcode::G_READSTEADYCOUNTER;
1980 case Intrinsic::ptrmask:
1981 return TargetOpcode::G_PTRMASK;
1982 case Intrinsic::lrint:
1983 return TargetOpcode::G_INTRINSIC_LRINT;
1984 case Intrinsic::llrint:
1985 return TargetOpcode::G_INTRINSIC_LLRINT;
1986 // FADD/FMUL require checking the FMF, so are handled elsewhere.
1987 case Intrinsic::vector_reduce_fmin:
1988 return TargetOpcode::G_VECREDUCE_FMIN;
1989 case Intrinsic::vector_reduce_fmax:
1990 return TargetOpcode::G_VECREDUCE_FMAX;
1991 case Intrinsic::vector_reduce_fminimum:
1992 return TargetOpcode::G_VECREDUCE_FMINIMUM;
1993 case Intrinsic::vector_reduce_fmaximum:
1994 return TargetOpcode::G_VECREDUCE_FMAXIMUM;
1995 case Intrinsic::vector_reduce_add:
1996 return TargetOpcode::G_VECREDUCE_ADD;
1997 case Intrinsic::vector_reduce_mul:
1998 return TargetOpcode::G_VECREDUCE_MUL;
1999 case Intrinsic::vector_reduce_and:
2000 return TargetOpcode::G_VECREDUCE_AND;
2001 case Intrinsic::vector_reduce_or:
2002 return TargetOpcode::G_VECREDUCE_OR;
2003 case Intrinsic::vector_reduce_xor:
2004 return TargetOpcode::G_VECREDUCE_XOR;
2005 case Intrinsic::vector_reduce_smax:
2006 return TargetOpcode::G_VECREDUCE_SMAX;
2007 case Intrinsic::vector_reduce_smin:
2008 return TargetOpcode::G_VECREDUCE_SMIN;
2009 case Intrinsic::vector_reduce_umax:
2010 return TargetOpcode::G_VECREDUCE_UMAX;
2011 case Intrinsic::vector_reduce_umin:
2012 return TargetOpcode::G_VECREDUCE_UMIN;
2013 case Intrinsic::experimental_vector_compress:
2014 return TargetOpcode::G_VECTOR_COMPRESS;
2015 case Intrinsic::lround:
2016 return TargetOpcode::G_LROUND;
2017 case Intrinsic::llround:
2018 return TargetOpcode::G_LLROUND;
2019 case Intrinsic::get_fpenv:
2020 return TargetOpcode::G_GET_FPENV;
2021 case Intrinsic::get_fpmode:
2022 return TargetOpcode::G_GET_FPMODE;
2023 }
2025}
2026
2027bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
2029 MachineIRBuilder &MIRBuilder) {
2030
2031 unsigned Op = getSimpleIntrinsicOpcode(ID);
2032
2033 // Is this a simple intrinsic?
2035 return false;
2036
2037 // Yes. Let's translate it.
2039 for (const auto &Arg : CI.args())
2040 VRegs.push_back(getOrCreateVReg(*Arg));
2041
2042 MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
2044 return true;
2045}
2046
2047// TODO: Include ConstainedOps.def when all strict instructions are defined.
2049 switch (ID) {
2050 case Intrinsic::experimental_constrained_fadd:
2051 return TargetOpcode::G_STRICT_FADD;
2052 case Intrinsic::experimental_constrained_fsub:
2053 return TargetOpcode::G_STRICT_FSUB;
2054 case Intrinsic::experimental_constrained_fmul:
2055 return TargetOpcode::G_STRICT_FMUL;
2056 case Intrinsic::experimental_constrained_fdiv:
2057 return TargetOpcode::G_STRICT_FDIV;
2058 case Intrinsic::experimental_constrained_frem:
2059 return TargetOpcode::G_STRICT_FREM;
2060 case Intrinsic::experimental_constrained_fma:
2061 return TargetOpcode::G_STRICT_FMA;
2062 case Intrinsic::experimental_constrained_sqrt:
2063 return TargetOpcode::G_STRICT_FSQRT;
2064 case Intrinsic::experimental_constrained_ldexp:
2065 return TargetOpcode::G_STRICT_FLDEXP;
2066 default:
2067 return 0;
2068 }
2069}
2070
2071bool IRTranslator::translateConstrainedFPIntrinsic(
2072 const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
2074
2075 unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
2076 if (!Opcode)
2077 return false;
2078
2082
2084 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
2085 VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(I)));
2086
2087 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
2088 return true;
2089}
2090
2091std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {
2092 auto VRegs = getOrCreateVRegs(Arg);
2093 if (VRegs.size() != 1)
2094 return std::nullopt;
2095
2096 // Arguments are lowered as a copy of a livein physical register.
2097 auto *VRegDef = MF->getRegInfo().getVRegDef(VRegs[0]);
2098 if (!VRegDef || !VRegDef->isCopy())
2099 return std::nullopt;
2100 return VRegDef->getOperand(1).getReg().asMCReg();
2101}
2102
2103bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,
2104 const DILocalVariable *Var,
2105 const DIExpression *Expr,
2106 const DebugLoc &DL,
2107 MachineIRBuilder &MIRBuilder) {
2108 auto *Arg = dyn_cast<Argument>(Val);
2109 if (!Arg)
2110 return false;
2111
2112 if (!Expr->isEntryValue())
2113 return false;
2114
2115 std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);
2116 if (!PhysReg) {
2117 LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")
2118 << ": expression is entry_value but "
2119 << "couldn't find a physical register\n");
2120 LLVM_DEBUG(dbgs() << *Var << "\n");
2121 return true;
2122 }
2123
2124 if (isDeclare) {
2125 // Append an op deref to account for the fact that this is a dbg_declare.
2126 Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);
2127 MF->setVariableDbgInfo(Var, Expr, *PhysReg, DL);
2128 } else {
2129 MIRBuilder.buildDirectDbgValue(*PhysReg, Var, Expr);
2130 }
2131
2132 return true;
2133}
2134
2136 switch (ID) {
2137 default:
2138 llvm_unreachable("Unexpected intrinsic");
2139 case Intrinsic::experimental_convergence_anchor:
2140 return TargetOpcode::CONVERGENCECTRL_ANCHOR;
2141 case Intrinsic::experimental_convergence_entry:
2142 return TargetOpcode::CONVERGENCECTRL_ENTRY;
2143 case Intrinsic::experimental_convergence_loop:
2144 return TargetOpcode::CONVERGENCECTRL_LOOP;
2145 }
2146}
2147
2148bool IRTranslator::translateConvergenceControlIntrinsic(
2149 const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {
2151 Register OutputReg = getOrCreateConvergenceTokenVReg(CI);
2152 MIB.addDef(OutputReg);
2153
2154 if (ID == Intrinsic::experimental_convergence_loop) {
2156 assert(Bundle && "Expected a convergence control token.");
2157 Register InputReg =
2158 getOrCreateConvergenceTokenVReg(*Bundle->Inputs[0].get());
2159 MIB.addUse(InputReg);
2160 }
2161
2162 return true;
2163}
2164
2165bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
2166 MachineIRBuilder &MIRBuilder) {
2167 if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
2168 if (ORE->enabled()) {
2169 if (MemoryOpRemark::canHandle(MI, *LibInfo)) {
2170 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2171 R.visit(MI);
2172 }
2173 }
2174 }
2175
2176 // If this is a simple intrinsic (that is, we just need to add a def of
2177 // a vreg, and uses for each arg operand, then translate it.
2178 if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
2179 return true;
2180
2181 switch (ID) {
2182 default:
2183 break;
2184 case Intrinsic::lifetime_start:
2185 case Intrinsic::lifetime_end: {
2186 // No stack colouring in O0, discard region information.
2188 MF->getFunction().hasOptNone())
2189 return true;
2190
2191 unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
2192 : TargetOpcode::LIFETIME_END;
2193
2194 // Get the underlying objects for the location passed on the lifetime
2195 // marker.
2197 getUnderlyingObjects(CI.getArgOperand(1), Allocas);
2198
2199 // Iterate over each underlying object, creating lifetime markers for each
2200 // static alloca. Quit if we find a non-static alloca.
2201 for (const Value *V : Allocas) {
2202 const AllocaInst *AI = dyn_cast<AllocaInst>(V);
2203 if (!AI)
2204 continue;
2205
2206 if (!AI->isStaticAlloca())
2207 return true;
2208
2209 MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
2210 }
2211 return true;
2212 }
2213 case Intrinsic::fake_use: {
2215 for (const auto &Arg : CI.args())
2216 for (auto VReg : getOrCreateVRegs(*Arg))
2217 VRegs.push_back(VReg);
2218 MIRBuilder.buildInstr(TargetOpcode::FAKE_USE, {}, VRegs);
2219 MF->setHasFakeUses(true);
2220 return true;
2221 }
2222 case Intrinsic::dbg_declare: {
2223 const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
2224 assert(DI.getVariable() && "Missing variable");
2225 translateDbgDeclareRecord(DI.getAddress(), DI.hasArgList(), DI.getVariable(),
2226 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2227 return true;
2228 }
2229 case Intrinsic::dbg_label: {
2230 const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
2231 assert(DI.getLabel() && "Missing label");
2232
2234 MIRBuilder.getDebugLoc()) &&
2235 "Expected inlined-at fields to agree");
2236
2237 MIRBuilder.buildDbgLabel(DI.getLabel());
2238 return true;
2239 }
2240 case Intrinsic::vaend:
2241 // No target I know of cares about va_end. Certainly no in-tree target
2242 // does. Simplest intrinsic ever!
2243 return true;
2244 case Intrinsic::vastart: {
2245 Value *Ptr = CI.getArgOperand(0);
2246 unsigned ListSize = TLI->getVaListSizeInBits(*DL) / 8;
2247 Align Alignment = getKnownAlignment(Ptr, *DL);
2248
2249 MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
2250 .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),
2252 ListSize, Alignment));
2253 return true;
2254 }
2255 case Intrinsic::dbg_assign:
2256 // A dbg.assign is a dbg.value with more information about stack locations,
2257 // typically produced during optimisation of variables with leaked
2258 // addresses. We can treat it like a normal dbg_value intrinsic here; to
2259 // benefit from the full analysis of stack/SSA locations, GlobalISel would
2260 // need to register for and use the AssignmentTrackingAnalysis pass.
2261 [[fallthrough]];
2262 case Intrinsic::dbg_value: {
2263 // This form of DBG_VALUE is target-independent.
2264 const DbgValueInst &DI = cast<DbgValueInst>(CI);
2265 translateDbgValueRecord(DI.getValue(), DI.hasArgList(), DI.getVariable(),
2266 DI.getExpression(), DI.getDebugLoc(), MIRBuilder);
2267 return true;
2268 }
2269 case Intrinsic::uadd_with_overflow:
2270 return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
2271 case Intrinsic::sadd_with_overflow:
2272 return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
2273 case Intrinsic::usub_with_overflow:
2274 return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
2275 case Intrinsic::ssub_with_overflow:
2276 return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
2277 case Intrinsic::umul_with_overflow:
2278 return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
2279 case Intrinsic::smul_with_overflow:
2280 return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
2281 case Intrinsic::uadd_sat:
2282 return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
2283 case Intrinsic::sadd_sat:
2284 return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
2285 case Intrinsic::usub_sat:
2286 return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
2287 case Intrinsic::ssub_sat:
2288 return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
2289 case Intrinsic::ushl_sat:
2290 return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
2291 case Intrinsic::sshl_sat:
2292 return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
2293 case Intrinsic::umin:
2294 return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
2295 case Intrinsic::umax:
2296 return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
2297 case Intrinsic::smin:
2298 return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
2299 case Intrinsic::smax:
2300 return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
2301 case Intrinsic::abs:
2302 // TODO: Preserve "int min is poison" arg in GMIR?
2303 return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
2304 case Intrinsic::smul_fix:
2305 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
2306 case Intrinsic::umul_fix:
2307 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
2308 case Intrinsic::smul_fix_sat:
2309 return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
2310 case Intrinsic::umul_fix_sat:
2311 return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
2312 case Intrinsic::sdiv_fix:
2313 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
2314 case Intrinsic::udiv_fix:
2315 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
2316 case Intrinsic::sdiv_fix_sat:
2317 return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
2318 case Intrinsic::udiv_fix_sat:
2319 return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
2320 case Intrinsic::fmuladd: {
2321 const TargetMachine &TM = MF->getTarget();
2322 Register Dst = getOrCreateVReg(CI);
2323 Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
2324 Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
2325 Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
2326 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
2328 TLI->getValueType(*DL, CI.getType()))) {
2329 // TODO: Revisit this to see if we should move this part of the
2330 // lowering to the combiner.
2331 MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,
2333 } else {
2334 LLT Ty = getLLTForType(*CI.getType(), *DL);
2335 auto FMul = MIRBuilder.buildFMul(
2336 Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));
2337 MIRBuilder.buildFAdd(Dst, FMul, Op2,
2339 }
2340 return true;
2341 }
2342 case Intrinsic::convert_from_fp16:
2343 // FIXME: This intrinsic should probably be removed from the IR.
2344 MIRBuilder.buildFPExt(getOrCreateVReg(CI),
2345 getOrCreateVReg(*CI.getArgOperand(0)),
2347 return true;
2348 case Intrinsic::convert_to_fp16:
2349 // FIXME: This intrinsic should probably be removed from the IR.
2350 MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
2351 getOrCreateVReg(*CI.getArgOperand(0)),
2353 return true;
2354 case Intrinsic::frexp: {
2355 ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
2356 MIRBuilder.buildFFrexp(VRegs[0], VRegs[1],
2357 getOrCreateVReg(*CI.getArgOperand(0)),
2359 return true;
2360 }
2361 case Intrinsic::sincos: {
2362 ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
2363 MIRBuilder.buildFSincos(VRegs[0], VRegs[1],
2364 getOrCreateVReg(*CI.getArgOperand(0)),
2366 return true;
2367 }
2368 case Intrinsic::fptosi_sat:
2369 MIRBuilder.buildFPTOSI_SAT(getOrCreateVReg(CI),
2370 getOrCreateVReg(*CI.getArgOperand(0)));
2371 return true;
2372 case Intrinsic::fptoui_sat:
2373 MIRBuilder.buildFPTOUI_SAT(getOrCreateVReg(CI),
2374 getOrCreateVReg(*CI.getArgOperand(0)));
2375 return true;
2376 case Intrinsic::memcpy_inline:
2377 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
2378 case Intrinsic::memcpy:
2379 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
2380 case Intrinsic::memmove:
2381 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
2382 case Intrinsic::memset:
2383 return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
2384 case Intrinsic::eh_typeid_for: {
2386 Register Reg = getOrCreateVReg(CI);
2387 unsigned TypeID = MF->getTypeIDFor(GV);
2388 MIRBuilder.buildConstant(Reg, TypeID);
2389 return true;
2390 }
2391 case Intrinsic::objectsize:
2392 llvm_unreachable("llvm.objectsize.* should have been lowered already");
2393
2394 case Intrinsic::is_constant:
2395 llvm_unreachable("llvm.is.constant.* should have been lowered already");
2396
2397 case Intrinsic::stackguard:
2398 getStackGuard(getOrCreateVReg(CI), MIRBuilder);
2399 return true;
2400 case Intrinsic::stackprotector: {
2401 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2402 Register GuardVal;
2403 if (TLI->useLoadStackGuardNode(*CI.getModule())) {
2404 GuardVal = MRI->createGenericVirtualRegister(PtrTy);
2405 getStackGuard(GuardVal, MIRBuilder);
2406 } else
2407 GuardVal = getOrCreateVReg(*CI.getArgOperand(0)); // The guard's value.
2408
2409 AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
2410 int FI = getOrCreateFrameIndex(*Slot);
2412
2413 MIRBuilder.buildStore(
2414 GuardVal, getOrCreateVReg(*Slot),
2418 PtrTy, Align(8)));
2419 return true;
2420 }
2421 case Intrinsic::stacksave: {
2422 MIRBuilder.buildInstr(TargetOpcode::G_STACKSAVE, {getOrCreateVReg(CI)}, {});
2423 return true;
2424 }
2425 case Intrinsic::stackrestore: {
2426 MIRBuilder.buildInstr(TargetOpcode::G_STACKRESTORE, {},
2427 {getOrCreateVReg(*CI.getArgOperand(0))});
2428 return true;
2429 }
2430 case Intrinsic::cttz:
2431 case Intrinsic::ctlz: {
2432 ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
2433 bool isTrailing = ID == Intrinsic::cttz;
2434 unsigned Opcode = isTrailing
2435 ? Cst->isZero() ? TargetOpcode::G_CTTZ
2436 : TargetOpcode::G_CTTZ_ZERO_UNDEF
2437 : Cst->isZero() ? TargetOpcode::G_CTLZ
2438 : TargetOpcode::G_CTLZ_ZERO_UNDEF;
2439 MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
2440 {getOrCreateVReg(*CI.getArgOperand(0))});
2441 return true;
2442 }
2443 case Intrinsic::invariant_start: {
2444 LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
2446 MIRBuilder.buildUndef(Undef);
2447 return true;
2448 }
2449 case Intrinsic::invariant_end:
2450 return true;
2451 case Intrinsic::expect:
2452 case Intrinsic::expect_with_probability:
2453 case Intrinsic::annotation:
2454 case Intrinsic::ptr_annotation:
2455 case Intrinsic::launder_invariant_group:
2456 case Intrinsic::strip_invariant_group: {
2457 // Drop the intrinsic, but forward the value.
2458 MIRBuilder.buildCopy(getOrCreateVReg(CI),
2459 getOrCreateVReg(*CI.getArgOperand(0)));
2460 return true;
2461 }
2462 case Intrinsic::assume:
2463 case Intrinsic::experimental_noalias_scope_decl:
2464 case Intrinsic::var_annotation:
2465 case Intrinsic::sideeffect:
2466 // Discard annotate attributes, assumptions, and artificial side-effects.
2467 return true;
2468 case Intrinsic::read_volatile_register:
2469 case Intrinsic::read_register: {
2470 Value *Arg = CI.getArgOperand(0);
2471 MIRBuilder
2472 .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
2473 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
2474 return true;
2475 }
2476 case Intrinsic::write_register: {
2477 Value *Arg = CI.getArgOperand(0);
2478 MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
2479 .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
2480 .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
2481 return true;
2482 }
2483 case Intrinsic::localescape: {
2484 MachineBasicBlock &EntryMBB = MF->front();
2486
2487 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
2488 // is the same on all targets.
2489 for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
2491 if (isa<ConstantPointerNull>(Arg))
2492 continue; // Skip null pointers. They represent a hole in index space.
2493
2494 int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
2495 MCSymbol *FrameAllocSym =
2496 MF->getContext().getOrCreateFrameAllocSymbol(EscapedName, Idx);
2497
2498 // This should be inserted at the start of the entry block.
2499 auto LocalEscape =
2500 MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)
2501 .addSym(FrameAllocSym)
2502 .addFrameIndex(FI);
2503
2504 EntryMBB.insert(EntryMBB.begin(), LocalEscape);
2505 }
2506
2507 return true;
2508 }
2509 case Intrinsic::vector_reduce_fadd:
2510 case Intrinsic::vector_reduce_fmul: {
2511 // Need to check for the reassoc flag to decide whether we want a
2512 // sequential reduction opcode or not.
2513 Register Dst = getOrCreateVReg(CI);
2514 Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));
2515 Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));
2516 unsigned Opc = 0;
2517 if (!CI.hasAllowReassoc()) {
2518 // The sequential ordering case.
2519 Opc = ID == Intrinsic::vector_reduce_fadd
2520 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
2521 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
2522 MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
2524 return true;
2525 }
2526 // We split the operation into a separate G_FADD/G_FMUL + the reduce,
2527 // since the associativity doesn't matter.
2528 unsigned ScalarOpc;
2529 if (ID == Intrinsic::vector_reduce_fadd) {
2530 Opc = TargetOpcode::G_VECREDUCE_FADD;
2531 ScalarOpc = TargetOpcode::G_FADD;
2532 } else {
2533 Opc = TargetOpcode::G_VECREDUCE_FMUL;
2534 ScalarOpc = TargetOpcode::G_FMUL;
2535 }
2536 LLT DstTy = MRI->getType(Dst);
2537 auto Rdx = MIRBuilder.buildInstr(
2538 Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));
2539 MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
2541
2542 return true;
2543 }
2544 case Intrinsic::trap:
2545 return translateTrap(CI, MIRBuilder, TargetOpcode::G_TRAP);
2546 case Intrinsic::debugtrap:
2547 return translateTrap(CI, MIRBuilder, TargetOpcode::G_DEBUGTRAP);
2548 case Intrinsic::ubsantrap:
2549 return translateTrap(CI, MIRBuilder, TargetOpcode::G_UBSANTRAP);
2550 case Intrinsic::allow_runtime_check:
2551 case Intrinsic::allow_ubsan_check:
2552 MIRBuilder.buildCopy(getOrCreateVReg(CI),
2553 getOrCreateVReg(*ConstantInt::getTrue(CI.getType())));
2554 return true;
2555 case Intrinsic::amdgcn_cs_chain:
2556 return translateCallBase(CI, MIRBuilder);
2557 case Intrinsic::fptrunc_round: {
2559
2560 // Convert the metadata argument to a constant integer
2561 Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata();
2562 std::optional<RoundingMode> RoundMode =
2563 convertStrToRoundingMode(cast<MDString>(MD)->getString());
2564
2565 // Add the Rounding mode as an integer
2566 MIRBuilder
2567 .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,
2568 {getOrCreateVReg(CI)},
2569 {getOrCreateVReg(*CI.getArgOperand(0))}, Flags)
2570 .addImm((int)*RoundMode);
2571
2572 return true;
2573 }
2574 case Intrinsic::is_fpclass: {
2575 Value *FpValue = CI.getOperand(0);
2576 ConstantInt *TestMaskValue = cast<ConstantInt>(CI.getOperand(1));
2577
2578 MIRBuilder
2579 .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},
2580 {getOrCreateVReg(*FpValue)})
2581 .addImm(TestMaskValue->getZExtValue());
2582
2583 return true;
2584 }
2585 case Intrinsic::set_fpenv: {
2586 Value *FPEnv = CI.getOperand(0);
2587 MIRBuilder.buildSetFPEnv(getOrCreateVReg(*FPEnv));
2588 return true;
2589 }
2590 case Intrinsic::reset_fpenv:
2591 MIRBuilder.buildResetFPEnv();
2592 return true;
2593 case Intrinsic::set_fpmode: {
2594 Value *FPState = CI.getOperand(0);
2595 MIRBuilder.buildSetFPMode(getOrCreateVReg(*FPState));
2596 return true;
2597 }
2598 case Intrinsic::reset_fpmode:
2599 MIRBuilder.buildResetFPMode();
2600 return true;
2601 case Intrinsic::vscale: {
2602 MIRBuilder.buildVScale(getOrCreateVReg(CI), 1);
2603 return true;
2604 }
2605 case Intrinsic::scmp:
2606 MIRBuilder.buildSCmp(getOrCreateVReg(CI),
2607 getOrCreateVReg(*CI.getOperand(0)),
2608 getOrCreateVReg(*CI.getOperand(1)));
2609 return true;
2610 case Intrinsic::ucmp:
2611 MIRBuilder.buildUCmp(getOrCreateVReg(CI),
2612 getOrCreateVReg(*CI.getOperand(0)),
2613 getOrCreateVReg(*CI.getOperand(1)));
2614 return true;
2615 case Intrinsic::vector_extract:
2616 return translateExtractVector(CI, MIRBuilder);
2617 case Intrinsic::vector_insert:
2618 return translateInsertVector(CI, MIRBuilder);
2619 case Intrinsic::stepvector: {
2620 MIRBuilder.buildStepVector(getOrCreateVReg(CI), 1);
2621 return true;
2622 }
2623 case Intrinsic::prefetch: {
2624 Value *Addr = CI.getOperand(0);
2625 unsigned RW = cast<ConstantInt>(CI.getOperand(1))->getZExtValue();
2626 unsigned Locality = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
2627 unsigned CacheType = cast<ConstantInt>(CI.getOperand(3))->getZExtValue();
2628
2630 auto &MMO = *MF->getMachineMemOperand(MachinePointerInfo(Addr), Flags,
2631 LLT(), Align());
2632
2633 MIRBuilder.buildPrefetch(getOrCreateVReg(*Addr), RW, Locality, CacheType,
2634 MMO);
2635
2636 return true;
2637 }
2638
2639 case Intrinsic::vector_interleave2:
2640 case Intrinsic::vector_deinterleave2: {
2641 // Both intrinsics have at least one operand.
2642 Value *Op0 = CI.getOperand(0);
2643 LLT ResTy = getLLTForType(*Op0->getType(), MIRBuilder.getDataLayout());
2644 if (!ResTy.isFixedVector())
2645 return false;
2646
2647 if (CI.getIntrinsicID() == Intrinsic::vector_interleave2)
2648 return translateVectorInterleave2Intrinsic(CI, MIRBuilder);
2649
2650 return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);
2651 }
2652
2653#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
2654 case Intrinsic::INTRINSIC:
2655#include "llvm/IR/ConstrainedOps.def"
2656 return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
2657 MIRBuilder);
2658 case Intrinsic::experimental_convergence_anchor:
2659 case Intrinsic::experimental_convergence_entry:
2660 case Intrinsic::experimental_convergence_loop:
2661 return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);
2662 }
2663 return false;
2664}
2665
2666bool IRTranslator::translateInlineAsm(const CallBase &CB,
2667 MachineIRBuilder &MIRBuilder) {
2668 if (containsBF16Type(CB))
2669 return false;
2670
2672
2673 if (!ALI) {
2674 LLVM_DEBUG(
2675 dbgs() << "Inline asm lowering is not supported for this target yet\n");
2676 return false;
2677 }
2678
2679 return ALI->lowerInlineAsm(
2680 MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });
2681}
2682
2683bool IRTranslator::translateCallBase(const CallBase &CB,
2684 MachineIRBuilder &MIRBuilder) {
2685 ArrayRef<Register> Res = getOrCreateVRegs(CB);
2686
2688 Register SwiftInVReg = 0;
2689 Register SwiftErrorVReg = 0;
2690 for (const auto &Arg : CB.args()) {
2691 if (CLI->supportSwiftError() && isSwiftError(Arg)) {
2692 assert(SwiftInVReg == 0 && "Expected only one swift error argument");
2693 LLT Ty = getLLTForType(*Arg->getType(), *DL);
2694 SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
2695 MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
2696 &CB, &MIRBuilder.getMBB(), Arg));
2697 Args.emplace_back(ArrayRef(SwiftInVReg));
2698 SwiftErrorVReg =
2699 SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
2700 continue;
2701 }
2702 Args.push_back(getOrCreateVRegs(*Arg));
2703 }
2704
2705 if (auto *CI = dyn_cast<CallInst>(&CB)) {
2706 if (ORE->enabled()) {
2707 if (MemoryOpRemark::canHandle(CI, *LibInfo)) {
2708 MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);
2709 R.visit(CI);
2710 }
2711 }
2712 }
2713
2714 std::optional<CallLowering::PtrAuthInfo> PAI;
2715 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_ptrauth)) {
2716 // Functions should never be ptrauth-called directly.
2717 assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
2718
2719 const Value *Key = Bundle->Inputs[0];
2720 const Value *Discriminator = Bundle->Inputs[1];
2721
2722 // Look through ptrauth constants to try to eliminate the matching bundle
2723 // and turn this into a direct call with no ptrauth.
2724 // CallLowering will use the raw pointer if it doesn't find the PAI.
2725 const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CB.getCalledOperand());
2726 if (!CalleeCPA || !isa<Function>(CalleeCPA->getPointer()) ||
2727 !CalleeCPA->isKnownCompatibleWith(Key, Discriminator, *DL)) {
2728 // If we can't make it direct, package the bundle into PAI.
2729 Register DiscReg = getOrCreateVReg(*Discriminator);
2730 PAI = CallLowering::PtrAuthInfo{cast<ConstantInt>(Key)->getZExtValue(),
2731 DiscReg};
2732 }
2733 }
2734
2735 Register ConvergenceCtrlToken = 0;
2736 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
2737 const auto &Token = *Bundle->Inputs[0].get();
2738 ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);
2739 }
2740
2741 // We don't set HasCalls on MFI here yet because call lowering may decide to
2742 // optimize into tail calls. Instead, we defer that to selection where a final
2743 // scan is done to check if any instructions are calls.
2744 bool Success = CLI->lowerCall(
2745 MIRBuilder, CB, Res, Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
2746 [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
2747
2748 // Check if we just inserted a tail call.
2749 if (Success) {
2750 assert(!HasTailCall && "Can't tail call return twice from block?");
2752 HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
2753 }
2754
2755 return Success;
2756}
2757
2758bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
2759 if (containsBF16Type(U))
2760 return false;
2761
2762 const CallInst &CI = cast<CallInst>(U);
2763 auto TII = MF->getTarget().getIntrinsicInfo();
2764 const Function *F = CI.getCalledFunction();
2765
2766 // FIXME: support Windows dllimport function calls and calls through
2767 // weak symbols.
2768 if (F && (F->hasDLLImportStorageClass() ||
2770 F->hasExternalWeakLinkage())))
2771 return false;
2772
2773 // FIXME: support control flow guard targets.
2775 return false;
2776
2777 // FIXME: support statepoints and related.
2778 if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
2779 return false;
2780
2781 if (CI.isInlineAsm())
2782 return translateInlineAsm(CI, MIRBuilder);
2783
2784 diagnoseDontCall(CI);
2785
2787 if (F && F->isIntrinsic()) {
2788 ID = F->getIntrinsicID();
2790 ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
2791 }
2792
2793 if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
2794 return translateCallBase(CI, MIRBuilder);
2795
2796 assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
2797
2798 if (translateKnownIntrinsic(CI, ID, MIRBuilder))
2799 return true;
2800
2801 ArrayRef<Register> ResultRegs;
2802 if (!CI.getType()->isVoidTy())
2803 ResultRegs = getOrCreateVRegs(CI);
2804
2805 // Ignore the callsite attributes. Backend code is most likely not expecting
2806 // an intrinsic to sometimes have side effects and sometimes not.
2807 MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);
2808 if (isa<FPMathOperator>(CI))
2809 MIB->copyIRFlags(CI);
2810
2811 for (const auto &Arg : enumerate(CI.args())) {
2812 // If this is required to be an immediate, don't materialize it in a
2813 // register.
2814 if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
2815 if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
2816 // imm arguments are more convenient than cimm (and realistically
2817 // probably sufficient), so use them.
2818 assert(CI->getBitWidth() <= 64 &&
2819 "large intrinsic immediates not handled");
2820 MIB.addImm(CI->getSExtValue());
2821 } else {
2822 MIB.addFPImm(cast<ConstantFP>(Arg.value()));
2823 }
2824 } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {
2825 auto *MD = MDVal->getMetadata();
2826 auto *MDN = dyn_cast<MDNode>(MD);
2827 if (!MDN) {
2828 if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))
2829 MDN = MDNode::get(MF->getFunction().getContext(), ConstMD);
2830 else // This was probably an MDString.
2831 return false;
2832 }
2833 MIB.addMetadata(MDN);
2834 } else {
2835 ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
2836 if (VRegs.size() > 1)
2837 return false;
2838 MIB.addUse(VRegs[0]);
2839 }
2840 }
2841
2842 // Add a MachineMemOperand if it is a target mem intrinsic.
2844 // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
2845 if (TLI->getTgtMemIntrinsic(Info, CI, *MF, ID)) {
2846 Align Alignment = Info.align.value_or(
2847 DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
2848 LLT MemTy = Info.memVT.isSimple()
2849 ? getLLTForMVT(Info.memVT.getSimpleVT())
2850 : LLT::scalar(Info.memVT.getStoreSizeInBits());
2851
2852 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
2853 // didn't yield anything useful.
2855 if (Info.ptrVal)
2856 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
2857 else if (Info.fallbackAddressSpace)
2858 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
2859 MIB.addMemOperand(
2860 MF->getMachineMemOperand(MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata()));
2861 }
2862
2863 if (CI.isConvergent()) {
2864 if (auto Bundle = CI.getOperandBundle(LLVMContext::OB_convergencectrl)) {
2865 auto *Token = Bundle->Inputs[0].get();
2866 Register TokenReg = getOrCreateVReg(*Token);
2867 MIB.addUse(TokenReg, RegState::Implicit);
2868 }
2869 }
2870
2871 return true;
2872}
2873
2874bool IRTranslator::findUnwindDestinations(
2875 const BasicBlock *EHPadBB,
2876 BranchProbability Prob,
2877 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2878 &UnwindDests) {
2880 EHPadBB->getParent()->getFunction().getPersonalityFn());
2881 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2882 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2883 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2884 bool IsSEH = isAsynchronousEHPersonality(Personality);
2885
2886 if (IsWasmCXX) {
2887 // Ignore this for now.
2888 return false;
2889 }
2890
2891 while (EHPadBB) {
2893 BasicBlock *NewEHPadBB = nullptr;
2894 if (isa<LandingPadInst>(Pad)) {
2895 // Stop on landingpads. They are not funclets.
2896 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2897 break;
2898 }
2899 if (isa<CleanupPadInst>(Pad)) {
2900 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2901 // personalities.
2902 UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
2903 UnwindDests.back().first->setIsEHScopeEntry();
2904 UnwindDests.back().first->setIsEHFuncletEntry();
2905 break;
2906 }
2907 if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2908 // Add the catchpad handlers to the possible destinations.
2909 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2910 UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
2911 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2912 if (IsMSVCCXX || IsCoreCLR)
2913 UnwindDests.back().first->setIsEHFuncletEntry();
2914 if (!IsSEH)
2915 UnwindDests.back().first->setIsEHScopeEntry();
2916 }
2917 NewEHPadBB = CatchSwitch->getUnwindDest();
2918 } else {
2919 continue;
2920 }
2921
2922 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2923 if (BPI && NewEHPadBB)
2924 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2925 EHPadBB = NewEHPadBB;
2926 }
2927 return true;
2928}
2929
2930bool IRTranslator::translateInvoke(const User &U,
2931 MachineIRBuilder &MIRBuilder) {
2932 const InvokeInst &I = cast<InvokeInst>(U);
2933 MCContext &Context = MF->getContext();
2934
2935 const BasicBlock *ReturnBB = I.getSuccessor(0);
2936 const BasicBlock *EHPadBB = I.getSuccessor(1);
2937
2938 const Function *Fn = I.getCalledFunction();
2939
2940 // FIXME: support invoking patchpoint and statepoint intrinsics.
2941 if (Fn && Fn->isIntrinsic())
2942 return false;
2943
2944 // FIXME: support whatever these are.
2945 if (I.hasDeoptState())
2946 return false;
2947
2948 // FIXME: support control flow guard targets.
2949 if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
2950 return false;
2951
2952 // FIXME: support Windows exception handling.
2953 if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHIIt()))
2954 return false;
2955
2956 // FIXME: support Windows dllimport function calls and calls through
2957 // weak symbols.
2958 if (Fn && (Fn->hasDLLImportStorageClass() ||
2960 Fn->hasExternalWeakLinkage())))
2961 return false;
2962
2963 bool LowerInlineAsm = I.isInlineAsm();
2964 bool NeedEHLabel = true;
2965
2966 // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
2967 // the region covered by the try.
2968 MCSymbol *BeginSymbol = nullptr;
2969 if (NeedEHLabel) {
2970 MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START);
2971 BeginSymbol = Context.createTempSymbol();
2972 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
2973 }
2974
2975 if (LowerInlineAsm) {
2976 if (!translateInlineAsm(I, MIRBuilder))
2977 return false;
2978 } else if (!translateCallBase(I, MIRBuilder))
2979 return false;
2980
2981 MCSymbol *EndSymbol = nullptr;
2982 if (NeedEHLabel) {
2983 EndSymbol = Context.createTempSymbol();
2984 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
2985 }
2986
2988 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2989 MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
2990 BranchProbability EHPadBBProb =
2991 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2993
2994 if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
2995 return false;
2996
2997 MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
2998 &ReturnMBB = getMBB(*ReturnBB);
2999 // Update successor info.
3000 addSuccessorWithProb(InvokeMBB, &ReturnMBB);
3001 for (auto &UnwindDest : UnwindDests) {
3002 UnwindDest.first->setIsEHPad();
3003 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3004 }
3005 InvokeMBB->normalizeSuccProbs();
3006
3007 if (NeedEHLabel) {
3008 assert(BeginSymbol && "Expected a begin symbol!");
3009 assert(EndSymbol && "Expected an end symbol!");
3010 MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
3011 }
3012
3013 MIRBuilder.buildBr(ReturnMBB);
3014 return true;
3015}
3016
3017bool IRTranslator::translateCallBr(const User &U,
3018 MachineIRBuilder &MIRBuilder) {
3019 // FIXME: Implement this.
3020 return false;
3021}
3022
3023bool IRTranslator::translateLandingPad(const User &U,
3024 MachineIRBuilder &MIRBuilder) {
3025 const LandingPadInst &LP = cast<LandingPadInst>(U);
3026
3027 MachineBasicBlock &MBB = MIRBuilder.getMBB();
3028
3029 MBB.setIsEHPad();
3030
3031 // If there aren't registers to copy the values into (e.g., during SjLj
3032 // exceptions), then don't bother.
3033 const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
3034 if (TLI->getExceptionPointerRegister(PersonalityFn) == 0 &&
3035 TLI->getExceptionSelectorRegister(PersonalityFn) == 0)
3036 return true;
3037
3038 // If landingpad's return type is token type, we don't create DAG nodes
3039 // for its exception pointer and selector value. The extraction of exception
3040 // pointer or selector value from token type landingpads is not currently
3041 // supported.
3042 if (LP.getType()->isTokenTy())
3043 return true;
3044
3045 // Add a label to mark the beginning of the landing pad. Deletion of the
3046 // landing pad can thus be detected via the MachineModuleInfo.
3047 MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
3048 .addSym(MF->addLandingPad(&MBB));
3049
3050 // If the unwinder does not preserve all registers, ensure that the
3051 // function marks the clobbered registers as used.
3053 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
3055
3056 LLT Ty = getLLTForType(*LP.getType(), *DL);
3058 MIRBuilder.buildUndef(Undef);
3059
3061 for (Type *Ty : cast<StructType>(LP.getType())->elements())
3062 Tys.push_back(getLLTForType(*Ty, *DL));
3063 assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
3064
3065 // Mark exception register as live in.
3066 Register ExceptionReg = TLI->getExceptionPointerRegister(PersonalityFn);
3067 if (!ExceptionReg)
3068 return false;
3069
3070 MBB.addLiveIn(ExceptionReg);
3071 ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
3072 MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
3073
3074 Register SelectorReg = TLI->getExceptionSelectorRegister(PersonalityFn);
3075 if (!SelectorReg)
3076 return false;
3077
3078 MBB.addLiveIn(SelectorReg);
3079 Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
3080 MIRBuilder.buildCopy(PtrVReg, SelectorReg);
3081 MIRBuilder.buildCast(ResRegs[1], PtrVReg);
3082
3083 return true;
3084}
3085
3086bool IRTranslator::translateAlloca(const User &U,
3087 MachineIRBuilder &MIRBuilder) {
3088 auto &AI = cast<AllocaInst>(U);
3089
3090 if (AI.isSwiftError())
3091 return true;
3092
3093 if (AI.isStaticAlloca()) {
3094 Register Res = getOrCreateVReg(AI);
3095 int FI = getOrCreateFrameIndex(AI);
3096 MIRBuilder.buildFrameIndex(Res, FI);
3097 return true;
3098 }
3099
3100 // FIXME: support stack probing for Windows.
3102 return false;
3103
3104 // Now we're in the harder dynamic case.
3105 Register NumElts = getOrCreateVReg(*AI.getArraySize());
3106 Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
3107 LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
3108 if (MRI->getType(NumElts) != IntPtrTy) {
3109 Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
3110 MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
3111 NumElts = ExtElts;
3112 }
3113
3114 Type *Ty = AI.getAllocatedType();
3115
3116 Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
3117 Register TySize =
3118 getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
3119 MIRBuilder.buildMul(AllocSize, NumElts, TySize);
3120
3121 // Round the size of the allocation up to the stack alignment size
3122 // by add SA-1 to the size. This doesn't overflow because we're computing
3123 // an address inside an alloca.
3124 Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
3125 auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
3126 auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
3128 auto AlignCst =
3129 MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
3130 auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
3131
3132 Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
3133 if (Alignment <= StackAlign)
3134 Alignment = Align(1);
3135 MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
3136
3137 MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
3139 return true;
3140}
3141
3142bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
3143 // FIXME: We may need more info about the type. Because of how LLT works,
3144 // we're completely discarding the i64/double distinction here (amongst
3145 // others). Fortunately the ABIs I know of where that matters don't use va_arg
3146 // anyway but that's not guaranteed.
3147 MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
3148 {getOrCreateVReg(*U.getOperand(0)),
3149 DL->getABITypeAlign(U.getType()).value()});
3150 return true;
3151}
3152
3153bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
3155 return true;
3156
3157 auto &UI = cast<UnreachableInst>(U);
3158
3159 // We may be able to ignore unreachable behind a noreturn call.
3160 if (const CallInst *Call = dyn_cast_or_null<CallInst>(UI.getPrevNode());
3161 Call && Call->doesNotReturn()) {
3163 return true;
3164 // Do not emit an additional trap instruction.
3165 if (Call->isNonContinuableTrap())
3166 return true;
3167 }
3168
3169 MIRBuilder.buildTrap();
3170 return true;
3171}
3172
3173bool IRTranslator::translateInsertElement(const User &U,
3174 MachineIRBuilder &MIRBuilder) {
3175 // If it is a <1 x Ty> vector, use the scalar as it is
3176 // not a legal vector type in LLT.
3177 if (auto *FVT = dyn_cast<FixedVectorType>(U.getType());
3178 FVT && FVT->getNumElements() == 1)
3179 return translateCopy(U, *U.getOperand(1), MIRBuilder);
3180
3181 Register Res = getOrCreateVReg(U);
3182 Register Val = getOrCreateVReg(*U.getOperand(0));
3183 Register Elt = getOrCreateVReg(*U.getOperand(1));
3184 unsigned PreferredVecIdxWidth = TLI->getVectorIdxTy(*DL).getSizeInBits();
3185 Register Idx;
3186 if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(2))) {
3187 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3188 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
3189 auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
3190 Idx = getOrCreateVReg(*NewIdxCI);
3191 }
3192 }
3193 if (!Idx)
3194 Idx = getOrCreateVReg(*U.getOperand(2));
3195 if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
3196 const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
3197 Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
3198 }
3199 MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
3200 return true;
3201}
3202
3203bool IRTranslator::translateInsertVector(const User &U,
3204 MachineIRBuilder &MIRBuilder) {
3205 Register Dst = getOrCreateVReg(U);
3206 Register Vec = getOrCreateVReg(*U.getOperand(0));
3207 Register Elt = getOrCreateVReg(*U.getOperand(1));
3208
3209 ConstantInt *CI = cast<ConstantInt>(U.getOperand(2));
3210 unsigned PreferredVecIdxWidth = TLI->getVectorIdxTy(*DL).getSizeInBits();
3211
3212 // Resize Index to preferred index width.
3213 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3214 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
3215 CI = ConstantInt::get(CI->getContext(), NewIdx);
3216 }
3217
3218 // If it is a <1 x Ty> vector, we have to use other means.
3219 if (auto *ResultType = dyn_cast<FixedVectorType>(U.getOperand(1)->getType());
3220 ResultType && ResultType->getNumElements() == 1) {
3221 if (auto *InputType = dyn_cast<FixedVectorType>(U.getOperand(0)->getType());
3222 InputType && InputType->getNumElements() == 1) {
3223 // We are inserting an illegal fixed vector into an illegal
3224 // fixed vector, use the scalar as it is not a legal vector type
3225 // in LLT.
3226 return translateCopy(U, *U.getOperand(0), MIRBuilder);
3227 }
3228 if (isa<FixedVectorType>(U.getOperand(0)->getType())) {
3229 // We are inserting an illegal fixed vector into a legal fixed
3230 // vector, use the scalar as it is not a legal vector type in
3231 // LLT.
3232 Register Idx = getOrCreateVReg(*CI);
3233 MIRBuilder.buildInsertVectorElement(Dst, Vec, Elt, Idx);
3234 return true;
3235 }
3236 if (isa<ScalableVectorType>(U.getOperand(0)->getType())) {
3237 // We are inserting an illegal fixed vector into a scalable
3238 // vector, use a scalar element insert.
3239 LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
3240 Register Idx = getOrCreateVReg(*CI);
3241 auto ScaledIndex = MIRBuilder.buildMul(
3242 VecIdxTy, MIRBuilder.buildVScale(VecIdxTy, 1), Idx);
3243 MIRBuilder.buildInsertVectorElement(Dst, Vec, Elt, ScaledIndex);
3244 return true;
3245 }
3246 }
3247
3248 MIRBuilder.buildInsertSubvector(
3249 getOrCreateVReg(U), getOrCreateVReg(*U.getOperand(0)),
3250 getOrCreateVReg(*U.getOperand(1)), CI->getZExtValue());
3251 return true;
3252}
3253
3254bool IRTranslator::translateExtractElement(const User &U,
3255 MachineIRBuilder &MIRBuilder) {
3256 // If it is a <1 x Ty> vector, use the scalar as it is
3257 // not a legal vector type in LLT.
3258 if (const FixedVectorType *FVT =
3259 dyn_cast<FixedVectorType>(U.getOperand(0)->getType()))
3260 if (FVT->getNumElements() == 1)
3261 return translateCopy(U, *U.getOperand(0), MIRBuilder);
3262
3263 Register Res = getOrCreateVReg(U);
3264 Register Val = getOrCreateVReg(*U.getOperand(0));
3265 unsigned PreferredVecIdxWidth = TLI->getVectorIdxTy(*DL).getSizeInBits();
3266 Register Idx;
3267 if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
3268 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3269 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
3270 auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
3271 Idx = getOrCreateVReg(*NewIdxCI);
3272 }
3273 }
3274 if (!Idx)
3275 Idx = getOrCreateVReg(*U.getOperand(1));
3276 if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
3277 const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
3278 Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);
3279 }
3280 MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
3281 return true;
3282}
3283
3284bool IRTranslator::translateExtractVector(const User &U,
3285 MachineIRBuilder &MIRBuilder) {
3286 Register Res = getOrCreateVReg(U);
3287 Register Vec = getOrCreateVReg(*U.getOperand(0));
3288 ConstantInt *CI = cast<ConstantInt>(U.getOperand(1));
3289 unsigned PreferredVecIdxWidth = TLI->getVectorIdxTy(*DL).getSizeInBits();
3290
3291 // Resize Index to preferred index width.
3292 if (CI->getBitWidth() != PreferredVecIdxWidth) {
3293 APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);
3294 CI = ConstantInt::get(CI->getContext(), NewIdx);
3295 }
3296
3297 // If it is a <1 x Ty> vector, we have to use other means.
3298 if (auto *ResultType = dyn_cast<FixedVectorType>(U.getType());
3299 ResultType && ResultType->getNumElements() == 1) {
3300 if (auto *InputType = dyn_cast<FixedVectorType>(U.getOperand(0)->getType());
3301 InputType && InputType->getNumElements() == 1) {
3302 // We are extracting an illegal fixed vector from an illegal fixed vector,
3303 // use the scalar as it is not a legal vector type in LLT.
3304 return translateCopy(U, *U.getOperand(0), MIRBuilder);
3305 }
3306 if (isa<FixedVectorType>(U.getOperand(0)->getType())) {
3307 // We are extracting an illegal fixed vector from a legal fixed
3308 // vector, use the scalar as it is not a legal vector type in
3309 // LLT.
3310 Register Idx = getOrCreateVReg(*CI);
3311 MIRBuilder.buildExtractVectorElement(Res, Vec, Idx);
3312 return true;
3313 }
3314 if (isa<ScalableVectorType>(U.getOperand(0)->getType())) {
3315 // We are extracting an illegal fixed vector from a scalable
3316 // vector, use a scalar element extract.
3317 LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
3318 Register Idx = getOrCreateVReg(*CI);
3319 auto ScaledIndex = MIRBuilder.buildMul(
3320 VecIdxTy, MIRBuilder.buildVScale(VecIdxTy, 1), Idx);
3321 MIRBuilder.buildExtractVectorElement(Res, Vec, ScaledIndex);
3322 return true;
3323 }
3324 }
3325
3326 MIRBuilder.buildExtractSubvector(getOrCreateVReg(U),
3327 getOrCreateVReg(*U.getOperand(0)),
3328 CI->getZExtValue());
3329 return true;
3330}
3331
3332bool IRTranslator::translateShuffleVector(const User &U,
3333 MachineIRBuilder &MIRBuilder) {
3334 // A ShuffleVector that operates on scalable vectors is a splat vector where
3335 // the value of the splat vector is the 0th element of the first operand,
3336 // since the index mask operand is the zeroinitializer (undef and
3337 // poison are treated as zeroinitializer here).
3338 if (U.getOperand(0)->getType()->isScalableTy()) {
3339 Register Val = getOrCreateVReg(*U.getOperand(0));
3340 auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(
3341 MRI->getType(Val).getElementType(), Val, 0);
3342 MIRBuilder.buildSplatVector(getOrCreateVReg(U), SplatVal);
3343 return true;
3344 }
3345
3347 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
3348 Mask = SVI->getShuffleMask();
3349 else
3350 Mask = cast<ConstantExpr>(U).getShuffleMask();
3351 ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
3352 MIRBuilder
3353 .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
3354 {getOrCreateVReg(*U.getOperand(0)),
3355 getOrCreateVReg(*U.getOperand(1))})
3356 .addShuffleMask(MaskAlloc);
3357 return true;
3358}
3359
3360bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
3361 const PHINode &PI = cast<PHINode>(U);
3362
3364 for (auto Reg : getOrCreateVRegs(PI)) {
3365 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
3366 Insts.push_back(MIB.getInstr());
3367 }
3368
3369 PendingPHIs.emplace_back(&PI, std::move(Insts));
3370 return true;
3371}
3372
3373bool IRTranslator::translateAtomicCmpXchg(const User &U,
3374 MachineIRBuilder &MIRBuilder) {
3375 const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
3376
3377 auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);
3378
3379 auto Res = getOrCreateVRegs(I);
3380 Register OldValRes = Res[0];
3381 Register SuccessRes = Res[1];
3382 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3383 Register Cmp = getOrCreateVReg(*I.getCompareOperand());
3384 Register NewVal = getOrCreateVReg(*I.getNewValOperand());
3385
3387 OldValRes, SuccessRes, Addr, Cmp, NewVal,
3389 MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
3390 getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3391 I.getSuccessOrdering(), I.getFailureOrdering()));
3392 return true;
3393}
3394
3395bool IRTranslator::translateAtomicRMW(const User &U,
3396 MachineIRBuilder &MIRBuilder) {
3397 if (containsBF16Type(U))
3398 return false;
3399
3400 const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
3401 auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);
3402
3403 Register Res = getOrCreateVReg(I);
3404 Register Addr = getOrCreateVReg(*I.getPointerOperand());
3405 Register Val = getOrCreateVReg(*I.getValOperand());
3406
3407 unsigned Opcode = 0;
3408 switch (I.getOperation()) {
3409 default:
3410 return false;
3412 Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
3413 break;
3414 case AtomicRMWInst::Add:
3415 Opcode = TargetOpcode::G_ATOMICRMW_ADD;
3416 break;
3417 case AtomicRMWInst::Sub:
3418 Opcode = TargetOpcode::G_ATOMICRMW_SUB;
3419 break;
3420 case AtomicRMWInst::And:
3421 Opcode = TargetOpcode::G_ATOMICRMW_AND;
3422 break;
3424 Opcode = TargetOpcode::G_ATOMICRMW_NAND;
3425 break;
3426 case AtomicRMWInst::Or:
3427 Opcode = TargetOpcode::G_ATOMICRMW_OR;
3428 break;
3429 case AtomicRMWInst::Xor:
3430 Opcode = TargetOpcode::G_ATOMICRMW_XOR;
3431 break;
3432 case AtomicRMWInst::Max:
3433 Opcode = TargetOpcode::G_ATOMICRMW_MAX;
3434 break;
3435 case AtomicRMWInst::Min:
3436 Opcode = TargetOpcode::G_ATOMICRMW_MIN;
3437 break;
3439 Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
3440 break;
3442 Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
3443 break;
3445 Opcode = TargetOpcode::G_ATOMICRMW_FADD;
3446 break;
3448 Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
3449 break;
3451 Opcode = TargetOpcode::G_ATOMICRMW_FMAX;
3452 break;
3454 Opcode = TargetOpcode::G_ATOMICRMW_FMIN;
3455 break;
3457 Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;
3458 break;
3460 Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;
3461 break;
3463 Opcode = TargetOpcode::G_ATOMICRMW_USUB_COND;
3464 break;
3466 Opcode = TargetOpcode::G_ATOMICRMW_USUB_SAT;
3467 break;
3468 }
3469
3470 MIRBuilder.buildAtomicRMW(
3471 Opcode, Res, Addr, Val,
3472 *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3473 Flags, MRI->getType(Val), getMemOpAlign(I),
3474 I.getAAMetadata(), nullptr, I.getSyncScopeID(),
3475 I.getOrdering()));
3476 return true;
3477}
3478
3479bool IRTranslator::translateFence(const User &U,
3480 MachineIRBuilder &MIRBuilder) {
3481 const FenceInst &Fence = cast<FenceInst>(U);
3482 MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
3483 Fence.getSyncScopeID());
3484 return true;
3485}
3486
3487bool IRTranslator::translateFreeze(const User &U,
3488 MachineIRBuilder &MIRBuilder) {
3489 const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);
3490 const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));
3491
3492 assert(DstRegs.size() == SrcRegs.size() &&
3493 "Freeze with different source and destination type?");
3494
3495 for (unsigned I = 0; I < DstRegs.size(); ++I) {
3496 MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);
3497 }
3498
3499 return true;
3500}
3501
3502void IRTranslator::finishPendingPhis() {
3503#ifndef NDEBUG
3504 DILocationVerifier Verifier;
3505 GISelObserverWrapper WrapperObserver(&Verifier);
3506 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
3507#endif // ifndef NDEBUG
3508 for (auto &Phi : PendingPHIs) {
3509 const PHINode *PI = Phi.first;
3510 if (PI->getType()->isEmptyTy())
3511 continue;
3512 ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
3513 MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
3514 EntryBuilder->setDebugLoc(PI->getDebugLoc());
3515#ifndef NDEBUG
3516 Verifier.setCurrentInst(PI);
3517#endif // ifndef NDEBUG
3518
3520 for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
3521 auto IRPred = PI->getIncomingBlock(i);
3522 ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
3523 for (auto *Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
3524 if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
3525 continue;
3526 SeenPreds.insert(Pred);
3527 for (unsigned j = 0; j < ValRegs.size(); ++j) {
3528 MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
3529 MIB.addUse(ValRegs[j]);
3530 MIB.addMBB(Pred);
3531 }
3532 }
3533 }
3534 }
3535}
3536
3537void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,
3538 const DILocalVariable *Variable,
3539 const DIExpression *Expression,
3540 const DebugLoc &DL,
3541 MachineIRBuilder &MIRBuilder) {
3542 assert(Variable->isValidLocationForIntrinsic(DL) &&
3543 "Expected inlined-at fields to agree");
3544 // Act as if we're handling a debug intrinsic.
3545 MIRBuilder.setDebugLoc(DL);
3546
3547 if (!V || HasArgList) {
3548 // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
3549 // terminate any prior location.
3550 MIRBuilder.buildIndirectDbgValue(0, Variable, Expression);
3551 return;
3552 }
3553
3554 if (const auto *CI = dyn_cast<Constant>(V)) {
3555 MIRBuilder.buildConstDbgValue(*CI, Variable, Expression);
3556 return;
3557 }
3558
3559 if (auto *AI = dyn_cast<AllocaInst>(V);
3560 AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {
3561 // If the value is an alloca and the expression starts with a
3562 // dereference, track a stack slot instead of a register, as registers
3563 // may be clobbered.
3564 auto ExprOperands = Expression->getElements();
3565 auto *ExprDerefRemoved =
3566 DIExpression::get(AI->getContext(), ExprOperands.drop_front());
3567 MIRBuilder.buildFIDbgValue(getOrCreateFrameIndex(*AI), Variable,
3568 ExprDerefRemoved);
3569 return;
3570 }
3571 if (translateIfEntryValueArgument(false, V, Variable, Expression, DL,
3572 MIRBuilder))
3573 return;
3574 for (Register Reg : getOrCreateVRegs(*V)) {
3575 // FIXME: This does not handle register-indirect values at offset 0. The
3576 // direct/indirect thing shouldn't really be handled by something as
3577 // implicit as reg+noreg vs reg+imm in the first place, but it seems
3578 // pretty baked in right now.
3579 MIRBuilder.buildDirectDbgValue(Reg, Variable, Expression);
3580 }
3581}
3582
3583void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,
3584 const DILocalVariable *Variable,
3585 const DIExpression *Expression,
3586 const DebugLoc &DL,
3587 MachineIRBuilder &MIRBuilder) {
3588 if (!Address || isa<UndefValue>(Address)) {
3589 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");
3590 return;
3591 }
3592
3593 assert(Variable->isValidLocationForIntrinsic(DL) &&
3594 "Expected inlined-at fields to agree");
3595 auto AI = dyn_cast<AllocaInst>(Address);
3596 if (AI && AI->isStaticAlloca()) {
3597 // Static allocas are tracked at the MF level, no need for DBG_VALUE
3598 // instructions (in fact, they get ignored if they *do* exist).
3599 MF->setVariableDbgInfo(Variable, Expression,
3600 getOrCreateFrameIndex(*AI), DL);
3601 return;
3602 }
3603
3604 if (translateIfEntryValueArgument(true, Address, Variable,
3605 Expression, DL,
3606 MIRBuilder))
3607 return;
3608
3609 // A dbg.declare describes the address of a source variable, so lower it
3610 // into an indirect DBG_VALUE.
3611 MIRBuilder.setDebugLoc(DL);
3612 MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
3613 Variable, Expression);
3614 return;
3615}
3616
3617void IRTranslator::translateDbgInfo(const Instruction &Inst,
3618 MachineIRBuilder &MIRBuilder) {
3619 for (DbgRecord &DR : Inst.getDbgRecordRange()) {
3620 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
3621 MIRBuilder.setDebugLoc(DLR->getDebugLoc());
3622 assert(DLR->getLabel() && "Missing label");
3623 assert(DLR->getLabel()->isValidLocationForIntrinsic(
3624 MIRBuilder.getDebugLoc()) &&
3625 "Expected inlined-at fields to agree");
3626 MIRBuilder.buildDbgLabel(DLR->getLabel());
3627 continue;
3628 }
3629 DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
3630 const DILocalVariable *Variable = DVR.getVariable();
3631 const DIExpression *Expression = DVR.getExpression();
3632 Value *V = DVR.getVariableLocationOp(0);
3633 if (DVR.isDbgDeclare())
3634 translateDbgDeclareRecord(V, DVR.hasArgList(), Variable, Expression,
3635 DVR.getDebugLoc(), MIRBuilder);
3636 else
3637 translateDbgValueRecord(V, DVR.hasArgList(), Variable, Expression,
3638 DVR.getDebugLoc(), MIRBuilder);
3639 }
3640}
3641
3642bool IRTranslator::translate(const Instruction &Inst) {
3643 CurBuilder->setDebugLoc(Inst.getDebugLoc());
3644 CurBuilder->setPCSections(Inst.getMetadata(LLVMContext::MD_pcsections));
3645 CurBuilder->setMMRAMetadata(Inst.getMetadata(LLVMContext::MD_mmra));
3646
3647 if (TLI->fallBackToDAGISel(Inst))
3648 return false;
3649
3650 switch (Inst.getOpcode()) {
3651#define HANDLE_INST(NUM, OPCODE, CLASS) \
3652 case Instruction::OPCODE: \
3653 return translate##OPCODE(Inst, *CurBuilder.get());
3654#include "llvm/IR/Instruction.def"
3655 default:
3656 return false;
3657 }
3658}
3659
3660bool IRTranslator::translate(const Constant &C, Register Reg) {
3661 // We only emit constants into the entry block from here. To prevent jumpy
3662 // debug behaviour remove debug line.
3663 if (auto CurrInstDL = CurBuilder->getDL())
3664 EntryBuilder->setDebugLoc(DebugLoc());
3665
3666 if (auto CI = dyn_cast<ConstantInt>(&C))
3667 EntryBuilder->buildConstant(Reg, *CI);
3668 else if (auto CF = dyn_cast<ConstantFP>(&C))
3669 EntryBuilder->buildFConstant(Reg, *CF);
3670 else if (isa<UndefValue>(C))
3671 EntryBuilder->buildUndef(Reg);
3672 else if (isa<ConstantPointerNull>(C))
3673 EntryBuilder->buildConstant(Reg, 0);
3674 else if (auto GV = dyn_cast<GlobalValue>(&C))
3675 EntryBuilder->buildGlobalValue(Reg, GV);
3676 else if (auto CPA = dyn_cast<ConstantPtrAuth>(&C)) {
3677 Register Addr = getOrCreateVReg(*CPA->getPointer());
3678 Register AddrDisc = getOrCreateVReg(*CPA->getAddrDiscriminator());
3679 EntryBuilder->buildConstantPtrAuth(Reg, CPA, Addr, AddrDisc);
3680 } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
3681 Constant &Elt = *CAZ->getElementValue(0u);
3682 if (isa<ScalableVectorType>(CAZ->getType())) {
3683 EntryBuilder->buildSplatVector(Reg, getOrCreateVReg(Elt));
3684 return true;
3685 }
3686 // Return the scalar if it is a <1 x Ty> vector.
3687 unsigned NumElts = CAZ->getElementCount().getFixedValue();
3688 if (NumElts == 1)
3689 return translateCopy(C, Elt, *EntryBuilder);
3690 // All elements are zero so we can just use the first one.
3691 EntryBuilder->buildSplatBuildVector(Reg, getOrCreateVReg(Elt));
3692 } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
3693 // Return the scalar if it is a <1 x Ty> vector.
3694 if (CV->getNumElements() == 1)
3695 return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder);
3697 for (unsigned i = 0; i < CV->getNumElements(); ++i) {
3698 Constant &Elt = *CV->getElementAsConstant(i);
3699 Ops.push_back(getOrCreateVReg(Elt));
3700 }
3701 EntryBuilder->buildBuildVector(Reg, Ops);
3702 } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
3703 switch(CE->getOpcode()) {
3704#define HANDLE_INST(NUM, OPCODE, CLASS) \
3705 case Instruction::OPCODE: \
3706 return translate##OPCODE(*CE, *EntryBuilder.get());
3707#include "llvm/IR/Instruction.def"
3708 default:
3709 return false;
3710 }
3711 } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
3712 if (CV->getNumOperands() == 1)
3713 return translateCopy(C, *CV->getOperand(0), *EntryBuilder);
3715 for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
3716 Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
3717 }
3718 EntryBuilder->buildBuildVector(Reg, Ops);
3719 } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
3720 EntryBuilder->buildBlockAddress(Reg, BA);
3721 } else
3722 return false;
3723
3724 return true;
3725}
3726
3727bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
3729 for (auto &BTB : SL->BitTestCases) {
3730 // Emit header first, if it wasn't already emitted.
3731 if (!BTB.Emitted)
3732 emitBitTestHeader(BTB, BTB.Parent);
3733
3734 BranchProbability UnhandledProb = BTB.Prob;
3735 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
3736 UnhandledProb -= BTB.Cases[j].ExtraProb;
3737 // Set the current basic block to the mbb we wish to insert the code into
3738 MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
3739 // If all cases cover a contiguous range, it is not necessary to jump to
3740 // the default block after the last bit test fails. This is because the
3741 // range check during bit test header creation has guaranteed that every
3742 // case here doesn't go outside the range. In this case, there is no need
3743 // to perform the last bit test, as it will always be true. Instead, make
3744 // the second-to-last bit-test fall through to the target of the last bit
3745 // test, and delete the last bit test.
3746
3747 MachineBasicBlock *NextMBB;
3748 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3749 // Second-to-last bit-test with contiguous range: fall through to the
3750 // target of the final bit test.
3751 NextMBB = BTB.Cases[j + 1].TargetBB;
3752 } else if (j + 1 == ej) {
3753 // For the last bit test, fall through to Default.
3754 NextMBB = BTB.Default;
3755 } else {
3756 // Otherwise, fall through to the next bit test.
3757 NextMBB = BTB.Cases[j + 1].ThisBB;
3758 }
3759
3760 emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
3761
3762 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
3763 // We need to record the replacement phi edge here that normally
3764 // happens in emitBitTestCase before we delete the case, otherwise the
3765 // phi edge will be lost.
3766 addMachineCFGPred({BTB.Parent->getBasicBlock(),
3767 BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
3768 MBB);
3769 // Since we're not going to use the final bit test, remove it.
3770 BTB.Cases.pop_back();
3771 break;
3772 }
3773 }
3774 // This is "default" BB. We have two jumps to it. From "header" BB and from
3775 // last "case" BB, unless the latter was skipped.
3776 CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
3777 BTB.Default->getBasicBlock()};
3778 addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
3779 if (!BTB.ContiguousRange) {
3780 addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
3781 }
3782 }
3783 SL->BitTestCases.clear();
3784
3785 for (auto &JTCase : SL->JTCases) {
3786 // Emit header first, if it wasn't already emitted.
3787 if (!JTCase.first.Emitted)
3788 emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
3789
3790 emitJumpTable(JTCase.second, JTCase.second.MBB);
3791 }
3792 SL->JTCases.clear();
3793
3794 for (auto &SwCase : SL->SwitchCases)
3795 emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
3796 SL->SwitchCases.clear();
3797
3798 // Check if we need to generate stack-protector guard checks.
3799 StackProtector &SP = getAnalysis<StackProtector>();
3800 if (SP.shouldEmitSDCheck(BB)) {
3801 bool FunctionBasedInstrumentation =
3803 SPDescriptor.initialize(&BB, &MBB, FunctionBasedInstrumentation);
3804 }
3805 // Handle stack protector.
3806 if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
3807 LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
3808 return false;
3809 } else if (SPDescriptor.shouldEmitStackProtector()) {
3810 MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
3811 MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
3812
3813 // Find the split point to split the parent mbb. At the same time copy all
3814 // physical registers used in the tail of parent mbb into virtual registers
3815 // before the split point and back into physical registers after the split
3816 // point. This prevents us needing to deal with Live-ins and many other
3817 // register allocation issues caused by us splitting the parent mbb. The
3818 // register allocator will clean up said virtual copies later on.
3820 ParentMBB, *MF->getSubtarget().getInstrInfo());
3821
3822 // Splice the terminator of ParentMBB into SuccessMBB.
3823 SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
3824 ParentMBB->end());
3825
3826 // Add compare/jump on neq/jump to the parent BB.
3827 if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))
3828 return false;
3829
3830 // CodeGen Failure MBB if we have not codegened it yet.
3831 MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
3832 if (FailureMBB->empty()) {
3833 if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))
3834 return false;
3835 }
3836
3837 // Clear the Per-BB State.
3838 SPDescriptor.resetPerBBState();
3839 }
3840 return true;
3841}
3842
3843bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
3844 MachineBasicBlock *ParentBB) {
3845 CurBuilder->setInsertPt(*ParentBB, ParentBB->end());
3846 // First create the loads to the guard/stack slot for the comparison.
3848 const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
3849 LLT PtrMemTy = getLLTForMVT(TLI->getPointerMemTy(*DL));
3850
3851 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3852 int FI = MFI.getStackProtectorIndex();
3853
3854 Register Guard;
3855 Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);
3856 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3857 Align Align = DL->getPrefTypeAlign(PointerType::getUnqual(M.getContext()));
3858
3859 // Generate code to load the content of the guard slot.
3860 Register GuardVal =
3861 CurBuilder
3862 ->buildLoad(PtrMemTy, StackSlotPtr,
3865 .getReg(0);
3866
3867 if (TLI->useStackGuardXorFP()) {
3868 LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
3869 return false;
3870 }
3871
3872 // Retrieve guard check function, nullptr if instrumentation is inlined.
3873 if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M)) {
3874 // This path is currently untestable on GlobalISel, since the only platform
3875 // that needs this seems to be Windows, and we fall back on that currently.
3876 // The code still lives here in case that changes.
3877 // Silence warning about unused variable until the code below that uses
3878 // 'GuardCheckFn' is enabled.
3879 (void)GuardCheckFn;
3880 return false;
3881#if 0
3882 // The target provides a guard check function to validate the guard value.
3883 // Generate a call to that function with the content of the guard slot as
3884 // argument.
3885 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3886 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3888 if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
3889 Flags.setInReg();
3890 CallLowering::ArgInfo GuardArgInfo(
3891 {GuardVal, FnTy->getParamType(0), {Flags}});
3892
3894 Info.OrigArgs.push_back(GuardArgInfo);
3895 Info.CallConv = GuardCheckFn->getCallingConv();
3896 Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
3897 Info.OrigRet = {Register(), FnTy->getReturnType()};
3898 if (!CLI->lowerCall(MIRBuilder, Info)) {
3899 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
3900 return false;
3901 }
3902 return true;
3903#endif
3904 }
3905
3906 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3907 // Otherwise, emit a volatile load to retrieve the stack guard value.
3908 if (TLI->useLoadStackGuardNode(*ParentBB->getBasicBlock()->getModule())) {
3909 Guard =
3911 getStackGuard(Guard, *CurBuilder);
3912 } else {
3913 // TODO: test using android subtarget when we support @llvm.thread.pointer.
3914 const Value *IRGuard = TLI->getSDagStackGuard(M);
3915 Register GuardPtr = getOrCreateVReg(*IRGuard);
3916
3917 Guard = CurBuilder
3918 ->buildLoad(PtrMemTy, GuardPtr,
3922 .getReg(0);
3923 }
3924
3925 // Perform the comparison.
3926 auto Cmp =
3927 CurBuilder->buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Guard, GuardVal);
3928 // If the guard/stackslot do not equal, branch to failure MBB.
3929 CurBuilder->buildBrCond(Cmp, *SPD.getFailureMBB());
3930 // Otherwise branch to success MBB.
3931 CurBuilder->buildBr(*SPD.getSuccessMBB());
3932 return true;
3933}
3934
3935bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
3936 MachineBasicBlock *FailureBB) {
3937 CurBuilder->setInsertPt(*FailureBB, FailureBB->end());
3938
3939 const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
3940 const char *Name = TLI->getLibcallName(Libcall);
3941
3943 Info.CallConv = TLI->getLibcallCallingConv(Libcall);
3945 Info.OrigRet = {Register(), Type::getVoidTy(MF->getFunction().getContext()),
3946 0};
3947 if (!CLI->lowerCall(*CurBuilder, Info)) {
3948 LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
3949 return false;
3950 }
3951
3952 // Emit a trap instruction if we are required to do so.
3953 const TargetOptions &TargetOpts = TLI->getTargetMachine().Options;
3954 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3955 CurBuilder->buildInstr(TargetOpcode::G_TRAP);
3956
3957 return true;
3958}
3959
3960void IRTranslator::finalizeFunction() {
3961 // Release the memory used by the different maps we
3962 // needed during the translation.
3963 PendingPHIs.clear();
3964 VMap.reset();
3965 FrameIndices.clear();
3966 MachinePreds.clear();
3967 // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
3968 // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
3969 // destroying it twice (in ~IRTranslator() and ~LLVMContext())
3970 EntryBuilder.reset();
3971 CurBuilder.reset();
3972 FuncInfo.clear();
3973 SPDescriptor.resetPerFunctionState();
3974}
3975
3976/// Returns true if a BasicBlock \p BB within a variadic function contains a
3977/// variadic musttail call.
3978static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
3979 if (!IsVarArg)
3980 return false;
3981
3982 // Walk the block backwards, because tail calls usually only appear at the end
3983 // of a block.
3984 return llvm::any_of(llvm::reverse(BB), [](const Instruction &I) {
3985 const auto *CI = dyn_cast<CallInst>(&I);
3986 return CI && CI->isMustTailCall();
3987 });
3988}
3989
3991 MF = &CurMF;
3992 const Function &F = MF->getFunction();
3994 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
3995 // Set the CSEConfig and run the analysis.
3996 GISelCSEInfo *CSEInfo = nullptr;
3997 TPC = &getAnalysis<TargetPassConfig>();
4000 : TPC->isGISelCSEEnabled();
4001 TLI = MF->getSubtarget().getTargetLowering();
4002
4003 if (EnableCSE) {
4004 EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
4005 CSEInfo = &Wrapper.get(TPC->getCSEConfig());
4006 EntryBuilder->setCSEInfo(CSEInfo);
4007 CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
4008 CurBuilder->setCSEInfo(CSEInfo);
4009 } else {
4010 EntryBuilder = std::make_unique<MachineIRBuilder>();
4011 CurBuilder = std::make_unique<MachineIRBuilder>();
4012 }
4013 CLI = MF->getSubtarget().getCallLowering();
4014 CurBuilder->setMF(*MF);
4015 EntryBuilder->setMF(*MF);
4016 MRI = &MF->getRegInfo();
4017 DL = &F.getDataLayout();
4018 ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
4019 const TargetMachine &TM = MF->getTarget();
4020 TM.resetTargetOptions(F);
4021 EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);
4022 FuncInfo.MF = MF;
4023 if (EnableOpts) {
4024 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4025 FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
4026 } else {
4027 AA = nullptr;
4028 FuncInfo.BPI = nullptr;
4029 }
4030
4031 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
4032 MF->getFunction());
4033 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
4034 FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
4035
4036 SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
4037 SL->init(*TLI, TM, *DL);
4038
4039 assert(PendingPHIs.empty() && "stale PHIs");
4040
4041 // Targets which want to use big endian can enable it using
4042 // enableBigEndian()
4043 if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
4044 // Currently we don't properly handle big endian code.
4045 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4046 F.getSubprogram(), &F.getEntryBlock());
4047 R << "unable to translate in big endian mode";
4048 reportTranslationError(*MF, *TPC, *ORE, R);
4049 return false;
4050 }
4051
4052 // Release the per-function state when we return, whether we succeeded or not.
4053 auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
4054
4055 // Setup a separate basic-block for the arguments and constants
4057 MF->push_back(EntryBB);
4058 EntryBuilder->setMBB(*EntryBB);
4059
4060 DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHIIt()->getDebugLoc();
4061 SwiftError.setFunction(CurMF);
4062 SwiftError.createEntriesInEntryBlock(DbgLoc);
4063
4064 bool IsVarArg = F.isVarArg();
4065 bool HasMustTailInVarArgFn = false;
4066
4067 // Create all blocks, in IR order, to preserve the layout.
4068 FuncInfo.MBBMap.resize(F.getMaxBlockNumber());
4069 for (const BasicBlock &BB: F) {
4070 auto *&MBB = FuncInfo.MBBMap[BB.getNumber()];
4071
4072 MBB = MF->CreateMachineBasicBlock(&BB);
4073 MF->push_back(MBB);
4074
4075 if (BB.hasAddressTaken())
4076 MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
4077
4078 if (!HasMustTailInVarArgFn)
4079 HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
4080 }
4081
4082 MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
4083
4084 // Make our arguments/constants entry block fallthrough to the IR entry block.
4085 EntryBB->addSuccessor(&getMBB(F.front()));
4086
4087 if (CLI->fallBackToDAGISel(*MF)) {
4088 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4089 F.getSubprogram(), &F.getEntryBlock());
4090 R << "unable to lower function: "
4091 << ore::NV("Prototype", F.getFunctionType());
4092 reportTranslationError(*MF, *TPC, *ORE, R);
4093 return false;
4094 }
4095
4096 // Lower the actual args into this basic block.
4097 SmallVector<ArrayRef<Register>, 8> VRegArgs;
4098 for (const Argument &Arg: F.args()) {
4099 if (DL->getTypeStoreSize(Arg.getType()).isZero())
4100 continue; // Don't handle zero sized types.
4101 ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
4102 VRegArgs.push_back(VRegs);
4103
4104 if (Arg.hasSwiftErrorAttr()) {
4105 assert(VRegs.size() == 1 && "Too many vregs for Swift error");
4106 SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
4107 }
4108 }
4109
4110 if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) {
4111 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4112 F.getSubprogram(), &F.getEntryBlock());
4113 R << "unable to lower arguments: "
4114 << ore::NV("Prototype", F.getFunctionType());
4115 reportTranslationError(*MF, *TPC, *ORE, R);
4116 return false;
4117 }
4118
4119 // Need to visit defs before uses when translating instructions.
4120 GISelObserverWrapper WrapperObserver;
4121 if (EnableCSE && CSEInfo)
4122 WrapperObserver.addObserver(CSEInfo);
4123 {
4125#ifndef NDEBUG
4126 DILocationVerifier Verifier;
4127 WrapperObserver.addObserver(&Verifier);
4128#endif // ifndef NDEBUG
4129 RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);
4130 for (const BasicBlock *BB : RPOT) {
4131 MachineBasicBlock &MBB = getMBB(*BB);
4132 // Set the insertion point of all the following translations to
4133 // the end of this basic block.
4134 CurBuilder->setMBB(MBB);
4135 HasTailCall = false;
4136 for (const Instruction &Inst : *BB) {
4137 // If we translated a tail call in the last step, then we know
4138 // everything after the call is either a return, or something that is
4139 // handled by the call itself. (E.g. a lifetime marker or assume
4140 // intrinsic.) In this case, we should stop translating the block and
4141 // move on.
4142 if (HasTailCall)
4143 break;
4144#ifndef NDEBUG
4145 Verifier.setCurrentInst(&Inst);
4146#endif // ifndef NDEBUG
4147
4148 // Translate any debug-info attached to the instruction.
4149 translateDbgInfo(Inst, *CurBuilder);
4150
4151 if (translate(Inst))
4152 continue;
4153
4154 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4155 Inst.getDebugLoc(), BB);
4156 R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
4157
4158 if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
4159 std::string InstStrStorage;
4160 raw_string_ostream InstStr(InstStrStorage);
4161 InstStr << Inst;
4162
4163 R << ": '" << InstStrStorage << "'";
4164 }
4165
4166 reportTranslationError(*MF, *TPC, *ORE, R);
4167 return false;
4168 }
4169
4170 if (!finalizeBasicBlock(*BB, MBB)) {
4171 OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
4172 BB->getTerminator()->getDebugLoc(), BB);
4173 R << "unable to translate basic block";
4174 reportTranslationError(*MF, *TPC, *ORE, R);
4175 return false;
4176 }
4177 }
4178#ifndef NDEBUG
4179 WrapperObserver.removeObserver(&Verifier);
4180#endif
4181 }
4182
4183 finishPendingPhis();
4184
4185 SwiftError.propagateVRegs();
4186
4187 // Merge the argument lowering and constants block with its single
4188 // successor, the LLVM-IR entry block. We want the basic block to
4189 // be maximal.
4190 assert(EntryBB->succ_size() == 1 &&
4191 "Custom BB used for lowering should have only one successor");
4192 // Get the successor of the current entry block.
4193 MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
4194 assert(NewEntryBB.pred_size() == 1 &&
4195 "LLVM-IR entry block has a predecessor!?");
4196 // Move all the instruction from the current entry block to the
4197 // new entry block.
4198 NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
4199 EntryBB->end());
4200
4201 // Update the live-in information for the new entry block.
4202 for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
4203 NewEntryBB.addLiveIn(LiveIn);
4204 NewEntryBB.sortUniqueLiveIns();
4205
4206 // Get rid of the now empty basic block.
4207 EntryBB->removeSuccessor(&NewEntryBB);
4208 MF->remove(EntryBB);
4209 MF->deleteMachineBasicBlock(EntryBB);
4210
4211 assert(&MF->front() == &NewEntryBB &&
4212 "New entry wasn't next in the list of basic block!");
4213
4214 // Initialize stack protector information.
4215 StackProtector &SP = getAnalysis<StackProtector>();
4217
4218 return false;
4219}
unsigned SubReg
#define Success
aarch64 promote const
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Provides analysis for continuously CSEing during GISel passes.
This file implements a version of MachineIRBuilder which CSEs insts within a MachineBasicBlock.
This file describes how to lower LLVM calls to machine code calls.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(...)
Definition: Debug.h:106
uint64_t Addr
std::string Name
uint64_t Size
This contains common code to allow clients to notify changes to machine instr.
const HexagonInstrInfo * TII
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB)
Returns true if a BasicBlock BB within a variadic function contains a variadic musttail call.
static bool containsBF16Type(const User &U)
static unsigned getConvOpcode(Intrinsic::ID ID)
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
static unsigned getConstrainedOpcode(Intrinsic::ID ID)
IRTranslator LLVM IR MI
#define DEBUG_TYPE
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
static bool isValInBlock(const Value *V, const BasicBlock *BB)
static bool isSwiftError(const Value *V)
This file declares the IRTranslator pass.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file describes how to lower LLVM inline asm to machine code INLINEASM.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:80
Implement a low-level type suitable for MachineInstr level instruction selection.
Implement a low-level type suitable for MachineInstr level instruction selection.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
This file contains the declarations for metadata subclasses.
uint64_t High
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:57
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
verify safepoint Safepoint IR Verifier
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
Target-Independent Code Generator Pass Configuration Options pass.
Value * RHS
Value * LHS
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:78
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition: APInt.cpp:1007
an instruction to allocate memory on the stack
Definition: Instructions.h:63
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:149
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:124
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:99
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:117
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:95
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:157
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:168
iterator begin() const
Definition: ArrayRef.h:156
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:163
An immutable pass that tracks lazily created AssumptionCache objects.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:501
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:704
@ Add
*p = old + v
Definition: Instructions.h:720
@ FAdd
*p = old + v
Definition: Instructions.h:741
@ USubCond
Subtract only if no unsigned overflow.
Definition: Instructions.h:764
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:734
@ Or
*p = old | v
Definition: Instructions.h:728
@ Sub
*p = old - v
Definition: Instructions.h:722
@ And
*p = old & v
Definition: Instructions.h:724
@ Xor
*p = old ^ v
Definition: Instructions.h:730
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition: Instructions.h:768
@ FSub
*p = old - v
Definition: Instructions.h:744
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:756
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:732
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:738
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:752
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:736
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:748
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:760
@ Nand
*p = ~(old & v)
Definition: Instructions.h:726
Attribute getFnAttr(Attribute::AttrKind Kind) const
Return the attribute object that exists for the function.
Definition: Attributes.h:892
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:392
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
unsigned getNumber() const
Definition: BasicBlock.h:104
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
Definition: BasicBlock.h:661
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
Definition: BasicBlock.cpp:374
InstListType::const_iterator const_iterator
Definition: BasicBlock.h:178
InstListType::const_iterator getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:387
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:220
const Instruction & back() const
Definition: BasicBlock.h:476
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:292
Legacy analysis pass which computes BlockFrequencyInfo.
Conditional or Unconditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
Value * getCondition() const
Legacy analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1112
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1408
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2053
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1341
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1261
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2029
Value * getCalledOperand() const
Definition: InstrTypes.h:1334
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1286
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1267
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:1937
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1277
unsigned arg_size() const
Definition: InstrTypes.h:1284
AttributeList getAttributes() const
Return the attributes for this call.
Definition: InstrTypes.h:1417
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
bool checkReturnTypeForCallConv(MachineFunction &MF) const
Toplevel function to check the return type based on the target calling convention.
virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, ArrayRef< ArrayRef< Register > > VRegs, FunctionLoweringInfo &FLI) const
This hook must be implemented to lower the incoming (formal) arguments, described by VRegs,...
Definition: CallLowering.h:555
virtual bool enableBigEndian() const
For targets which want to use big-endian can enable it with enableBigEndian() hook.
Definition: CallLowering.h:604
virtual bool supportSwiftError() const
Definition: CallLowering.h:458
virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, ArrayRef< Register > VRegs, FunctionLoweringInfo &FLI, Register SwiftErrorVReg) const
This hook must be implemented to lower outgoing return values, described by Val, into the specified v...
Definition: CallLowering.h:523
virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info) const
This hook must be implemented to lower the given call instruction, including argument and return valu...
Definition: CallLowering.h:567
virtual bool fallBackToDAGISel(const MachineFunction &MF) const
Definition: CallLowering.h:541
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:661
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:673
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:690
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:702
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:703
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:696
@ ICMP_EQ
equal
Definition: InstrTypes.h:694
@ ICMP_NE
not equal
Definition: InstrTypes.h:695
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:699
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:675
bool isFPPredicate() const
Definition: InstrTypes.h:780
bool isIntPredicate() const
Definition: InstrTypes.h:781
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:866
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:208
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition: Constants.h:151
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:157
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:148
This is an important base class in LLVM.
Definition: Constant.h:42
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:420
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:373
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
unsigned getNonMetadataArgCount() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:364
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:709
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
Definition: DataLayout.cpp:878
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:457
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:617
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:421
Align getPointerABIAlignment(unsigned AS) const
Layout pointer alignment.
Definition: DataLayout.cpp:731
This represents the llvm.dbg.declare instruction.
Value * getAddress() const
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
Value * getValue(unsigned OpIdx=0) const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
A debug info location.
Definition: DebugLoc.h:33
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
An instruction for ordering other memory operations.
Definition: Instructions.h:424
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:460
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:449
Class to represent fixed width SIMD vectors.
Definition: DerivedTypes.h:563
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:791
BranchProbabilityInfo * BPI
void clear()
clear - Clear out all the function-specific state.
MachineBasicBlock * getMBB(const BasicBlock *BB) const
SmallVector< MachineBasicBlock * > MBBMap
A mapping from LLVM basic block number to their machine block.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:178
const BasicBlock & getEntryBlock() const
Definition: Function.h:809
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1874
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:704
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1048
const Function & getFunction() const
Definition: Function.h:171
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:256
bool hasOptNone() const
Do not optimize this function (-O0).
Definition: Function.h:701
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:369
The actual analysis pass wrapper.
Definition: CSEInfo.h:225
Simple wrapper that does the following.
Definition: CSEInfo.h:207
The CSE Analysis object.
Definition: CSEInfo.h:70
Abstract class that contains various methods for clients to notify about changes.
Simple wrapper observer that takes several observers, and calls each one for each event.
void removeObserver(GISelChangeObserver *O)
void addObserver(GISelChangeObserver *O)
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:568
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:530
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:279
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:657
bool isTailCall(const MachineInstr &MI) const override
This instruction compares its operands according to the predicate given to the constructor.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
IRTranslator(CodeGenOptLevel OptLevel=CodeGenOptLevel::None)
static char ID
Definition: IRTranslator.h:68
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Indirect Branch Instruction.
bool lowerInlineAsm(MachineIRBuilder &MIRBuilder, const CallBase &CB, std::function< ArrayRef< Register >(const Value &Val)> GetOrCreateVRegs) const
Lower the given inline asm call instruction GetOrCreateVRegs is a callback to materialize a register ...
This instruction inserts a struct field of array element value into an aggregate value.
iterator_range< simple_ilist< DbgRecord >::iterator > getDbgRecordRange() const
Return a range over the DbgRecords attached to this instruction.
Definition: Instruction.h:104
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:492
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:68
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:385
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:407
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1750
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:291
bool hasAllowReassoc() const LLVM_READONLY
Determine whether the allow-reassociation flag is set.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:55
Invoke instruction.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
Definition: LowLevelType.h:211
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelType.h:159
constexpr bool isVector() const
Definition: LowLevelType.h:148
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelType.h:57
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:190
constexpr bool isPointer() const
Definition: LowLevelType.h:149
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
Definition: LowLevelType.h:277
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelType.h:100
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
Definition: LowLevelType.h:177
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:176
Value * getPointerOperand()
Definition: Instructions.h:255
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:220
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:230
static LocationSize precise(uint64_t Value)
Context object for machine code objects.
Definition: MCContext.h:83
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:345
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:235
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
Metadata node.
Definition: Metadata.h:1073
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1549
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
unsigned pred_size() const
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
void setAddressTakenIRBlock(BasicBlock *BB)
Set this block to reflect that it corresponds to an IR-level basic block with a BlockAddress.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
void setHasMustTailInVarArgFunc(bool B)
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
ArrayRef< int > allocateShuffleMask(ArrayRef< int > Mask)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
void push_back(MachineBasicBlock *MBB)
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
void deleteMachineBasicBlock(MachineBasicBlock *MBB)
DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
Function & getFunction()
Return the LLVM function that this machine code represents.
void remove(iterator MBBI)
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Helper class to build MachineInstr.
MachineInstrBuilder buildFPTOUI_SAT(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_FPTOUI_SAT Src0.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildResetFPMode()
Build and insert G_RESET_FPMODE.
MachineInstrBuilder buildFPExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPEXT Op.
MachineInstrBuilder buildFPTOSI_SAT(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_FPTOSI_SAT Src0.
MachineInstrBuilder buildUCmp(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_UCMP Op0, Op1.
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
MachineInstrBuilder buildSCmp(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_SCMP Op0, Op1.
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, const SrcOp &Src2, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FMA Op0, Op1, Op2.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildInsertSubvector(const DstOp &Res, const SrcOp &Src0, const SrcOp &Src1, unsigned Index)
Build and insert Res = G_INSERT_SUBVECTOR Src0, Src1, Idx.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildVScale(const DstOp &Res, unsigned MinElts)
Build and insert Res = G_VSCALE MinElts.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildSetFPMode(const SrcOp &Src)
Build and insert G_SET_FPMODE Src.
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
MachineInstrBuilder buildBrCond(const SrcOp &Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
MachineInstrBuilder buildDynStackAlloc(const DstOp &Res, const SrcOp &Size, Align Alignment)
Build and insert Res = G_DYN_STACKALLOC Size, Align.
MachineInstrBuilder buildFIDbgValue(int FI, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in th...
MachineInstrBuilder buildResetFPEnv()
Build and insert G_RESET_FPENV.
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(const DstOp &OldValRes, const DstOp &SuccessRes, const SrcOp &Addr, const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr,...
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
const DebugLoc & getDebugLoc()
Get the current instruction's debug location.
MachineInstrBuilder buildTrap(bool Debug=false)
Build and insert G_TRAP or G_DEBUGTRAP.
MachineInstrBuilder buildFFrexp(const DstOp &Fract, const DstOp &Exp, const SrcOp &Src, std::optional< unsigned > Flags=std::nullopt)
Build and insert Fract, Exp = G_FFREXP Src.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildFSincos(const DstOp &Sin, const DstOp &Cos, const SrcOp &Src, std::optional< unsigned > Flags=std::nullopt)
Build and insert Sin, Cos = G_FSINCOS Src.
MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef< int > Mask)
Build and insert Res = G_SHUFFLE_VECTOR Src1, Src2, Mask.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildPrefetch(const SrcOp &Addr, unsigned RW, unsigned Locality, unsigned CacheType, MachineMemOperand &MMO)
Build and insert G_PREFETCH Addr, RW, Locality, CacheType.
MachineInstrBuilder buildExtractSubvector(const DstOp &Res, const SrcOp &Src, unsigned Index)
Build and insert Res = G_EXTRACT_SUBVECTOR Src, Idx0.
const DataLayout & getDataLayout() const
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Val)
Build and insert Res = G_SPLAT_VECTOR Val.
MachineInstrBuilder buildStepVector(const DstOp &Res, unsigned Step)
Build and insert Res = G_STEP_VECTOR Step.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
MachineInstrBuilder buildSetFPEnv(const SrcOp &Src)
Build and insert G_SET_FPENV Src.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:71
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
static uint32_t copyFlagsFromInstruction(const Instruction &I)
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:587
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
Representation for a specific memory location.
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
The optimization diagnostic interface.
Diagnostic information for missed-optimization remarks.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:686
Class to install both of the above.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
Definition: Register.h:110
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
This class represents the LLVM 'select' instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:384
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:519
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:132
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:175
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:181
size_t size() const
Definition: SmallVector.h:78
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:573
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
void initialize(const BasicBlock *BB, MachineBasicBlock *MBB, bool FunctionBasedInstrumentation)
Initialize the stack protector descriptor structure for a new basic block.
MachineBasicBlock * getSuccessMBB()
void resetPerBBState()
Reset state that changes when we handle different basic blocks.
void resetPerFunctionState()
Reset state that only changes when we switch functions.
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitStackProtector() const
Returns true if all fields of the stack protector descriptor are initialized implying that we should/...
bool shouldEmitFunctionBasedCheckStackProtector() const
bool shouldEmitSDCheck(const BasicBlock &BB) const
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
An instruction for storing to memory.
Definition: Instructions.h:292
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:147
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:144
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:596
Class to represent struct types.
Definition: DerivedTypes.h:218
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
Multiway switch.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
virtual unsigned getVaListSizeInBits(const DataLayout &DL) const
Returns the size of the platform's va_list object.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
const TargetMachine & getTargetMachine() const
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool fallBackToDAGISel(const Instruction &Inst) const
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
const Triple & getTargetTriple() const
TargetOptions Options
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
Target-Independent Code Generator Pass Configuration Options.
virtual std::unique_ptr< CSEConfigBase > getCSEConfig() const
Returns the CSEConfig object to use for the current optimization level.
virtual bool isGISelCSEEnabled() const
Check whether continuous CSE should be enabled in GISel passes.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const InlineAsmLowering * getInlineAsmLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const CallLowering * getCallLowering() const
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetLowering * getTargetLowering() const
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:655
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:310
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:303
static IntegerType * getInt32Ty(LLVMContext &C)
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:234
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
Value * getOperand(unsigned i) const
Definition: User.h:228
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:694
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
int getNumOccurrences() const
Definition: CommandLine.h:399
constexpr bool isZero() const
Definition: TypeSize.h:156
const ParentTy * getParent() const
Definition: ilist_node.h:32
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:353
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:87
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Key
PAL metadata keys.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:125
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
ID ArrayRef< Type * > Tys
Definition: Intrinsics.h:102
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:885
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Undef
Value of the register doesn't matter.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1612
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
@ CE
Windows NT (Windows on ARM)
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< PhiNode * > Phi
Definition: RDFGraph.h:390
NodeAddr< CodeNode * > Code
Definition: RDFGraph.h:388
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
@ Offset
Definition: DWP.cpp:480
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:256
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition: STLExtras.h:2448
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
gep_type_iterator gep_type_end(const User *GEP)
MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition: Local.h:242
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1746
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:420
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
Definition: Analysis.cpp:141
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1664
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
auto succ_size(const MachineBasicBlock *BB)
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
@ Global
Append to llvm.global_dtors.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:1168
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1978
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ FMul
Product of floats.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1841
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, const LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:764
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Pair of physical register and lane mask.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static bool canHandle(const Instruction *I, const TargetLibraryInfo &TLI)
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
struct PredInfoPair PredInfo