LLVM  10.0.0svn
IRTranslator.cpp
Go to the documentation of this file.
1 //===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the IRTranslator class.
10 //===----------------------------------------------------------------------===//
11 
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/ScopeExit.h"
16 #include "llvm/ADT/SmallSet.h"
17 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/CodeGen/Analysis.h"
39 #include "llvm/IR/BasicBlock.h"
40 #include "llvm/IR/CFG.h"
41 #include "llvm/IR/Constant.h"
42 #include "llvm/IR/Constants.h"
43 #include "llvm/IR/DataLayout.h"
44 #include "llvm/IR/DebugInfo.h"
45 #include "llvm/IR/DerivedTypes.h"
46 #include "llvm/IR/Function.h"
48 #include "llvm/IR/InlineAsm.h"
49 #include "llvm/IR/InstrTypes.h"
50 #include "llvm/IR/Instructions.h"
51 #include "llvm/IR/IntrinsicInst.h"
52 #include "llvm/IR/Intrinsics.h"
53 #include "llvm/IR/LLVMContext.h"
54 #include "llvm/IR/Metadata.h"
55 #include "llvm/IR/Type.h"
56 #include "llvm/IR/User.h"
57 #include "llvm/IR/Value.h"
58 #include "llvm/MC/MCContext.h"
59 #include "llvm/Pass.h"
60 #include "llvm/Support/Casting.h"
61 #include "llvm/Support/CodeGen.h"
62 #include "llvm/Support/Debug.h"
69 #include <algorithm>
70 #include <cassert>
71 #include <cstdint>
72 #include <iterator>
73 #include <string>
74 #include <utility>
75 #include <vector>
76 
77 #define DEBUG_TYPE "irtranslator"
78 
79 using namespace llvm;
80 
81 static cl::opt<bool>
82  EnableCSEInIRTranslator("enable-cse-in-irtranslator",
83  cl::desc("Should enable CSE in irtranslator"),
84  cl::Optional, cl::init(false));
85 char IRTranslator::ID = 0;
86 
87 INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
88  false, false)
91 INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
92  false, false)
93 
99 
100  // Print the function name explicitly if we don't have a debug location (which
101  // makes the diagnostic less useful) or if we're going to emit a raw error.
102  if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
103  R << (" (in function: " + MF.getName() + ")").str();
104 
105  if (TPC.isGlobalISelAbortEnabled())
106  report_fatal_error(R.getMsg());
107  else
108  ORE.emit(R);
109 }
110 
112 
113 #ifndef NDEBUG
114 namespace {
115 /// Verify that every instruction created has the same DILocation as the
116 /// instruction being translated.
117 class DILocationVerifier : public GISelChangeObserver {
118  const Instruction *CurrInst = nullptr;
119 
120 public:
121  DILocationVerifier() = default;
122  ~DILocationVerifier() = default;
123 
124  const Instruction *getCurrentInst() const { return CurrInst; }
125  void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
126 
127  void erasingInstr(MachineInstr &MI) override {}
128  void changingInstr(MachineInstr &MI) override {}
129  void changedInstr(MachineInstr &MI) override {}
130 
131  void createdInstr(MachineInstr &MI) override {
132  assert(getCurrentInst() && "Inserted instruction without a current MI");
133 
134  // Only print the check message if we're actually checking it.
135 #ifndef NDEBUG
136  LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
137  << " was copied to " << MI);
138 #endif
139  // We allow insts in the entry block to have a debug loc line of 0 because
140  // they could have originated from constants, and we don't want a jumpy
141  // debug experience.
142  assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
143  MI.getDebugLoc().getLine() == 0) &&
144  "Line info was not transferred to all instructions");
145  }
146 };
147 } // namespace
148 #endif // ifndef NDEBUG
149 
150 
157 }
158 
160 IRTranslator::allocateVRegs(const Value &Val) {
161  assert(!VMap.contains(Val) && "Value already allocated in VMap");
162  auto *Regs = VMap.getVRegs(Val);
163  auto *Offsets = VMap.getOffsets(Val);
164  SmallVector<LLT, 4> SplitTys;
165  computeValueLLTs(*DL, *Val.getType(), SplitTys,
166  Offsets->empty() ? Offsets : nullptr);
167  for (unsigned i = 0; i < SplitTys.size(); ++i)
168  Regs->push_back(0);
169  return *Regs;
170 }
171 
172 ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
173  auto VRegsIt = VMap.findVRegs(Val);
174  if (VRegsIt != VMap.vregs_end())
175  return *VRegsIt->second;
176 
177  if (Val.getType()->isVoidTy())
178  return *VMap.getVRegs(Val);
179 
180  // Create entry for this type.
181  auto *VRegs = VMap.getVRegs(Val);
182  auto *Offsets = VMap.getOffsets(Val);
183 
184  assert(Val.getType()->isSized() &&
185  "Don't know how to create an empty vreg");
186 
187  SmallVector<LLT, 4> SplitTys;
188  computeValueLLTs(*DL, *Val.getType(), SplitTys,
189  Offsets->empty() ? Offsets : nullptr);
190 
191  if (!isa<Constant>(Val)) {
192  for (auto Ty : SplitTys)
193  VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
194  return *VRegs;
195  }
196 
197  if (Val.getType()->isAggregateType()) {
198  // UndefValue, ConstantAggregateZero
199  auto &C = cast<Constant>(Val);
200  unsigned Idx = 0;
201  while (auto Elt = C.getAggregateElement(Idx++)) {
202  auto EltRegs = getOrCreateVRegs(*Elt);
203  llvm::copy(EltRegs, std::back_inserter(*VRegs));
204  }
205  } else {
206  assert(SplitTys.size() == 1 && "unexpectedly split LLT");
207  VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
208  bool Success = translate(cast<Constant>(Val), VRegs->front());
209  if (!Success) {
210  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
211  MF->getFunction().getSubprogram(),
212  &MF->getFunction().getEntryBlock());
213  R << "unable to translate constant: " << ore::NV("Type", Val.getType());
214  reportTranslationError(*MF, *TPC, *ORE, R);
215  return *VRegs;
216  }
217  }
218 
219  return *VRegs;
220 }
221 
222 int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
223  if (FrameIndices.find(&AI) != FrameIndices.end())
224  return FrameIndices[&AI];
225 
226  unsigned ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
227  unsigned Size =
228  ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
229 
230  // Always allocate at least one byte.
231  Size = std::max(Size, 1u);
232 
233  unsigned Alignment = AI.getAlignment();
234  if (!Alignment)
235  Alignment = DL->getABITypeAlignment(AI.getAllocatedType());
236 
237  int &FI = FrameIndices[&AI];
238  FI = MF->getFrameInfo().CreateStackObject(Size, Alignment, false, &AI);
239  return FI;
240 }
241 
242 unsigned IRTranslator::getMemOpAlignment(const Instruction &I) {
243  unsigned Alignment = 0;
244  Type *ValTy = nullptr;
245  if (const StoreInst *SI = dyn_cast<StoreInst>(&I)) {
246  Alignment = SI->getAlignment();
247  ValTy = SI->getValueOperand()->getType();
248  } else if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
249  Alignment = LI->getAlignment();
250  ValTy = LI->getType();
251  } else if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
252  // TODO(PR27168): This instruction has no alignment attribute, but unlike
253  // the default alignment for load/store, the default here is to assume
254  // it has NATURAL alignment, not DataLayout-specified alignment.
255  const DataLayout &DL = AI->getModule()->getDataLayout();
256  Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
257  ValTy = AI->getCompareOperand()->getType();
258  } else if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
259  // TODO(PR27168): This instruction has no alignment attribute, but unlike
260  // the default alignment for load/store, the default here is to assume
261  // it has NATURAL alignment, not DataLayout-specified alignment.
262  const DataLayout &DL = AI->getModule()->getDataLayout();
263  Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType());
264  ValTy = AI->getType();
265  } else {
266  OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
267  R << "unable to translate memop: " << ore::NV("Opcode", &I);
268  reportTranslationError(*MF, *TPC, *ORE, R);
269  return 1;
270  }
271 
272  return Alignment ? Alignment : DL->getABITypeAlignment(ValTy);
273 }
274 
275 MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
276  MachineBasicBlock *&MBB = BBToMBB[&BB];
277  assert(MBB && "BasicBlock was not encountered before");
278  return *MBB;
279 }
280 
281 void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
282  assert(NewPred && "new predecessor must be a real MachineBasicBlock");
283  MachinePreds[Edge].push_back(NewPred);
284 }
285 
286 bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
287  MachineIRBuilder &MIRBuilder) {
288  // Get or create a virtual register for each value.
289  // Unless the value is a Constant => loadimm cst?
290  // or inline constant each time?
291  // Creation of a virtual register needs to have a size.
292  Register Op0 = getOrCreateVReg(*U.getOperand(0));
293  Register Op1 = getOrCreateVReg(*U.getOperand(1));
294  Register Res = getOrCreateVReg(U);
295  uint16_t Flags = 0;
296  if (isa<Instruction>(U)) {
297  const Instruction &I = cast<Instruction>(U);
299  }
300 
301  MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
302  return true;
303 }
304 
305 bool IRTranslator::translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
306  // -0.0 - X --> G_FNEG
307  if (isa<Constant>(U.getOperand(0)) &&
309  Register Op1 = getOrCreateVReg(*U.getOperand(1));
310  Register Res = getOrCreateVReg(U);
311  uint16_t Flags = 0;
312  if (isa<Instruction>(U)) {
313  const Instruction &I = cast<Instruction>(U);
315  }
316  // Negate the last operand of the FSUB
317  MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op1}, Flags);
318  return true;
319  }
320  return translateBinaryOp(TargetOpcode::G_FSUB, U, MIRBuilder);
321 }
322 
323 bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
324  Register Op0 = getOrCreateVReg(*U.getOperand(0));
325  Register Res = getOrCreateVReg(U);
326  uint16_t Flags = 0;
327  if (isa<Instruction>(U)) {
328  const Instruction &I = cast<Instruction>(U);
330  }
331  MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op0}, Flags);
332  return true;
333 }
334 
335 bool IRTranslator::translateCompare(const User &U,
336  MachineIRBuilder &MIRBuilder) {
337  const CmpInst *CI = dyn_cast<CmpInst>(&U);
338  Register Op0 = getOrCreateVReg(*U.getOperand(0));
339  Register Op1 = getOrCreateVReg(*U.getOperand(1));
340  Register Res = getOrCreateVReg(U);
341  CmpInst::Predicate Pred =
342  CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
343  cast<ConstantExpr>(U).getPredicate());
344  if (CmpInst::isIntPredicate(Pred))
345  MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
346  else if (Pred == CmpInst::FCMP_FALSE)
347  MIRBuilder.buildCopy(
348  Res, getOrCreateVReg(*Constant::getNullValue(CI->getType())));
349  else if (Pred == CmpInst::FCMP_TRUE)
350  MIRBuilder.buildCopy(
351  Res, getOrCreateVReg(*Constant::getAllOnesValue(CI->getType())));
352  else {
353  MIRBuilder.buildInstr(TargetOpcode::G_FCMP, {Res}, {Pred, Op0, Op1},
355  }
356 
357  return true;
358 }
359 
360 bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
361  const ReturnInst &RI = cast<ReturnInst>(U);
362  const Value *Ret = RI.getReturnValue();
363  if (Ret && DL->getTypeStoreSize(Ret->getType()) == 0)
364  Ret = nullptr;
365 
366  ArrayRef<Register> VRegs;
367  if (Ret)
368  VRegs = getOrCreateVRegs(*Ret);
369 
370  Register SwiftErrorVReg = 0;
371  if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
372  SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
373  &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
374  }
375 
376  // The target may mess up with the insertion point, but
377  // this is not important as a return is the last instruction
378  // of the block anyway.
379  return CLI->lowerReturn(MIRBuilder, Ret, VRegs, SwiftErrorVReg);
380 }
381 
382 bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
383  const BranchInst &BrInst = cast<BranchInst>(U);
384  unsigned Succ = 0;
385  if (!BrInst.isUnconditional()) {
386  // We want a G_BRCOND to the true BB followed by an unconditional branch.
387  Register Tst = getOrCreateVReg(*BrInst.getCondition());
388  const BasicBlock &TrueTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ++));
389  MachineBasicBlock &TrueBB = getMBB(TrueTgt);
390  MIRBuilder.buildBrCond(Tst, TrueBB);
391  }
392 
393  const BasicBlock &BrTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ));
394  MachineBasicBlock &TgtBB = getMBB(BrTgt);
395  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
396 
397  // If the unconditional target is the layout successor, fallthrough.
398  if (!CurBB.isLayoutSuccessor(&TgtBB))
399  MIRBuilder.buildBr(TgtBB);
400 
401  // Link successors.
402  for (const BasicBlock *Succ : successors(&BrInst))
403  CurBB.addSuccessor(&getMBB(*Succ));
404  return true;
405 }
406 
407 void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
408  MachineBasicBlock *Dst,
409  BranchProbability Prob) {
410  if (!FuncInfo.BPI) {
411  Src->addSuccessorWithoutProb(Dst);
412  return;
413  }
414  if (Prob.isUnknown())
415  Prob = getEdgeProbability(Src, Dst);
416  Src->addSuccessor(Dst, Prob);
417 }
418 
420 IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
421  const MachineBasicBlock *Dst) const {
422  const BasicBlock *SrcBB = Src->getBasicBlock();
423  const BasicBlock *DstBB = Dst->getBasicBlock();
424  if (!FuncInfo.BPI) {
425  // If BPI is not available, set the default probability as 1 / N, where N is
426  // the number of successors.
427  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
428  return BranchProbability(1, SuccSize);
429  }
430  return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
431 }
432 
433 bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
434  using namespace SwitchCG;
435  // Extract cases from the switch.
436  const SwitchInst &SI = cast<SwitchInst>(U);
437  BranchProbabilityInfo *BPI = FuncInfo.BPI;
438  CaseClusterVector Clusters;
439  Clusters.reserve(SI.getNumCases());
440  for (auto &I : SI.cases()) {
441  MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
442  assert(Succ && "Could not find successor mbb in mapping");
443  const ConstantInt *CaseVal = I.getCaseValue();
444  BranchProbability Prob =
445  BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
446  : BranchProbability(1, SI.getNumCases() + 1);
447  Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
448  }
449 
450  MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
451 
452  // Cluster adjacent cases with the same destination. We do this at all
453  // optimization levels because it's cheap to do and will make codegen faster
454  // if there are many clusters.
455  sortAndRangeify(Clusters);
456 
457  MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
458 
459  // If there is only the default destination, jump there directly.
460  if (Clusters.empty()) {
461  SwitchMBB->addSuccessor(DefaultMBB);
462  if (DefaultMBB != SwitchMBB->getNextNode())
463  MIB.buildBr(*DefaultMBB);
464  return true;
465  }
466 
467  SL->findJumpTables(Clusters, &SI, DefaultMBB);
468 
469  LLVM_DEBUG({
470  dbgs() << "Case clusters: ";
471  for (const CaseCluster &C : Clusters) {
472  if (C.Kind == CC_JumpTable)
473  dbgs() << "JT:";
474  if (C.Kind == CC_BitTests)
475  dbgs() << "BT:";
476 
477  C.Low->getValue().print(dbgs(), true);
478  if (C.Low != C.High) {
479  dbgs() << '-';
480  C.High->getValue().print(dbgs(), true);
481  }
482  dbgs() << ' ';
483  }
484  dbgs() << '\n';
485  });
486 
487  assert(!Clusters.empty());
488  SwitchWorkList WorkList;
489  CaseClusterIt First = Clusters.begin();
490  CaseClusterIt Last = Clusters.end() - 1;
491  auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
492  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
493 
494  // FIXME: At the moment we don't do any splitting optimizations here like
495  // SelectionDAG does, so this worklist only has one entry.
496  while (!WorkList.empty()) {
497  SwitchWorkListItem W = WorkList.back();
498  WorkList.pop_back();
499  if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
500  return false;
501  }
502  return true;
503 }
504 
505 void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
506  MachineBasicBlock *MBB) {
507  // Emit the code for the jump table
508  assert(JT.Reg != -1U && "Should lower JT Header first!");
509  MachineIRBuilder MIB(*MBB->getParent());
510  MIB.setMBB(*MBB);
511  MIB.setDebugLoc(CurBuilder->getDebugLoc());
512 
513  Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
514  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
515 
516  auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
517  MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
518 }
519 
520 bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
522  MachineBasicBlock *HeaderBB) {
523  MachineIRBuilder MIB(*HeaderBB->getParent());
524  MIB.setMBB(*HeaderBB);
525  MIB.setDebugLoc(CurBuilder->getDebugLoc());
526 
527  const Value &SValue = *JTH.SValue;
528  // Subtract the lowest switch case value from the value being switched on.
529  const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
530  Register SwitchOpReg = getOrCreateVReg(SValue);
531  auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
532  auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
533 
534  // This value may be smaller or larger than the target's pointer type, and
535  // therefore require extension or truncating.
536  Type *PtrIRTy = SValue.getType()->getPointerTo();
537  const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
538  Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
539 
540  JT.Reg = Sub.getReg(0);
541 
542  if (JTH.OmitRangeCheck) {
543  if (JT.MBB != HeaderBB->getNextNode())
544  MIB.buildBr(*JT.MBB);
545  return true;
546  }
547 
548  // Emit the range check for the jump table, and branch to the default block
549  // for the switch statement if the value being switched on exceeds the
550  // largest case in the switch.
551  auto Cst = getOrCreateVReg(
552  *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
553  Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
554  auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
555 
556  auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
557 
558  // Avoid emitting unnecessary branches to the next block.
559  if (JT.MBB != HeaderBB->getNextNode())
560  BrCond = MIB.buildBr(*JT.MBB);
561  return true;
562 }
563 
564 void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
565  MachineBasicBlock *SwitchBB,
566  MachineIRBuilder &MIB) {
567  Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
568  Register Cond;
569  DebugLoc OldDbgLoc = MIB.getDebugLoc();
570  MIB.setDebugLoc(CB.DbgLoc);
571  MIB.setMBB(*CB.ThisBB);
572 
573  if (CB.PredInfo.NoCmp) {
574  // Branch or fall through to TrueBB.
575  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
576  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
577  CB.ThisBB);
579  if (CB.TrueBB != CB.ThisBB->getNextNode())
580  MIB.buildBr(*CB.TrueBB);
581  MIB.setDebugLoc(OldDbgLoc);
582  return;
583  }
584 
585  const LLT i1Ty = LLT::scalar(1);
586  // Build the compare.
587  if (!CB.CmpMHS) {
588  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
589  Cond = MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
590  } else {
592  "Can only handle ULE ranges");
593 
594  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
595  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
596 
597  Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
598  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
599  Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
600  Cond =
601  MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, CmpOpReg, CondRHS).getReg(0);
602  } else {
603  const LLT &CmpTy = MRI->getType(CmpOpReg);
604  auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
605  auto Diff = MIB.buildConstant(CmpTy, High - Low);
606  Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
607  }
608  }
609 
610  // Update successor info
611  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
612 
613  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
614  CB.ThisBB);
615 
616  // TrueBB and FalseBB are always different unless the incoming IR is
617  // degenerate. This only happens when running llc on weird IR.
618  if (CB.TrueBB != CB.FalseBB)
619  addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
621 
622  // if (SwitchBB->getBasicBlock() != CB.FalseBB->getBasicBlock())
623  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
624  CB.ThisBB);
625 
626  // If the lhs block is the next block, invert the condition so that we can
627  // fall through to the lhs instead of the rhs block.
628  if (CB.TrueBB == CB.ThisBB->getNextNode()) {
629  std::swap(CB.TrueBB, CB.FalseBB);
630  auto True = MIB.buildConstant(i1Ty, 1);
631  Cond = MIB.buildInstr(TargetOpcode::G_XOR, {i1Ty}, {Cond, True}, None)
632  .getReg(0);
633  }
634 
635  MIB.buildBrCond(Cond, *CB.TrueBB);
636  MIB.buildBr(*CB.FalseBB);
637  MIB.setDebugLoc(OldDbgLoc);
638 }
639 
640 bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
641  MachineBasicBlock *SwitchMBB,
642  MachineBasicBlock *CurMBB,
643  MachineBasicBlock *DefaultMBB,
644  MachineIRBuilder &MIB,
646  BranchProbability UnhandledProbs,
648  MachineBasicBlock *Fallthrough,
649  bool FallthroughUnreachable) {
650  using namespace SwitchCG;
651  MachineFunction *CurMF = SwitchMBB->getParent();
652  // FIXME: Optimize away range check based on pivot comparisons.
653  JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
654  SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
655  BranchProbability DefaultProb = W.DefaultProb;
656 
657  // The jump block hasn't been inserted yet; insert it here.
658  MachineBasicBlock *JumpMBB = JT->MBB;
659  CurMF->insert(BBI, JumpMBB);
660 
661  // Since the jump table block is separate from the switch block, we need
662  // to keep track of it as a machine predecessor to the default block,
663  // otherwise we lose the phi edges.
664  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
665  CurMBB);
666  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
667  JumpMBB);
668 
669  auto JumpProb = I->Prob;
670  auto FallthroughProb = UnhandledProbs;
671 
672  // If the default statement is a target of the jump table, we evenly
673  // distribute the default probability to successors of CurMBB. Also
674  // update the probability on the edge from JumpMBB to Fallthrough.
676  SE = JumpMBB->succ_end();
677  SI != SE; ++SI) {
678  if (*SI == DefaultMBB) {
679  JumpProb += DefaultProb / 2;
680  FallthroughProb -= DefaultProb / 2;
681  JumpMBB->setSuccProbability(SI, DefaultProb / 2);
682  JumpMBB->normalizeSuccProbs();
683  } else {
684  // Also record edges from the jump table block to it's successors.
685  addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
686  JumpMBB);
687  }
688  }
689 
690  // Skip the range check if the fallthrough block is unreachable.
691  if (FallthroughUnreachable)
692  JTH->OmitRangeCheck = true;
693 
694  if (!JTH->OmitRangeCheck)
695  addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
696  addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
697  CurMBB->normalizeSuccProbs();
698 
699  // The jump table header will be inserted in our current block, do the
700  // range check, and fall through to our fallthrough block.
701  JTH->HeaderBB = CurMBB;
702  JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
703 
704  // If we're in the right place, emit the jump table header right now.
705  if (CurMBB == SwitchMBB) {
706  if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
707  return false;
708  JTH->Emitted = true;
709  }
710  return true;
711 }
712 bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
713  Value *Cond,
714  MachineBasicBlock *Fallthrough,
715  bool FallthroughUnreachable,
716  BranchProbability UnhandledProbs,
717  MachineBasicBlock *CurMBB,
718  MachineIRBuilder &MIB,
719  MachineBasicBlock *SwitchMBB) {
720  using namespace SwitchCG;
721  const Value *RHS, *LHS, *MHS;
722  CmpInst::Predicate Pred;
723  if (I->Low == I->High) {
724  // Check Cond == I->Low.
725  Pred = CmpInst::ICMP_EQ;
726  LHS = Cond;
727  RHS = I->Low;
728  MHS = nullptr;
729  } else {
730  // Check I->Low <= Cond <= I->High.
731  Pred = CmpInst::ICMP_ULE;
732  LHS = I->Low;
733  MHS = Cond;
734  RHS = I->High;
735  }
736 
737  // If Fallthrough is unreachable, fold away the comparison.
738  // The false probability is the sum of all unhandled cases.
739  CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
740  CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
741 
742  emitSwitchCase(CB, SwitchMBB, MIB);
743  return true;
744 }
745 
746 bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
747  Value *Cond,
748  MachineBasicBlock *SwitchMBB,
749  MachineBasicBlock *DefaultMBB,
750  MachineIRBuilder &MIB) {
751  using namespace SwitchCG;
752  MachineFunction *CurMF = FuncInfo.MF;
753  MachineBasicBlock *NextMBB = nullptr;
755  if (++BBI != FuncInfo.MF->end())
756  NextMBB = &*BBI;
757 
758  if (EnableOpts) {
759  // Here, we order cases by probability so the most likely case will be
760  // checked first. However, two clusters can have the same probability in
761  // which case their relative ordering is non-deterministic. So we use Low
762  // as a tie-breaker as clusters are guaranteed to never overlap.
764  [](const CaseCluster &a, const CaseCluster &b) {
765  return a.Prob != b.Prob
766  ? a.Prob > b.Prob
767  : a.Low->getValue().slt(b.Low->getValue());
768  });
769 
770  // Rearrange the case blocks so that the last one falls through if possible
771  // without changing the order of probabilities.
772  for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
773  --I;
774  if (I->Prob > W.LastCluster->Prob)
775  break;
776  if (I->Kind == CC_Range && I->MBB == NextMBB) {
777  std::swap(*I, *W.LastCluster);
778  break;
779  }
780  }
781  }
782 
783  // Compute total probability.
784  BranchProbability DefaultProb = W.DefaultProb;
785  BranchProbability UnhandledProbs = DefaultProb;
786  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
787  UnhandledProbs += I->Prob;
788 
789  MachineBasicBlock *CurMBB = W.MBB;
790  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
791  bool FallthroughUnreachable = false;
792  MachineBasicBlock *Fallthrough;
793  if (I == W.LastCluster) {
794  // For the last cluster, fall through to the default destination.
795  Fallthrough = DefaultMBB;
796  FallthroughUnreachable = isa<UnreachableInst>(
797  DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
798  } else {
799  Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
800  CurMF->insert(BBI, Fallthrough);
801  }
802  UnhandledProbs -= I->Prob;
803 
804  switch (I->Kind) {
805  case CC_BitTests: {
806  LLVM_DEBUG(dbgs() << "Switch to bit test optimization unimplemented");
807  return false; // Bit tests currently unimplemented.
808  }
809  case CC_JumpTable: {
810  if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
811  UnhandledProbs, I, Fallthrough,
812  FallthroughUnreachable)) {
813  LLVM_DEBUG(dbgs() << "Failed to lower jump table");
814  return false;
815  }
816  break;
817  }
818  case CC_Range: {
819  if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
820  FallthroughUnreachable, UnhandledProbs,
821  CurMBB, MIB, SwitchMBB)) {
822  LLVM_DEBUG(dbgs() << "Failed to lower switch range");
823  return false;
824  }
825  break;
826  }
827  }
828  CurMBB = Fallthrough;
829  }
830 
831  return true;
832 }
833 
834 bool IRTranslator::translateIndirectBr(const User &U,
835  MachineIRBuilder &MIRBuilder) {
836  const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
837 
838  const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
839  MIRBuilder.buildBrIndirect(Tgt);
840 
841  // Link successors.
842  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
843  for (const BasicBlock *Succ : successors(&BrInst))
844  CurBB.addSuccessor(&getMBB(*Succ));
845 
846  return true;
847 }
848 
849 static bool isSwiftError(const Value *V) {
850  if (auto Arg = dyn_cast<Argument>(V))
851  return Arg->hasSwiftErrorAttr();
852  if (auto AI = dyn_cast<AllocaInst>(V))
853  return AI->isSwiftError();
854  return false;
855 }
856 
857 bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
858  const LoadInst &LI = cast<LoadInst>(U);
859 
860  auto Flags = LI.isVolatile() ? MachineMemOperand::MOVolatile
862  Flags |= MachineMemOperand::MOLoad;
863 
864  if (DL->getTypeStoreSize(LI.getType()) == 0)
865  return true;
866 
867  ArrayRef<Register> Regs = getOrCreateVRegs(LI);
868  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
869  Register Base = getOrCreateVReg(*LI.getPointerOperand());
870 
871  Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType());
872  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
873 
874  if (CLI->supportSwiftError() && isSwiftError(LI.getPointerOperand())) {
875  assert(Regs.size() == 1 && "swifterror should be single pointer");
876  Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(),
877  LI.getPointerOperand());
878  MIRBuilder.buildCopy(Regs[0], VReg);
879  return true;
880  }
881 
882  const MDNode *Ranges =
883  Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
884  for (unsigned i = 0; i < Regs.size(); ++i) {
885  Register Addr;
886  MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
887 
888  MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
889  unsigned BaseAlign = getMemOpAlignment(LI);
890  auto MMO = MF->getMachineMemOperand(
891  Ptr, Flags, (MRI->getType(Regs[i]).getSizeInBits() + 7) / 8,
892  MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), Ranges,
893  LI.getSyncScopeID(), LI.getOrdering());
894  MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
895  }
896 
897  return true;
898 }
899 
900 bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
901  const StoreInst &SI = cast<StoreInst>(U);
902  auto Flags = SI.isVolatile() ? MachineMemOperand::MOVolatile
905 
906  if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
907  return true;
908 
909  ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
910  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
911  Register Base = getOrCreateVReg(*SI.getPointerOperand());
912 
913  Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType());
914  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
915 
916  if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
917  assert(Vals.size() == 1 && "swifterror should be single pointer");
918 
919  Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
920  SI.getPointerOperand());
921  MIRBuilder.buildCopy(VReg, Vals[0]);
922  return true;
923  }
924 
925  for (unsigned i = 0; i < Vals.size(); ++i) {
926  Register Addr;
927  MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
928 
929  MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
930  unsigned BaseAlign = getMemOpAlignment(SI);
931  auto MMO = MF->getMachineMemOperand(
932  Ptr, Flags, (MRI->getType(Vals[i]).getSizeInBits() + 7) / 8,
933  MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), nullptr,
934  SI.getSyncScopeID(), SI.getOrdering());
935  MIRBuilder.buildStore(Vals[i], Addr, *MMO);
936  }
937  return true;
938 }
939 
940 static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
941  const Value *Src = U.getOperand(0);
943 
944  // getIndexedOffsetInType is designed for GEPs, so the first index is the
945  // usual array element rather than looking into the actual aggregate.
946  SmallVector<Value *, 1> Indices;
947  Indices.push_back(ConstantInt::get(Int32Ty, 0));
948 
949  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
950  for (auto Idx : EVI->indices())
951  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
952  } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
953  for (auto Idx : IVI->indices())
954  Indices.push_back(ConstantInt::get(Int32Ty, Idx));
955  } else {
956  for (unsigned i = 1; i < U.getNumOperands(); ++i)
957  Indices.push_back(U.getOperand(i));
958  }
959 
960  return 8 * static_cast<uint64_t>(
961  DL.getIndexedOffsetInType(Src->getType(), Indices));
962 }
963 
964 bool IRTranslator::translateExtractValue(const User &U,
965  MachineIRBuilder &MIRBuilder) {
966  const Value *Src = U.getOperand(0);
967  uint64_t Offset = getOffsetFromIndices(U, *DL);
968  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
969  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
970  unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
971  auto &DstRegs = allocateVRegs(U);
972 
973  for (unsigned i = 0; i < DstRegs.size(); ++i)
974  DstRegs[i] = SrcRegs[Idx++];
975 
976  return true;
977 }
978 
979 bool IRTranslator::translateInsertValue(const User &U,
980  MachineIRBuilder &MIRBuilder) {
981  const Value *Src = U.getOperand(0);
982  uint64_t Offset = getOffsetFromIndices(U, *DL);
983  auto &DstRegs = allocateVRegs(U);
984  ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
985  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
986  ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
987  auto InsertedIt = InsertedRegs.begin();
988 
989  for (unsigned i = 0; i < DstRegs.size(); ++i) {
990  if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
991  DstRegs[i] = *InsertedIt++;
992  else
993  DstRegs[i] = SrcRegs[i];
994  }
995 
996  return true;
997 }
998 
999 bool IRTranslator::translateSelect(const User &U,
1000  MachineIRBuilder &MIRBuilder) {
1001  Register Tst = getOrCreateVReg(*U.getOperand(0));
1002  ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1003  ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1004  ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1005 
1006  const SelectInst &SI = cast<SelectInst>(U);
1007  uint16_t Flags = 0;
1008  if (const CmpInst *Cmp = dyn_cast<CmpInst>(SI.getCondition()))
1010 
1011  for (unsigned i = 0; i < ResRegs.size(); ++i) {
1012  MIRBuilder.buildInstr(TargetOpcode::G_SELECT, {ResRegs[i]},
1013  {Tst, Op0Regs[i], Op1Regs[i]}, Flags);
1014  }
1015 
1016  return true;
1017 }
1018 
1019 bool IRTranslator::translateBitCast(const User &U,
1020  MachineIRBuilder &MIRBuilder) {
1021  // If we're bitcasting to the source type, we can reuse the source vreg.
1022  if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1023  getLLTForType(*U.getType(), *DL)) {
1024  Register SrcReg = getOrCreateVReg(*U.getOperand(0));
1025  auto &Regs = *VMap.getVRegs(U);
1026  // If we already assigned a vreg for this bitcast, we can't change that.
1027  // Emit a copy to satisfy the users we already emitted.
1028  if (!Regs.empty())
1029  MIRBuilder.buildCopy(Regs[0], SrcReg);
1030  else {
1031  Regs.push_back(SrcReg);
1032  VMap.getOffsets(U)->push_back(0);
1033  }
1034  return true;
1035  }
1036  return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1037 }
1038 
1039 bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1040  MachineIRBuilder &MIRBuilder) {
1041  Register Op = getOrCreateVReg(*U.getOperand(0));
1042  Register Res = getOrCreateVReg(U);
1043  MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1044  return true;
1045 }
1046 
1047 bool IRTranslator::translateGetElementPtr(const User &U,
1048  MachineIRBuilder &MIRBuilder) {
1049  // FIXME: support vector GEPs.
1050  if (U.getType()->isVectorTy())
1051  return false;
1052 
1053  Value &Op0 = *U.getOperand(0);
1054  Register BaseReg = getOrCreateVReg(Op0);
1055  Type *PtrIRTy = Op0.getType();
1056  LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1057  Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
1058  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1059 
1060  int64_t Offset = 0;
1061  for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1062  GTI != E; ++GTI) {
1063  const Value *Idx = GTI.getOperand();
1064  if (StructType *StTy = GTI.getStructTypeOrNull()) {
1065  unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1066  Offset += DL->getStructLayout(StTy)->getElementOffset(Field);
1067  continue;
1068  } else {
1069  uint64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
1070 
1071  // If this is a scalar constant or a splat vector of constants,
1072  // handle it quickly.
1073  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1074  Offset += ElementSize * CI->getSExtValue();
1075  continue;
1076  }
1077 
1078  if (Offset != 0) {
1079  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1080  auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1081  BaseReg =
1082  MIRBuilder.buildGEP(PtrTy, BaseReg, OffsetMIB.getReg(0)).getReg(0);
1083  Offset = 0;
1084  }
1085 
1086  Register IdxReg = getOrCreateVReg(*Idx);
1087  if (MRI->getType(IdxReg) != OffsetTy)
1088  IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
1089 
1090  // N = N + Idx * ElementSize;
1091  // Avoid doing it for ElementSize of 1.
1092  Register GepOffsetReg;
1093  if (ElementSize != 1) {
1094  auto ElementSizeMIB = MIRBuilder.buildConstant(
1095  getLLTForType(*OffsetIRTy, *DL), ElementSize);
1096  GepOffsetReg =
1097  MIRBuilder.buildMul(OffsetTy, ElementSizeMIB, IdxReg).getReg(0);
1098  } else
1099  GepOffsetReg = IdxReg;
1100 
1101  BaseReg = MIRBuilder.buildGEP(PtrTy, BaseReg, GepOffsetReg).getReg(0);
1102  }
1103  }
1104 
1105  if (Offset != 0) {
1106  auto OffsetMIB =
1107  MIRBuilder.buildConstant(getLLTForType(*OffsetIRTy, *DL), Offset);
1108  MIRBuilder.buildGEP(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
1109  return true;
1110  }
1111 
1112  MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1113  return true;
1114 }
1115 
1116 bool IRTranslator::translateMemFunc(const CallInst &CI,
1117  MachineIRBuilder &MIRBuilder,
1118  Intrinsic::ID ID) {
1119 
1120  // If the source is undef, then just emit a nop.
1121  if (isa<UndefValue>(CI.getArgOperand(1)))
1122  return true;
1123 
1124  ArrayRef<Register> Res;
1125  auto ICall = MIRBuilder.buildIntrinsic(ID, Res, true);
1126  for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI)
1127  ICall.addUse(getOrCreateVReg(**AI));
1128 
1129  unsigned DstAlign = 0, SrcAlign = 0;
1130  unsigned IsVol =
1131  cast<ConstantInt>(CI.getArgOperand(CI.getNumArgOperands() - 1))
1132  ->getZExtValue();
1133 
1134  if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1135  DstAlign = std::max<unsigned>(MCI->getDestAlignment(), 1);
1136  SrcAlign = std::max<unsigned>(MCI->getSourceAlignment(), 1);
1137  } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1138  DstAlign = std::max<unsigned>(MMI->getDestAlignment(), 1);
1139  SrcAlign = std::max<unsigned>(MMI->getSourceAlignment(), 1);
1140  } else {
1141  auto *MSI = cast<MemSetInst>(&CI);
1142  DstAlign = std::max<unsigned>(MSI->getDestAlignment(), 1);
1143  }
1144 
1145  // Create mem operands to store the alignment and volatile info.
1147  ICall.addMemOperand(MF->getMachineMemOperand(
1149  MachineMemOperand::MOStore | VolFlag, 1, DstAlign));
1150  if (ID != Intrinsic::memset)
1151  ICall.addMemOperand(MF->getMachineMemOperand(
1153  MachineMemOperand::MOLoad | VolFlag, 1, SrcAlign));
1154 
1155  return true;
1156 }
1157 
1158 void IRTranslator::getStackGuard(Register DstReg,
1159  MachineIRBuilder &MIRBuilder) {
1160  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1161  MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1162  auto MIB = MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD);
1163  MIB.addDef(DstReg);
1164 
1165  auto &TLI = *MF->getSubtarget().getTargetLowering();
1166  Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
1167  if (!Global)
1168  return;
1169 
1170  MachinePointerInfo MPInfo(Global);
1174  MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
1175  DL->getPointerABIAlignment(0));
1176  MIB.setMemRefs({MemRef});
1177 }
1178 
1179 bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1180  MachineIRBuilder &MIRBuilder) {
1181  ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1182  MIRBuilder.buildInstr(Op)
1183  .addDef(ResRegs[0])
1184  .addDef(ResRegs[1])
1185  .addUse(getOrCreateVReg(*CI.getOperand(0)))
1186  .addUse(getOrCreateVReg(*CI.getOperand(1)));
1187 
1188  return true;
1189 }
1190 
1191 unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1192  switch (ID) {
1193  default:
1194  break;
1195  case Intrinsic::bswap:
1196  return TargetOpcode::G_BSWAP;
1197  case Intrinsic::ceil:
1198  return TargetOpcode::G_FCEIL;
1199  case Intrinsic::cos:
1200  return TargetOpcode::G_FCOS;
1201  case Intrinsic::ctpop:
1202  return TargetOpcode::G_CTPOP;
1203  case Intrinsic::exp:
1204  return TargetOpcode::G_FEXP;
1205  case Intrinsic::exp2:
1206  return TargetOpcode::G_FEXP2;
1207  case Intrinsic::fabs:
1208  return TargetOpcode::G_FABS;
1209  case Intrinsic::copysign:
1210  return TargetOpcode::G_FCOPYSIGN;
1211  case Intrinsic::minnum:
1212  return TargetOpcode::G_FMINNUM;
1213  case Intrinsic::maxnum:
1214  return TargetOpcode::G_FMAXNUM;
1215  case Intrinsic::minimum:
1216  return TargetOpcode::G_FMINIMUM;
1217  case Intrinsic::maximum:
1218  return TargetOpcode::G_FMAXIMUM;
1219  case Intrinsic::canonicalize:
1220  return TargetOpcode::G_FCANONICALIZE;
1221  case Intrinsic::floor:
1222  return TargetOpcode::G_FFLOOR;
1223  case Intrinsic::fma:
1224  return TargetOpcode::G_FMA;
1225  case Intrinsic::log:
1226  return TargetOpcode::G_FLOG;
1227  case Intrinsic::log2:
1228  return TargetOpcode::G_FLOG2;
1229  case Intrinsic::log10:
1230  return TargetOpcode::G_FLOG10;
1231  case Intrinsic::nearbyint:
1232  return TargetOpcode::G_FNEARBYINT;
1233  case Intrinsic::pow:
1234  return TargetOpcode::G_FPOW;
1235  case Intrinsic::rint:
1236  return TargetOpcode::G_FRINT;
1237  case Intrinsic::round:
1238  return TargetOpcode::G_INTRINSIC_ROUND;
1239  case Intrinsic::sin:
1240  return TargetOpcode::G_FSIN;
1241  case Intrinsic::sqrt:
1242  return TargetOpcode::G_FSQRT;
1243  case Intrinsic::trunc:
1244  return TargetOpcode::G_INTRINSIC_TRUNC;
1245  }
1246  return Intrinsic::not_intrinsic;
1247 }
1248 
1249 bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1250  Intrinsic::ID ID,
1251  MachineIRBuilder &MIRBuilder) {
1252 
1253  unsigned Op = getSimpleIntrinsicOpcode(ID);
1254 
1255  // Is this a simple intrinsic?
1256  if (Op == Intrinsic::not_intrinsic)
1257  return false;
1258 
1259  // Yes. Let's translate it.
1261  for (auto &Arg : CI.arg_operands())
1262  VRegs.push_back(getOrCreateVReg(*Arg));
1263 
1264  MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1266  return true;
1267 }
1268 
1269 bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
1270  MachineIRBuilder &MIRBuilder) {
1271 
1272  // If this is a simple intrinsic (that is, we just need to add a def of
1273  // a vreg, and uses for each arg operand, then translate it.
1274  if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
1275  return true;
1276 
1277  switch (ID) {
1278  default:
1279  break;
1280  case Intrinsic::lifetime_start:
1281  case Intrinsic::lifetime_end: {
1282  // No stack colouring in O0, discard region information.
1283  if (MF->getTarget().getOptLevel() == CodeGenOpt::None)
1284  return true;
1285 
1286  unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
1288 
1289  // Get the underlying objects for the location passed on the lifetime
1290  // marker.
1292  GetUnderlyingObjects(CI.getArgOperand(1), Allocas, *DL);
1293 
1294  // Iterate over each underlying object, creating lifetime markers for each
1295  // static alloca. Quit if we find a non-static alloca.
1296  for (const Value *V : Allocas) {
1297  const AllocaInst *AI = dyn_cast<AllocaInst>(V);
1298  if (!AI)
1299  continue;
1300 
1301  if (!AI->isStaticAlloca())
1302  return true;
1303 
1304  MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
1305  }
1306  return true;
1307  }
1308  case Intrinsic::dbg_declare: {
1309  const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
1310  assert(DI.getVariable() && "Missing variable");
1311 
1312  const Value *Address = DI.getAddress();
1313  if (!Address || isa<UndefValue>(Address)) {
1314  LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
1315  return true;
1316  }
1317 
1319  MIRBuilder.getDebugLoc()) &&
1320  "Expected inlined-at fields to agree");
1321  auto AI = dyn_cast<AllocaInst>(Address);
1322  if (AI && AI->isStaticAlloca()) {
1323  // Static allocas are tracked at the MF level, no need for DBG_VALUE
1324  // instructions (in fact, they get ignored if they *do* exist).
1325  MF->setVariableDbgInfo(DI.getVariable(), DI.getExpression(),
1326  getOrCreateFrameIndex(*AI), DI.getDebugLoc());
1327  } else {
1328  // A dbg.declare describes the address of a source variable, so lower it
1329  // into an indirect DBG_VALUE.
1330  MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
1331  DI.getVariable(), DI.getExpression());
1332  }
1333  return true;
1334  }
1335  case Intrinsic::dbg_label: {
1336  const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
1337  assert(DI.getLabel() && "Missing label");
1338 
1340  MIRBuilder.getDebugLoc()) &&
1341  "Expected inlined-at fields to agree");
1342 
1343  MIRBuilder.buildDbgLabel(DI.getLabel());
1344  return true;
1345  }
1346  case Intrinsic::vaend:
1347  // No target I know of cares about va_end. Certainly no in-tree target
1348  // does. Simplest intrinsic ever!
1349  return true;
1350  case Intrinsic::vastart: {
1351  auto &TLI = *MF->getSubtarget().getTargetLowering();
1352  Value *Ptr = CI.getArgOperand(0);
1353  unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
1354 
1355  // FIXME: Get alignment
1356  MIRBuilder.buildInstr(TargetOpcode::G_VASTART)
1357  .addUse(getOrCreateVReg(*Ptr))
1358  .addMemOperand(MF->getMachineMemOperand(
1359  MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 1));
1360  return true;
1361  }
1362  case Intrinsic::dbg_value: {
1363  // This form of DBG_VALUE is target-independent.
1364  const DbgValueInst &DI = cast<DbgValueInst>(CI);
1365  const Value *V = DI.getValue();
1367  MIRBuilder.getDebugLoc()) &&
1368  "Expected inlined-at fields to agree");
1369  if (!V) {
1370  // Currently the optimizer can produce this; insert an undef to
1371  // help debugging. Probably the optimizer should not do this.
1372  MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
1373  } else if (const auto *CI = dyn_cast<Constant>(V)) {
1374  MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
1375  } else {
1376  Register Reg = getOrCreateVReg(*V);
1377  // FIXME: This does not handle register-indirect values at offset 0. The
1378  // direct/indirect thing shouldn't really be handled by something as
1379  // implicit as reg+noreg vs reg+imm in the first palce, but it seems
1380  // pretty baked in right now.
1381  MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
1382  }
1383  return true;
1384  }
1385  case Intrinsic::uadd_with_overflow:
1386  return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
1387  case Intrinsic::sadd_with_overflow:
1388  return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
1389  case Intrinsic::usub_with_overflow:
1390  return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
1391  case Intrinsic::ssub_with_overflow:
1392  return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
1393  case Intrinsic::umul_with_overflow:
1394  return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
1395  case Intrinsic::smul_with_overflow:
1396  return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
1397  case Intrinsic::fmuladd: {
1398  const TargetMachine &TM = MF->getTarget();
1399  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1400  Register Dst = getOrCreateVReg(CI);
1401  Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
1402  Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
1403  Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
1405  TLI.isFMAFasterThanFMulAndFAdd(TLI.getValueType(*DL, CI.getType()))) {
1406  // TODO: Revisit this to see if we should move this part of the
1407  // lowering to the combiner.
1408  MIRBuilder.buildInstr(TargetOpcode::G_FMA, {Dst}, {Op0, Op1, Op2},
1410  } else {
1411  LLT Ty = getLLTForType(*CI.getType(), *DL);
1412  auto FMul = MIRBuilder.buildInstr(TargetOpcode::G_FMUL, {Ty}, {Op0, Op1},
1414  MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Dst}, {FMul, Op2},
1416  }
1417  return true;
1418  }
1419  case Intrinsic::memcpy:
1420  case Intrinsic::memmove:
1421  case Intrinsic::memset:
1422  return translateMemFunc(CI, MIRBuilder, ID);
1423  case Intrinsic::eh_typeid_for: {
1425  Register Reg = getOrCreateVReg(CI);
1426  unsigned TypeID = MF->getTypeIDFor(GV);
1427  MIRBuilder.buildConstant(Reg, TypeID);
1428  return true;
1429  }
1430  case Intrinsic::objectsize: {
1431  // If we don't know by now, we're never going to know.
1432  const ConstantInt *Min = cast<ConstantInt>(CI.getArgOperand(1));
1433 
1434  MIRBuilder.buildConstant(getOrCreateVReg(CI), Min->isZero() ? -1ULL : 0);
1435  return true;
1436  }
1437  case Intrinsic::is_constant:
1438  // If this wasn't constant-folded away by now, then it's not a
1439  // constant.
1440  MIRBuilder.buildConstant(getOrCreateVReg(CI), 0);
1441  return true;
1442  case Intrinsic::stackguard:
1443  getStackGuard(getOrCreateVReg(CI), MIRBuilder);
1444  return true;
1445  case Intrinsic::stackprotector: {
1446  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
1447  Register GuardVal = MRI->createGenericVirtualRegister(PtrTy);
1448  getStackGuard(GuardVal, MIRBuilder);
1449 
1450  AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
1451  int FI = getOrCreateFrameIndex(*Slot);
1452  MF->getFrameInfo().setStackProtectorIndex(FI);
1453 
1454  MIRBuilder.buildStore(
1455  GuardVal, getOrCreateVReg(*Slot),
1456  *MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
1459  PtrTy.getSizeInBits() / 8, 8));
1460  return true;
1461  }
1462  case Intrinsic::stacksave: {
1463  // Save the stack pointer to the location provided by the intrinsic.
1464  Register Reg = getOrCreateVReg(CI);
1465  Register StackPtr = MF->getSubtarget()
1466  .getTargetLowering()
1467  ->getStackPointerRegisterToSaveRestore();
1468 
1469  // If the target doesn't specify a stack pointer, then fall back.
1470  if (!StackPtr)
1471  return false;
1472 
1473  MIRBuilder.buildCopy(Reg, StackPtr);
1474  return true;
1475  }
1476  case Intrinsic::stackrestore: {
1477  // Restore the stack pointer from the location provided by the intrinsic.
1478  Register Reg = getOrCreateVReg(*CI.getArgOperand(0));
1479  Register StackPtr = MF->getSubtarget()
1480  .getTargetLowering()
1481  ->getStackPointerRegisterToSaveRestore();
1482 
1483  // If the target doesn't specify a stack pointer, then fall back.
1484  if (!StackPtr)
1485  return false;
1486 
1487  MIRBuilder.buildCopy(StackPtr, Reg);
1488  return true;
1489  }
1490  case Intrinsic::cttz:
1491  case Intrinsic::ctlz: {
1492  ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
1493  bool isTrailing = ID == Intrinsic::cttz;
1494  unsigned Opcode = isTrailing
1495  ? Cst->isZero() ? TargetOpcode::G_CTTZ
1496  : TargetOpcode::G_CTTZ_ZERO_UNDEF
1497  : Cst->isZero() ? TargetOpcode::G_CTLZ
1498  : TargetOpcode::G_CTLZ_ZERO_UNDEF;
1499  MIRBuilder.buildInstr(Opcode)
1500  .addDef(getOrCreateVReg(CI))
1501  .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
1502  return true;
1503  }
1504  case Intrinsic::invariant_start: {
1505  LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
1506  Register Undef = MRI->createGenericVirtualRegister(PtrTy);
1507  MIRBuilder.buildUndef(Undef);
1508  return true;
1509  }
1510  case Intrinsic::invariant_end:
1511  return true;
1512  case Intrinsic::assume:
1513  case Intrinsic::var_annotation:
1514  case Intrinsic::sideeffect:
1515  // Discard annotate attributes, assumptions, and artificial side-effects.
1516  return true;
1517  }
1518  return false;
1519 }
1520 
1521 bool IRTranslator::translateInlineAsm(const CallInst &CI,
1522  MachineIRBuilder &MIRBuilder) {
1523  const InlineAsm &IA = cast<InlineAsm>(*CI.getCalledValue());
1524  if (!IA.getConstraintString().empty())
1525  return false;
1526 
1527  unsigned ExtraInfo = 0;
1528  if (IA.hasSideEffects())
1529  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1530  if (IA.getDialect() == InlineAsm::AD_Intel)
1531  ExtraInfo |= InlineAsm::Extra_AsmDialect;
1532 
1534  .addExternalSymbol(IA.getAsmString().c_str())
1535  .addImm(ExtraInfo);
1536 
1537  return true;
1538 }
1539 
1540 bool IRTranslator::translateCallSite(const ImmutableCallSite &CS,
1541  MachineIRBuilder &MIRBuilder) {
1542  const Instruction &I = *CS.getInstruction();
1543  ArrayRef<Register> Res = getOrCreateVRegs(I);
1544 
1546  Register SwiftInVReg = 0;
1547  Register SwiftErrorVReg = 0;
1548  for (auto &Arg : CS.args()) {
1549  if (CLI->supportSwiftError() && isSwiftError(Arg)) {
1550  assert(SwiftInVReg == 0 && "Expected only one swift error argument");
1551  LLT Ty = getLLTForType(*Arg->getType(), *DL);
1552  SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
1553  MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
1554  &I, &MIRBuilder.getMBB(), Arg));
1555  Args.emplace_back(makeArrayRef(SwiftInVReg));
1556  SwiftErrorVReg =
1557  SwiftError.getOrCreateVRegDefAt(&I, &MIRBuilder.getMBB(), Arg);
1558  continue;
1559  }
1560  Args.push_back(getOrCreateVRegs(*Arg));
1561  }
1562 
1563  MF->getFrameInfo().setHasCalls(true);
1564  bool Success =
1565  CLI->lowerCall(MIRBuilder, CS, Res, Args, SwiftErrorVReg,
1566  [&]() { return getOrCreateVReg(*CS.getCalledValue()); });
1567 
1568  return Success;
1569 }
1570 
1571 bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
1572  const CallInst &CI = cast<CallInst>(U);
1573  auto TII = MF->getTarget().getIntrinsicInfo();
1574  const Function *F = CI.getCalledFunction();
1575 
1576  // FIXME: support Windows dllimport function calls.
1577  if (F && F->hasDLLImportStorageClass())
1578  return false;
1579 
1580  if (CI.isInlineAsm())
1581  return translateInlineAsm(CI, MIRBuilder);
1582 
1584  if (F && F->isIntrinsic()) {
1585  ID = F->getIntrinsicID();
1586  if (TII && ID == Intrinsic::not_intrinsic)
1587  ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
1588  }
1589 
1590  if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
1591  return translateCallSite(&CI, MIRBuilder);
1592 
1593  assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
1594 
1595  if (translateKnownIntrinsic(CI, ID, MIRBuilder))
1596  return true;
1597 
1598  ArrayRef<Register> ResultRegs;
1599  if (!CI.getType()->isVoidTy())
1600  ResultRegs = getOrCreateVRegs(CI);
1601 
1602  // Ignore the callsite attributes. Backend code is most likely not expecting
1603  // an intrinsic to sometimes have side effects and sometimes not.
1604  MachineInstrBuilder MIB =
1605  MIRBuilder.buildIntrinsic(ID, ResultRegs, !F->doesNotAccessMemory());
1606  if (isa<FPMathOperator>(CI))
1607  MIB->copyIRFlags(CI);
1608 
1609  for (auto &Arg : CI.arg_operands()) {
1610  // Some intrinsics take metadata parameters. Reject them.
1611  if (isa<MetadataAsValue>(Arg))
1612  return false;
1613  ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg);
1614  if (VRegs.size() > 1)
1615  return false;
1616  MIB.addUse(VRegs[0]);
1617  }
1618 
1619  // Add a MachineMemOperand if it is a target mem intrinsic.
1620  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1621  TargetLowering::IntrinsicInfo Info;
1622  // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
1623  if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
1624  MaybeAlign Align = Info.align;
1625  if (!Align)
1626  Align = MaybeAlign(
1627  DL->getABITypeAlignment(Info.memVT.getTypeForEVT(F->getContext())));
1628 
1629  uint64_t Size = Info.memVT.getStoreSize();
1630  MIB.addMemOperand(MF->getMachineMemOperand(
1631  MachinePointerInfo(Info.ptrVal), Info.flags, Size, Align->value()));
1632  }
1633 
1634  return true;
1635 }
1636 
1637 bool IRTranslator::translateInvoke(const User &U,
1638  MachineIRBuilder &MIRBuilder) {
1639  const InvokeInst &I = cast<InvokeInst>(U);
1640  MCContext &Context = MF->getContext();
1641 
1642  const BasicBlock *ReturnBB = I.getSuccessor(0);
1643  const BasicBlock *EHPadBB = I.getSuccessor(1);
1644 
1645  const Value *Callee = I.getCalledValue();
1646  const Function *Fn = dyn_cast<Function>(Callee);
1647  if (isa<InlineAsm>(Callee))
1648  return false;
1649 
1650  // FIXME: support invoking patchpoint and statepoint intrinsics.
1651  if (Fn && Fn->isIntrinsic())
1652  return false;
1653 
1654  // FIXME: support whatever these are.
1656  return false;
1657 
1658  // FIXME: support Windows exception handling.
1659  if (!isa<LandingPadInst>(EHPadBB->front()))
1660  return false;
1661 
1662  // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
1663  // the region covered by the try.
1664  MCSymbol *BeginSymbol = Context.createTempSymbol();
1665  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
1666 
1667  if (!translateCallSite(&I, MIRBuilder))
1668  return false;
1669 
1670  MCSymbol *EndSymbol = Context.createTempSymbol();
1671  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
1672 
1673  // FIXME: track probabilities.
1674  MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
1675  &ReturnMBB = getMBB(*ReturnBB);
1676  MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
1677  MIRBuilder.getMBB().addSuccessor(&ReturnMBB);
1678  MIRBuilder.getMBB().addSuccessor(&EHPadMBB);
1679  MIRBuilder.buildBr(ReturnMBB);
1680 
1681  return true;
1682 }
1683 
1684 bool IRTranslator::translateCallBr(const User &U,
1685  MachineIRBuilder &MIRBuilder) {
1686  // FIXME: Implement this.
1687  return false;
1688 }
1689 
1690 bool IRTranslator::translateLandingPad(const User &U,
1691  MachineIRBuilder &MIRBuilder) {
1692  const LandingPadInst &LP = cast<LandingPadInst>(U);
1693 
1694  MachineBasicBlock &MBB = MIRBuilder.getMBB();
1695 
1696  MBB.setIsEHPad();
1697 
1698  // If there aren't registers to copy the values into (e.g., during SjLj
1699  // exceptions), then don't bother.
1700  auto &TLI = *MF->getSubtarget().getTargetLowering();
1701  const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
1702  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
1703  TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
1704  return true;
1705 
1706  // If landingpad's return type is token type, we don't create DAG nodes
1707  // for its exception pointer and selector value. The extraction of exception
1708  // pointer or selector value from token type landingpads is not currently
1709  // supported.
1710  if (LP.getType()->isTokenTy())
1711  return true;
1712 
1713  // Add a label to mark the beginning of the landing pad. Deletion of the
1714  // landing pad can thus be detected via the MachineModuleInfo.
1716  .addSym(MF->addLandingPad(&MBB));
1717 
1718  LLT Ty = getLLTForType(*LP.getType(), *DL);
1719  Register Undef = MRI->createGenericVirtualRegister(Ty);
1720  MIRBuilder.buildUndef(Undef);
1721 
1722  SmallVector<LLT, 2> Tys;
1723  for (Type *Ty : cast<StructType>(LP.getType())->elements())
1724  Tys.push_back(getLLTForType(*Ty, *DL));
1725  assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
1726 
1727  // Mark exception register as live in.
1728  Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
1729  if (!ExceptionReg)
1730  return false;
1731 
1732  MBB.addLiveIn(ExceptionReg);
1733  ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
1734  MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
1735 
1736  Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
1737  if (!SelectorReg)
1738  return false;
1739 
1740  MBB.addLiveIn(SelectorReg);
1741  Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
1742  MIRBuilder.buildCopy(PtrVReg, SelectorReg);
1743  MIRBuilder.buildCast(ResRegs[1], PtrVReg);
1744 
1745  return true;
1746 }
1747 
1748 bool IRTranslator::translateAlloca(const User &U,
1749  MachineIRBuilder &MIRBuilder) {
1750  auto &AI = cast<AllocaInst>(U);
1751 
1752  if (AI.isSwiftError())
1753  return true;
1754 
1755  if (AI.isStaticAlloca()) {
1756  Register Res = getOrCreateVReg(AI);
1757  int FI = getOrCreateFrameIndex(AI);
1758  MIRBuilder.buildFrameIndex(Res, FI);
1759  return true;
1760  }
1761 
1762  // FIXME: support stack probing for Windows.
1763  if (MF->getTarget().getTargetTriple().isOSWindows())
1764  return false;
1765 
1766  // Now we're in the harder dynamic case.
1767  Type *Ty = AI.getAllocatedType();
1768  unsigned Align =
1769  std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI.getAlignment());
1770 
1771  Register NumElts = getOrCreateVReg(*AI.getArraySize());
1772 
1773  Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
1774  LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
1775  if (MRI->getType(NumElts) != IntPtrTy) {
1776  Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
1777  MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
1778  NumElts = ExtElts;
1779  }
1780 
1781  Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
1782  Register TySize =
1783  getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, -DL->getTypeAllocSize(Ty)));
1784  MIRBuilder.buildMul(AllocSize, NumElts, TySize);
1785 
1786  LLT PtrTy = getLLTForType(*AI.getType(), *DL);
1787  auto &TLI = *MF->getSubtarget().getTargetLowering();
1788  Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1789 
1790  Register SPTmp = MRI->createGenericVirtualRegister(PtrTy);
1791  MIRBuilder.buildCopy(SPTmp, SPReg);
1792 
1793  Register AllocTmp = MRI->createGenericVirtualRegister(PtrTy);
1794  MIRBuilder.buildGEP(AllocTmp, SPTmp, AllocSize);
1795 
1796  // Handle alignment. We have to realign if the allocation granule was smaller
1797  // than stack alignment, or the specific alloca requires more than stack
1798  // alignment.
1799  unsigned StackAlign =
1800  MF->getSubtarget().getFrameLowering()->getStackAlignment();
1801  Align = std::max(Align, StackAlign);
1802  if (Align > StackAlign || DL->getTypeAllocSize(Ty) % StackAlign != 0) {
1803  // Round the size of the allocation up to the stack alignment size
1804  // by add SA-1 to the size. This doesn't overflow because we're computing
1805  // an address inside an alloca.
1806  Register AlignedAlloc = MRI->createGenericVirtualRegister(PtrTy);
1807  MIRBuilder.buildPtrMask(AlignedAlloc, AllocTmp, Log2_32(Align));
1808  AllocTmp = AlignedAlloc;
1809  }
1810 
1811  MIRBuilder.buildCopy(SPReg, AllocTmp);
1812  MIRBuilder.buildCopy(getOrCreateVReg(AI), AllocTmp);
1813 
1814  MF->getFrameInfo().CreateVariableSizedObject(Align ? Align : 1, &AI);
1815  assert(MF->getFrameInfo().hasVarSizedObjects());
1816  return true;
1817 }
1818 
1819 bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
1820  // FIXME: We may need more info about the type. Because of how LLT works,
1821  // we're completely discarding the i64/double distinction here (amongst
1822  // others). Fortunately the ABIs I know of where that matters don't use va_arg
1823  // anyway but that's not guaranteed.
1824  MIRBuilder.buildInstr(TargetOpcode::G_VAARG)
1825  .addDef(getOrCreateVReg(U))
1826  .addUse(getOrCreateVReg(*U.getOperand(0)))
1827  .addImm(DL->getABITypeAlignment(U.getType()));
1828  return true;
1829 }
1830 
1831 bool IRTranslator::translateInsertElement(const User &U,
1832  MachineIRBuilder &MIRBuilder) {
1833  // If it is a <1 x Ty> vector, use the scalar as it is
1834  // not a legal vector type in LLT.
1835  if (U.getType()->getVectorNumElements() == 1) {
1836  Register Elt = getOrCreateVReg(*U.getOperand(1));
1837  auto &Regs = *VMap.getVRegs(U);
1838  if (Regs.empty()) {
1839  Regs.push_back(Elt);
1840  VMap.getOffsets(U)->push_back(0);
1841  } else {
1842  MIRBuilder.buildCopy(Regs[0], Elt);
1843  }
1844  return true;
1845  }
1846 
1847  Register Res = getOrCreateVReg(U);
1848  Register Val = getOrCreateVReg(*U.getOperand(0));
1849  Register Elt = getOrCreateVReg(*U.getOperand(1));
1850  Register Idx = getOrCreateVReg(*U.getOperand(2));
1851  MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
1852  return true;
1853 }
1854 
1855 bool IRTranslator::translateExtractElement(const User &U,
1856  MachineIRBuilder &MIRBuilder) {
1857  // If it is a <1 x Ty> vector, use the scalar as it is
1858  // not a legal vector type in LLT.
1859  if (U.getOperand(0)->getType()->getVectorNumElements() == 1) {
1860  Register Elt = getOrCreateVReg(*U.getOperand(0));
1861  auto &Regs = *VMap.getVRegs(U);
1862  if (Regs.empty()) {
1863  Regs.push_back(Elt);
1864  VMap.getOffsets(U)->push_back(0);
1865  } else {
1866  MIRBuilder.buildCopy(Regs[0], Elt);
1867  }
1868  return true;
1869  }
1870  Register Res = getOrCreateVReg(U);
1871  Register Val = getOrCreateVReg(*U.getOperand(0));
1872  const auto &TLI = *MF->getSubtarget().getTargetLowering();
1873  unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
1874  Register Idx;
1875  if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
1876  if (CI->getBitWidth() != PreferredVecIdxWidth) {
1877  APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth);
1878  auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
1879  Idx = getOrCreateVReg(*NewIdxCI);
1880  }
1881  }
1882  if (!Idx)
1883  Idx = getOrCreateVReg(*U.getOperand(1));
1884  if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
1885  const LLT &VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
1886  Idx = MIRBuilder.buildSExtOrTrunc(VecIdxTy, Idx)->getOperand(0).getReg();
1887  }
1888  MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
1889  return true;
1890 }
1891 
1892 bool IRTranslator::translateShuffleVector(const User &U,
1893  MachineIRBuilder &MIRBuilder) {
1894  MIRBuilder.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR)
1895  .addDef(getOrCreateVReg(U))
1896  .addUse(getOrCreateVReg(*U.getOperand(0)))
1897  .addUse(getOrCreateVReg(*U.getOperand(1)))
1898  .addShuffleMask(cast<Constant>(U.getOperand(2)));
1899  return true;
1900 }
1901 
1902 bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
1903  const PHINode &PI = cast<PHINode>(U);
1904 
1906  for (auto Reg : getOrCreateVRegs(PI)) {
1907  auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
1908  Insts.push_back(MIB.getInstr());
1909  }
1910 
1911  PendingPHIs.emplace_back(&PI, std::move(Insts));
1912  return true;
1913 }
1914 
1915 bool IRTranslator::translateAtomicCmpXchg(const User &U,
1916  MachineIRBuilder &MIRBuilder) {
1917  const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
1918 
1919  if (I.isWeak())
1920  return false;
1921 
1922  auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1925 
1926  Type *ResType = I.getType();
1927  Type *ValType = ResType->Type::getStructElementType(0);
1928 
1929  auto Res = getOrCreateVRegs(I);
1930  Register OldValRes = Res[0];
1931  Register SuccessRes = Res[1];
1932  Register Addr = getOrCreateVReg(*I.getPointerOperand());
1933  Register Cmp = getOrCreateVReg(*I.getCompareOperand());
1934  Register NewVal = getOrCreateVReg(*I.getNewValOperand());
1935 
1936  MIRBuilder.buildAtomicCmpXchgWithSuccess(
1937  OldValRes, SuccessRes, Addr, Cmp, NewVal,
1938  *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
1939  Flags, DL->getTypeStoreSize(ValType),
1940  getMemOpAlignment(I), AAMDNodes(), nullptr,
1942  I.getFailureOrdering()));
1943  return true;
1944 }
1945 
1946 bool IRTranslator::translateAtomicRMW(const User &U,
1947  MachineIRBuilder &MIRBuilder) {
1948  const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
1949 
1950  auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1953 
1954  Type *ResType = I.getType();
1955 
1956  Register Res = getOrCreateVReg(I);
1957  Register Addr = getOrCreateVReg(*I.getPointerOperand());
1958  Register Val = getOrCreateVReg(*I.getValOperand());
1959 
1960  unsigned Opcode = 0;
1961  switch (I.getOperation()) {
1962  default:
1963  return false;
1964  case AtomicRMWInst::Xchg:
1965  Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
1966  break;
1967  case AtomicRMWInst::Add:
1968  Opcode = TargetOpcode::G_ATOMICRMW_ADD;
1969  break;
1970  case AtomicRMWInst::Sub:
1971  Opcode = TargetOpcode::G_ATOMICRMW_SUB;
1972  break;
1973  case AtomicRMWInst::And:
1974  Opcode = TargetOpcode::G_ATOMICRMW_AND;
1975  break;
1976  case AtomicRMWInst::Nand:
1977  Opcode = TargetOpcode::G_ATOMICRMW_NAND;
1978  break;
1979  case AtomicRMWInst::Or:
1980  Opcode = TargetOpcode::G_ATOMICRMW_OR;
1981  break;
1982  case AtomicRMWInst::Xor:
1983  Opcode = TargetOpcode::G_ATOMICRMW_XOR;
1984  break;
1985  case AtomicRMWInst::Max:
1986  Opcode = TargetOpcode::G_ATOMICRMW_MAX;
1987  break;
1988  case AtomicRMWInst::Min:
1989  Opcode = TargetOpcode::G_ATOMICRMW_MIN;
1990  break;
1991  case AtomicRMWInst::UMax:
1992  Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
1993  break;
1994  case AtomicRMWInst::UMin:
1995  Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
1996  break;
1997  case AtomicRMWInst::FAdd:
1998  Opcode = TargetOpcode::G_ATOMICRMW_FADD;
1999  break;
2000  case AtomicRMWInst::FSub:
2001  Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
2002  break;
2003  }
2004 
2005  MIRBuilder.buildAtomicRMW(
2006  Opcode, Res, Addr, Val,
2007  *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
2008  Flags, DL->getTypeStoreSize(ResType),
2009  getMemOpAlignment(I), AAMDNodes(), nullptr,
2010  I.getSyncScopeID(), I.getOrdering()));
2011  return true;
2012 }
2013 
2014 bool IRTranslator::translateFence(const User &U,
2015  MachineIRBuilder &MIRBuilder) {
2016  const FenceInst &Fence = cast<FenceInst>(U);
2017  MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
2018  Fence.getSyncScopeID());
2019  return true;
2020 }
2021 
2022 void IRTranslator::finishPendingPhis() {
2023 #ifndef NDEBUG
2024  DILocationVerifier Verifier;
2025  GISelObserverWrapper WrapperObserver(&Verifier);
2026  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2027 #endif // ifndef NDEBUG
2028  for (auto &Phi : PendingPHIs) {
2029  const PHINode *PI = Phi.first;
2030  ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
2031  MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
2032  EntryBuilder->setDebugLoc(PI->getDebugLoc());
2033 #ifndef NDEBUG
2034  Verifier.setCurrentInst(PI);
2035 #endif // ifndef NDEBUG
2036 
2038  for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
2039  auto IRPred = PI->getIncomingBlock(i);
2040  ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
2041  for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
2042  if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
2043  continue;
2044  SeenPreds.insert(Pred);
2045  for (unsigned j = 0; j < ValRegs.size(); ++j) {
2046  MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
2047  MIB.addUse(ValRegs[j]);
2048  MIB.addMBB(Pred);
2049  }
2050  }
2051  }
2052  }
2053 }
2054 
2055 bool IRTranslator::valueIsSplit(const Value &V,
2057  SmallVector<LLT, 4> SplitTys;
2058  if (Offsets && !Offsets->empty())
2059  Offsets->clear();
2060  computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets);
2061  return SplitTys.size() > 1;
2062 }
2063 
2064 bool IRTranslator::translate(const Instruction &Inst) {
2065  CurBuilder->setDebugLoc(Inst.getDebugLoc());
2066  // We only emit constants into the entry block from here. To prevent jumpy
2067  // debug behaviour set the line to 0.
2068  if (const DebugLoc &DL = Inst.getDebugLoc())
2069  EntryBuilder->setDebugLoc(
2070  DebugLoc::get(0, 0, DL.getScope(), DL.getInlinedAt()));
2071  else
2072  EntryBuilder->setDebugLoc(DebugLoc());
2073 
2074  switch (Inst.getOpcode()) {
2075 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2076  case Instruction::OPCODE: \
2077  return translate##OPCODE(Inst, *CurBuilder.get());
2078 #include "llvm/IR/Instruction.def"
2079  default:
2080  return false;
2081  }
2082 }
2083 
2084 bool IRTranslator::translate(const Constant &C, Register Reg) {
2085  if (auto CI = dyn_cast<ConstantInt>(&C))
2086  EntryBuilder->buildConstant(Reg, *CI);
2087  else if (auto CF = dyn_cast<ConstantFP>(&C))
2088  EntryBuilder->buildFConstant(Reg, *CF);
2089  else if (isa<UndefValue>(C))
2090  EntryBuilder->buildUndef(Reg);
2091  else if (isa<ConstantPointerNull>(C)) {
2092  // As we are trying to build a constant val of 0 into a pointer,
2093  // insert a cast to make them correct with respect to types.
2094  unsigned NullSize = DL->getTypeSizeInBits(C.getType());
2095  auto *ZeroTy = Type::getIntNTy(C.getContext(), NullSize);
2096  auto *ZeroVal = ConstantInt::get(ZeroTy, 0);
2097  Register ZeroReg = getOrCreateVReg(*ZeroVal);
2098  EntryBuilder->buildCast(Reg, ZeroReg);
2099  } else if (auto GV = dyn_cast<GlobalValue>(&C))
2100  EntryBuilder->buildGlobalValue(Reg, GV);
2101  else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
2102  if (!CAZ->getType()->isVectorTy())
2103  return false;
2104  // Return the scalar if it is a <1 x Ty> vector.
2105  if (CAZ->getNumElements() == 1)
2106  return translate(*CAZ->getElementValue(0u), Reg);
2108  for (unsigned i = 0; i < CAZ->getNumElements(); ++i) {
2109  Constant &Elt = *CAZ->getElementValue(i);
2110  Ops.push_back(getOrCreateVReg(Elt));
2111  }
2112  EntryBuilder->buildBuildVector(Reg, Ops);
2113  } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
2114  // Return the scalar if it is a <1 x Ty> vector.
2115  if (CV->getNumElements() == 1)
2116  return translate(*CV->getElementAsConstant(0), Reg);
2118  for (unsigned i = 0; i < CV->getNumElements(); ++i) {
2119  Constant &Elt = *CV->getElementAsConstant(i);
2120  Ops.push_back(getOrCreateVReg(Elt));
2121  }
2122  EntryBuilder->buildBuildVector(Reg, Ops);
2123  } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
2124  switch(CE->getOpcode()) {
2125 #define HANDLE_INST(NUM, OPCODE, CLASS) \
2126  case Instruction::OPCODE: \
2127  return translate##OPCODE(*CE, *EntryBuilder.get());
2128 #include "llvm/IR/Instruction.def"
2129  default:
2130  return false;
2131  }
2132  } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
2133  if (CV->getNumOperands() == 1)
2134  return translate(*CV->getOperand(0), Reg);
2136  for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
2137  Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
2138  }
2139  EntryBuilder->buildBuildVector(Reg, Ops);
2140  } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
2141  EntryBuilder->buildBlockAddress(Reg, BA);
2142  } else
2143  return false;
2144 
2145  return true;
2146 }
2147 
2148 void IRTranslator::finalizeBasicBlock() {
2149  for (auto &JTCase : SL->JTCases) {
2150  // Emit header first, if it wasn't already emitted.
2151  if (!JTCase.first.Emitted)
2152  emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
2153 
2154  emitJumpTable(JTCase.second, JTCase.second.MBB);
2155  }
2156  SL->JTCases.clear();
2157 }
2158 
2159 void IRTranslator::finalizeFunction() {
2160  // Release the memory used by the different maps we
2161  // needed during the translation.
2162  PendingPHIs.clear();
2163  VMap.reset();
2164  FrameIndices.clear();
2165  MachinePreds.clear();
2166  // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
2167  // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
2168  // destroying it twice (in ~IRTranslator() and ~LLVMContext())
2169  EntryBuilder.reset();
2170  CurBuilder.reset();
2171  FuncInfo.clear();
2172 }
2173 
2175  MF = &CurMF;
2176  const Function &F = MF->getFunction();
2177  if (F.empty())
2178  return false;
2180  getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
2181  // Set the CSEConfig and run the analysis.
2182  GISelCSEInfo *CSEInfo = nullptr;
2183  TPC = &getAnalysis<TargetPassConfig>();
2184  bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
2186  : TPC->isGISelCSEEnabled();
2187 
2188  if (EnableCSE) {
2189  EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
2190  CSEInfo = &Wrapper.get(TPC->getCSEConfig());
2191  EntryBuilder->setCSEInfo(CSEInfo);
2192  CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
2193  CurBuilder->setCSEInfo(CSEInfo);
2194  } else {
2195  EntryBuilder = std::make_unique<MachineIRBuilder>();
2196  CurBuilder = std::make_unique<MachineIRBuilder>();
2197  }
2198  CLI = MF->getSubtarget().getCallLowering();
2199  CurBuilder->setMF(*MF);
2200  EntryBuilder->setMF(*MF);
2201  MRI = &MF->getRegInfo();
2202  DL = &F.getParent()->getDataLayout();
2203  ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
2204  FuncInfo.MF = MF;
2205  FuncInfo.BPI = nullptr;
2206  const auto &TLI = *MF->getSubtarget().getTargetLowering();
2207  const TargetMachine &TM = MF->getTarget();
2208  SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
2209  SL->init(TLI, TM, *DL);
2210 
2211  EnableOpts = TM.getOptLevel() != CodeGenOpt::None && !skipFunction(F);
2212 
2213  assert(PendingPHIs.empty() && "stale PHIs");
2214 
2215  if (!DL->isLittleEndian()) {
2216  // Currently we don't properly handle big endian code.
2217  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2218  F.getSubprogram(), &F.getEntryBlock());
2219  R << "unable to translate in big endian mode";
2220  reportTranslationError(*MF, *TPC, *ORE, R);
2221  }
2222 
2223  // Release the per-function state when we return, whether we succeeded or not.
2224  auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
2225 
2226  // Setup a separate basic-block for the arguments and constants
2227  MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
2228  MF->push_back(EntryBB);
2229  EntryBuilder->setMBB(*EntryBB);
2230 
2232  SwiftError.setFunction(CurMF);
2233  SwiftError.createEntriesInEntryBlock(DbgLoc);
2234 
2235  // Create all blocks, in IR order, to preserve the layout.
2236  for (const BasicBlock &BB: F) {
2237  auto *&MBB = BBToMBB[&BB];
2238 
2239  MBB = MF->CreateMachineBasicBlock(&BB);
2240  MF->push_back(MBB);
2241 
2242  if (BB.hasAddressTaken())
2243  MBB->setHasAddressTaken();
2244  }
2245 
2246  // Make our arguments/constants entry block fallthrough to the IR entry block.
2247  EntryBB->addSuccessor(&getMBB(F.front()));
2248 
2249  // Lower the actual args into this basic block.
2250  SmallVector<ArrayRef<Register>, 8> VRegArgs;
2251  for (const Argument &Arg: F.args()) {
2252  if (DL->getTypeStoreSize(Arg.getType()) == 0)
2253  continue; // Don't handle zero sized types.
2254  ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
2255  VRegArgs.push_back(VRegs);
2256 
2257  if (Arg.hasSwiftErrorAttr()) {
2258  assert(VRegs.size() == 1 && "Too many vregs for Swift error");
2259  SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
2260  }
2261  }
2262 
2263  if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs)) {
2264  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2265  F.getSubprogram(), &F.getEntryBlock());
2266  R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
2267  reportTranslationError(*MF, *TPC, *ORE, R);
2268  return false;
2269  }
2270 
2271  // Need to visit defs before uses when translating instructions.
2272  GISelObserverWrapper WrapperObserver;
2273  if (EnableCSE && CSEInfo)
2274  WrapperObserver.addObserver(CSEInfo);
2275  {
2277 #ifndef NDEBUG
2278  DILocationVerifier Verifier;
2279  WrapperObserver.addObserver(&Verifier);
2280 #endif // ifndef NDEBUG
2281  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2282  for (const BasicBlock *BB : RPOT) {
2283  MachineBasicBlock &MBB = getMBB(*BB);
2284  // Set the insertion point of all the following translations to
2285  // the end of this basic block.
2286  CurBuilder->setMBB(MBB);
2287 
2288  for (const Instruction &Inst : *BB) {
2289 #ifndef NDEBUG
2290  Verifier.setCurrentInst(&Inst);
2291 #endif // ifndef NDEBUG
2292  if (translate(Inst))
2293  continue;
2294 
2295  OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2296  Inst.getDebugLoc(), BB);
2297  R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
2298 
2299  if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
2300  std::string InstStrStorage;
2301  raw_string_ostream InstStr(InstStrStorage);
2302  InstStr << Inst;
2303 
2304  R << ": '" << InstStr.str() << "'";
2305  }
2306 
2307  reportTranslationError(*MF, *TPC, *ORE, R);
2308  return false;
2309  }
2310 
2311  finalizeBasicBlock();
2312  }
2313 #ifndef NDEBUG
2314  WrapperObserver.removeObserver(&Verifier);
2315 #endif
2316  }
2317 
2318  finishPendingPhis();
2319 
2320  SwiftError.propagateVRegs();
2321 
2322  // Merge the argument lowering and constants block with its single
2323  // successor, the LLVM-IR entry block. We want the basic block to
2324  // be maximal.
2325  assert(EntryBB->succ_size() == 1 &&
2326  "Custom BB used for lowering should have only one successor");
2327  // Get the successor of the current entry block.
2328  MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
2329  assert(NewEntryBB.pred_size() == 1 &&
2330  "LLVM-IR entry block has a predecessor!?");
2331  // Move all the instruction from the current entry block to the
2332  // new entry block.
2333  NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
2334  EntryBB->end());
2335 
2336  // Update the live-in information for the new entry block.
2337  for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
2338  NewEntryBB.addLiveIn(LiveIn);
2339  NewEntryBB.sortUniqueLiveIns();
2340 
2341  // Get rid of the now empty basic block.
2342  EntryBB->removeSuccessor(&NewEntryBB);
2343  MF->remove(EntryBB);
2344  MF->DeleteMachineBasicBlock(EntryBB);
2345 
2346  assert(&MF->front() == &NewEntryBB &&
2347  "New entry wasn't next in the list of basic block!");
2348 
2349  // Initialize stack protector information.
2350  StackProtector &SP = getAnalysis<StackProtector>();
2351  SP.copyToMachineFrameInfo(MF->getFrameInfo());
2352 
2353  return false;
2354 }
auto lower_bound(R &&Range, T &&Value) -> decltype(adl_begin(Range))
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1273
MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in Re...
uint64_t CallInst * C
Return a value (possibly void), from a function.
Value * getValueOperand()
Definition: Instructions.h:409
unsigned getNumCases() const
Return the number of &#39;cases&#39; in this switch instruction, excluding the default case.
bool isIntrinsic() const
isIntrinsic - Returns true if the function&#39;s name starts with "llvm.".
Definition: Function.h:198
A simple RAII based CSEInfo installer.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:641
MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI)
Build and insert Res = G_JUMP_TABLE JTI.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:722
bool empty() const
Definition: Function.h:686
The CSE Analysis object.
Definition: CSEInfo.h:71
Diagnostic information for missed-optimization remarks.
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:21
This instruction extracts a struct member or array element value from an aggregate value...
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
*p = old <signed v ? old : v
Definition: Instructions.h:723
iterator_range< CaseIt > cases()
Iteration adapter for range-for loops.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
LLVMContext & Context
DiagnosticInfoOptimizationBase::Argument NV
CaseClusterVector::iterator CaseClusterIt
This represents the llvm.dbg.label instruction.
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:288
MachineInstrBuilder buildBrCond(Register Tst, MachineBasicBlock &Dest)
Build and insert G_BRCOND Tst, Dest.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:412
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
bool isSized(SmallPtrSetImpl< Type *> *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:264
iterator begin() const
Definition: ArrayRef.h:136
Register getReg(unsigned Idx) const
Get the register for the operand index.
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
An instruction for ordering other memory operations.
Definition: Instructions.h:454
An instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:530
void push_back(const T &Elt)
Definition: SmallVector.h:211
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:385
const MachineInstrBuilder & addShuffleMask(const Constant *Val) const
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
IRTranslator LLVM IR static false void reportTranslationError(MachineFunction &MF, const TargetPassConfig &TPC, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R)
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
Value * getCondition() const
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:57
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this store instruction.
Definition: Instructions.h:384
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:758
const std::string & getAsmString() const
Definition: InlineAsm.h:80
*p = old <unsigned v ? old : v
Definition: Instructions.h:727
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
Definition: Instructions.h:135
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:247
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1100
*p = old >unsigned v ? old : v
Definition: Instructions.h:725
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:58
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:745
void addSuccessorWithoutProb(MachineBasicBlock *Succ)
Add Succ as a successor of this MachineBasicBlock.
Type * getPointerOperandType() const
Definition: Instructions.h:415
unsigned getLine() const
Definition: DebugLoc.cpp:25
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:265
BasicBlock * getSuccessor(unsigned i) const
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
Metadata node.
Definition: Metadata.h:863
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
F(f)
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1212
The actual analysis pass wrapper.
Definition: CSEInfo.h:218
An instruction for reading from memory.
Definition: Instructions.h:167
void setMF(MachineFunction &MF)
Definition: CSEInfo.cpp:78
MachineInstrBuilder buildAtomicRMW(unsigned Opcode, const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO.
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:693
Value * getCondition() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:229
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 maximum semantics.
Definition: APFloat.h:1272
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:158
*p = old >signed v ? old : v
Definition: Instructions.h:721
uint64_t High
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Optional< MachineInstrBuilder > materializeGEP(Register &Res, Register Op0, const LLT &ValueTy, uint64_t Value)
Materialize and insert Res = G_GEP Op0, (G_CONSTANT Value)
MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, Register IndexReg)
Build and insert G_BRJT TablePtr, JTI, IndexReg.
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:274
AtomicOrdering getFailureOrdering() const
Returns the failure ordering constraint of this cmpxchg instruction.
Definition: Instructions.h:597
bool hasSideEffects() const
Definition: InlineAsm.h:66
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1241
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:231
A description of a memory reference used in the backend.
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:1784
This class represents the LLVM &#39;select&#39; instruction.
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
#define DEBUG_TYPE
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
unsigned getAlignment() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:112
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
PointerType * getType() const
Overload to return most specific pointer type.
Definition: Instructions.h:96
Class to represent struct types.
Definition: DerivedTypes.h:233
ValTy * getCalledValue() const
Return the pointer to function that is being called.
Definition: CallSite.h:104
DILabel * getLabel() const
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:654
BinOp getOperation() const
Definition: Instructions.h:752
bool isWeak() const
Return true if this cmpxchg may spuriously fail.
Definition: Instructions.h:571
A cluster of cases suitable for jump table lowering.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 minimum semantics.
Definition: APFloat.h:1259
The memory access is dereferenceable (i.e., doesn&#39;t trap).
std::vector< CaseCluster > CaseClusterVector
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
Definition: Instructions.h:559
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Target-Independent Code Generator Pass Configuration Options.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:695
Context object for machine code objects.
Definition: MCContext.h:65
InstrTy * getInstruction() const
Definition: CallSite.h:96
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope)
Build and insert G_FENCE Ordering, Scope.
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
Definition: Lint.cpp:83
AtomicOrdering getSuccessOrdering() const
Returns the success ordering constraint of this cmpxchg instruction.
Definition: Instructions.h:584
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:234
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:125
iterator_range< User::op_iterator > arg_operands()
Definition: InstrTypes.h:1233
An instruction for storing to memory.
Definition: Instructions.h:320
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_SUB Op0, Op1.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
Value * getOperand(unsigned i) const
Definition: User.h:169
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
BasicBlockListType::iterator iterator
unsigned JTI
The JumpTableIndex for this jump table in the function.
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:877
bool doesNotAccessMemory() const
Determine if the function does not access memory.
Definition: Function.h:473
MachineInstrBuilder buildDbgLabel(const MDNode *Label)
Build and insert a DBG_LABEL instructions specifying that Label is given.
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
The memory access is volatile.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this label.
const BasicBlock & getEntryBlock() const
Definition: Function.h:664
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:614
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Abstract class that contains various methods for clients to notify about changes. ...
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fuse-fp-ops=xxx option.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:189
void GetUnderlyingObjects(const Value *V, SmallVectorImpl< const Value *> &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
Definition: Instructions.h:804
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned const MachineRegisterInfo * MRI
Value * getCalledValue() const
Definition: InstrTypes.h:1280
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1504
Conditional or Unconditional Branch instruction.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Value * getAddress() const
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1356
MachineBasicBlock * MBB
The MBB into which to emit the code for the indirect jump.
Value * getValue() const
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const Instruction & front() const
Definition: BasicBlock.h:280
void computeValueLLTs(const DataLayout &DL, Type &Ty, SmallVectorImpl< LLT > &ValueTys, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
computeValueLLTs - Given an LLVM IR type, compute a sequence of LLTs that represent all the individua...
Definition: Analysis.cpp:127
Indirect Branch Instruction.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
Helper class to build MachineInstr.
BasicBlock * getDefaultDest() const
DIExpression * getExpression() const
bool isPredecessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a predecessor of this block.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Represent the analysis usage information of a pass.
GISelCSEInfo & get(std::unique_ptr< CSEConfigBase > CSEOpt, bool ReCompute=false)
Takes a CSEConfigBase object that defines what opcodes get CSEd.
Definition: CSEInfo.cpp:364
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:732
void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition: AsmWriter.cpp:4278
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Value * getPointerOperand()
Definition: Instructions.h:284
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:180
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
MachineInstrBuilder buildGEP(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_GEP Op0, Op1.
const MachineInstrBuilder & addFrameIndex(int Idx) const
static double log2(double V)
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:328
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:749
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:205
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this cmpxchg instruction.
Definition: Instructions.h:610
BasicBlock * getSuccessor(unsigned i) const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const Value * getArraySize() const
Get the number of elements allocated.
Definition: Instructions.h:92
size_t size() const
Definition: SmallVector.h:52
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
static uint16_t copyFlagsFromInstruction(const Instruction &I)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:219
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
Definition: Instructions.h:791
Simple wrapper that does the following.
Definition: CSEInfo.h:200
This class contains a discriminated union of information about pointers in memory operands...
std::string & str()
Flushes the stream contents to the target string and returns the string&#39;s reference.
Definition: raw_ostream.h:519
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:1248
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
const std::string & getConstraintString() const
Definition: InlineAsm.h:81
iterator_range< IterTy > args() const
Definition: CallSite.h:222
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:40
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:105
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:703
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects)
Build and insert either a G_INTRINSIC (if HasSideEffects is false) or G_INTRINSIC_W_SIDE_EFFECTS inst...
void copyIRFlags(const Instruction &I)
Copy all flags to MachineInst MIFlags.
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1107
LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
The memory access writes data.
MachineInstrBuilder buildBr(MachineBasicBlock &Dest)
Build and insert G_BR Dest.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
Definition: BasicBlock.h:391
MachineInstrBuilder buildConstDbgValue(const Constant &C, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instructions specifying that Variable is given by C (suitably modified b...
struct PredInfoPair PredInfo
Value * getValOperand()
Definition: Instructions.h:817
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
Definition: PPCPredicates.h:87
unsigned getNumOperands() const
Definition: User.h:191
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_MUL Op0, Op1.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
This struct is a compact representation of a valid (power of two) or undefined (0) alignment...
Definition: Alignment.h:85
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
iterator end() const
Definition: ArrayRef.h:137
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:257
unsigned getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL)
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:643
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition: Type.cpp:179
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:640
DebugLoc getDebugLoc()
Get the current instruction&#39;s debug location.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
MachineInstrBuilder buildPtrMask(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTR_MASK Op0, NumBits.
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:538
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
unsigned getVectorNumElements() const
Definition: DerivedTypes.h:535
bool isIntPredicate() const
Definition: InstrTypes.h:825
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
Class for arbitrary precision integers.
Definition: APInt.h:69
amdgpu Simplify well known AMD library false FunctionCallee Callee
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
unsigned Reg
The virtual register containing the index of the jump table entry to jump to.
void setHasAddressTaken()
Set this block to reflect that it potentially is the target of an indirect branch.
static char ID
Definition: IRTranslator.h:63
MachineInstrBuilder buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def>, SuccessRes<def> = G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO.
virtual bool isFMAFasterThanFMulAndFAdd(EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
The memory access reads data.
#define Success
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1206
Representation of each machine instruction.
Definition: MachineInstr.h:64
unsigned succ_size(const Instruction *I)
Definition: CFG.h:256
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:807
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void addObserver(GISelChangeObserver *O)
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
Definition: Instructions.h:490
bool isVolatile() const
Return true if this is a store to a volatile memory location.
Definition: Instructions.h:353
Analysis providing branch probability information.
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1239
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:321
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Type * getPointerOperandType() const
Definition: Instructions.h:287
MachineBasicBlock * Default
The MBB of the default bb, which is a successor of the range check MBB.
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:175
void removeObserver(GISelChangeObserver *O)
A cluster of adjacent case labels with the same destination, or just one case.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
AtomicOrdering getOrdering() const
Returns the ordering constraint of this store instruction.
Definition: Instructions.h:372
This represents the llvm.dbg.value instruction.
bool isTokenTy() const
Return true if this is &#39;token&#39;.
Definition: Type.h:193
verify safepoint Safepoint IR Verifier
Value * getPointerOperand()
Definition: Instructions.h:813
MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr)
Build and insert a DBG_VALUE instruction expressing the fact that the associated Variable lives in me...
TargetOptions Options
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
Establish a view to a call site for examination.
Definition: CallSite.h:897
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1287
static cl::opt< bool > EnableCSEInIRTranslator("enable-cse-in-irtranslator", cl::desc("Should enable CSE in irtranslator"), cl::Optional, cl::init(false))
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
Definition: Instructions.h:259
void setMBB(MachineBasicBlock &MBB)
Set the insertion point to the end of MBB.
void push_back(MachineInstr *MI)
#define I(x, y, z)
Definition: MD5.cpp:58
static Constant * getZeroValueForNegation(Type *Ty)
Floating point negation must be implemented with f(x) = -0.0 - x.
Definition: Constants.cpp:789
Pair of physical register and lane mask.
The memory access always returns the same value (or traps).
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:192
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
DILocalVariable * getVariable() const
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool isUnconditional() const
AsmDialect getDialect() const
Definition: InlineAsm.h:68
Multiway switch.
This file declares the IRTranslator pass.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void insert(iterator MBBI, MachineBasicBlock *MBB)
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:503
aarch64 promote const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:575
LLVM Value Representation.
Definition: Value.h:73
static bool isSwiftError(const Value *V)
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:445
succ_range successors(Instruction *I)
Definition: CFG.h:259
This file describes how to lower LLVM calls to machine code calls.
INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI", false, false) INITIALIZE_PASS_END(IRTranslator
Invoke instruction.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
IRTranslator LLVM IR MI
unsigned greater than
Definition: InstrTypes.h:755
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Definition: Pass.cpp:166
const Instruction * getFirstNonPHIOrDbg() const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic...
Definition: BasicBlock.cpp:196
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
Definition: Instructions.h:478
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
Simple wrapper observer that takes several observers, and calls each one for each event...
Register getReg() const
getReg - Returns the register number.
bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size...
A cluster of cases suitable for bit test lowering.
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1229
This represents the llvm.dbg.declare instruction.
Value * getPointerOperand()
Definition: Instructions.h:412
The optimization diagnostic interface.
Statically lint checks LLVM IR
Definition: Lint.cpp:192
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:1237
std::vector< MachineBasicBlock * >::iterator succ_iterator
int64_t getIndexedOffsetInType(Type *ElemTy, ArrayRef< Value *> Indices) const
Returns the offset from the beginning of the type for the specified indices.
Definition: DataLayout.cpp:809
bool isVolatile() const
Return true if this is a RMW on a volatile memory location.
Definition: Instructions.h:776
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:734
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
IntegerType * Int32Ty
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
virtual const TargetRegisterClass * getPointerRegClass(const MachineFunction &MF, unsigned Kind=0) const
Returns a TargetRegisterClass used for pointer values.
an instruction to allocate memory on the stack
Definition: Instructions.h:59
This instruction inserts a struct field of array element value into an aggregate value.
gep_type_iterator gep_type_begin(const User *GEP)
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164