LLVM  7.0.0svn
FastISel.cpp
Go to the documentation of this file.
1 //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the implementation of the FastISel class.
11 //
12 // "Fast" instruction selection is designed to emit very poor code quickly.
13 // Also, it is not designed to be able to do much lowering, so most illegal
14 // types (e.g. i64 on 32-bit targets) and operations are not supported. It is
15 // also not intended to be able to do much optimization, except in a few cases
16 // where doing optimizations reduces overall compile time. For example, folding
17 // constants into immediate fields is often done, because it's cheap and it
18 // reduces the number of instructions later phases have to examine.
19 //
20 // "Fast" instruction selection is able to fail gracefully and transfer
21 // control to the SelectionDAG selector for operations that it doesn't
22 // support. In many cases, this allows us to avoid duplicating a lot of
23 // the complicated lowering logic that SelectionDAG currently has.
24 //
25 // The intended use for "fast" instruction selection is "-O0" mode
26 // compilation, where the quality of the generated code is irrelevant when
27 // weighed against the speed at which the code can be generated. Also,
28 // at -O0, the LLVM optimizers are not running, and this makes the
29 // compile time of codegen a much higher portion of the overall compile
30 // time. Despite its limitations, "fast" instruction selection is able to
31 // handle enough code on its own to provide noticeable overall speedups
32 // in -O0 compiles.
33 //
34 // Basic operations are supported in a target-independent way, by reading
35 // the same instruction descriptions that the SelectionDAG selector reads,
36 // and identifying simple arithmetic operations that can be directly selected
37 // from simple operators. More complicated operations currently require
38 // target-specific code.
39 //
40 //===----------------------------------------------------------------------===//
41 
42 #include "llvm/CodeGen/FastISel.h"
43 #include "llvm/ADT/APFloat.h"
44 #include "llvm/ADT/APSInt.h"
45 #include "llvm/ADT/DenseMap.h"
46 #include "llvm/ADT/Optional.h"
47 #include "llvm/ADT/SmallPtrSet.h"
48 #include "llvm/ADT/SmallString.h"
49 #include "llvm/ADT/SmallVector.h"
50 #include "llvm/ADT/Statistic.h"
53 #include "llvm/CodeGen/Analysis.h"
64 #include "llvm/CodeGen/StackMaps.h"
69 #include "llvm/IR/Argument.h"
70 #include "llvm/IR/Attributes.h"
71 #include "llvm/IR/BasicBlock.h"
72 #include "llvm/IR/CallSite.h"
73 #include "llvm/IR/CallingConv.h"
74 #include "llvm/IR/Constant.h"
75 #include "llvm/IR/Constants.h"
76 #include "llvm/IR/DataLayout.h"
77 #include "llvm/IR/DebugInfo.h"
78 #include "llvm/IR/DebugLoc.h"
79 #include "llvm/IR/DerivedTypes.h"
80 #include "llvm/IR/Function.h"
82 #include "llvm/IR/GlobalValue.h"
83 #include "llvm/IR/InlineAsm.h"
84 #include "llvm/IR/InstrTypes.h"
85 #include "llvm/IR/Instruction.h"
86 #include "llvm/IR/Instructions.h"
87 #include "llvm/IR/IntrinsicInst.h"
88 #include "llvm/IR/LLVMContext.h"
89 #include "llvm/IR/Mangler.h"
90 #include "llvm/IR/Metadata.h"
91 #include "llvm/IR/Operator.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCInstrDesc.h"
97 #include "llvm/MC/MCRegisterInfo.h"
98 #include "llvm/Support/Casting.h"
99 #include "llvm/Support/Debug.h"
102 #include "llvm/Support/MathExtras.h"
106 #include <algorithm>
107 #include <cassert>
108 #include <cstdint>
109 #include <iterator>
110 #include <utility>
111 
112 using namespace llvm;
113 
114 #define DEBUG_TYPE "isel"
115 
116 // FIXME: Remove this after the feature has proven reliable.
117 static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values",
118  cl::init(true), cl::Hidden,
119  cl::desc("Sink local values in FastISel"));
120 
121 STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
122  "target-independent selector");
123 STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
124  "target-specific selector");
125 STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
126 
127 /// Set the current block to which generated machine instructions will be
128 /// appended.
131  "local values should be cleared after finishing a BB");
132 
133  // Instructions are appended to FuncInfo.MBB. If the basic block already
134  // contains labels or copies, use the last instruction as the last local
135  // value.
136  EmitStartPt = nullptr;
137  if (!FuncInfo.MBB->empty())
138  EmitStartPt = &FuncInfo.MBB->back();
140 }
141 
142 /// Flush the local CSE map and sink anything we can.
143 void FastISel::finishBasicBlock() { flushLocalValueMap(); }
144 
147  // Fallback to SDISel argument lowering code to deal with sret pointer
148  // parameter.
149  return false;
150 
151  if (!fastLowerArguments())
152  return false;
153 
154  // Enter arguments into ValueMap for uses in non-entry BBs.
156  E = FuncInfo.Fn->arg_end();
157  I != E; ++I) {
159  assert(VI != LocalValueMap.end() && "Missed an argument?");
160  FuncInfo.ValueMap[&*I] = VI->second;
161  }
162  return true;
163 }
164 
165 /// Return the defined register if this instruction defines exactly one
166 /// virtual register and uses no other virtual registers. Otherwise return 0.
168  unsigned RegDef = 0;
169  for (const MachineOperand &MO : MI.operands()) {
170  if (!MO.isReg())
171  continue;
172  if (MO.isDef()) {
173  if (RegDef)
174  return 0;
175  RegDef = MO.getReg();
176  } else if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
177  // This is another use of a vreg. Don't try to sink it.
178  return 0;
179  }
180  }
181  return RegDef;
182 }
183 
184 void FastISel::flushLocalValueMap() {
185  // Try to sink local values down to their first use so that we can give them a
186  // better debug location. This has the side effect of shrinking local value
187  // live ranges, which helps out fast regalloc.
189  // Sink local value materialization instructions between EmitStartPt and
190  // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
191  // avoid inserting into the range that we're iterating over.
194  : FuncInfo.MBB->rend();
195  MachineBasicBlock::reverse_iterator RI(LastLocalValue);
196 
197  InstOrderMap OrderMap;
198  for (; RI != RE;) {
199  MachineInstr &LocalMI = *RI;
200  ++RI;
201  bool Store = true;
202  if (!LocalMI.isSafeToMove(nullptr, Store))
203  continue;
204  unsigned DefReg = findSinkableLocalRegDef(LocalMI);
205  if (DefReg == 0)
206  continue;
207 
208  sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
209  }
210  }
211 
215  SavedInsertPt = FuncInfo.InsertPt;
217 }
218 
219 static bool isRegUsedByPhiNodes(unsigned DefReg,
221  for (auto &P : FuncInfo.PHINodesToUpdate)
222  if (P.second == DefReg)
223  return true;
224  return false;
225 }
226 
227 /// Build a map of instruction orders. Return the first terminator and its
228 /// order. Consider EH_LABEL instructions to be terminators as well, since local
229 /// values for phis after invokes must be materialized before the call.
232  unsigned Order = 0;
233  for (MachineInstr &I : *MBB) {
234  if (!FirstTerminator &&
235  (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) {
236  FirstTerminator = &I;
237  FirstTerminatorOrder = Order;
238  }
239  Orders[&I] = Order++;
240 
241  // We don't need to order instructions past the last flush point.
242  if (I.getIterator() == LastFlushPoint)
243  break;
244  }
245 }
246 
247 void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
248  unsigned DefReg,
249  InstOrderMap &OrderMap) {
250  // If this register is used by a register fixup, MRI will not contain all
251  // the uses until after register fixups, so don't attempt to sink or DCE
252  // this instruction. Register fixups typically come from no-op cast
253  // instructions, which replace the cast instruction vreg with the local
254  // value vreg.
255  if (FuncInfo.RegsWithFixups.count(DefReg))
256  return;
257 
258  // We can DCE this instruction if there are no uses and it wasn't a
259  // materialized for a successor PHI node.
260  bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
261  if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
262  if (EmitStartPt == &LocalMI)
264  LLVM_DEBUG(dbgs() << "removing dead local value materialization "
265  << LocalMI);
266  OrderMap.Orders.erase(&LocalMI);
267  LocalMI.eraseFromParent();
268  return;
269  }
270 
271  // Number the instructions if we haven't yet so we can efficiently find the
272  // earliest use.
273  if (OrderMap.Orders.empty())
274  OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
275 
276  // Find the first user in the BB.
277  MachineInstr *FirstUser = nullptr;
278  unsigned FirstOrder = std::numeric_limits<unsigned>::max();
279  for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
280  auto I = OrderMap.Orders.find(&UseInst);
281  assert(I != OrderMap.Orders.end() &&
282  "local value used by instruction outside local region");
283  unsigned UseOrder = I->second;
284  if (UseOrder < FirstOrder) {
285  FirstOrder = UseOrder;
286  FirstUser = &UseInst;
287  }
288  }
289 
290  // The insertion point will be the first terminator or the first user,
291  // whichever came first. If there was no terminator, this must be a
292  // fallthrough block and the insertion point is the end of the block.
294  if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder) {
295  FirstOrder = OrderMap.FirstTerminatorOrder;
296  SinkPos = OrderMap.FirstTerminator->getIterator();
297  } else if (FirstUser) {
298  SinkPos = FirstUser->getIterator();
299  } else {
300  assert(UsedByPHI && "must be users if not used by a phi");
301  SinkPos = FuncInfo.MBB->instr_end();
302  }
303 
304  // Collect all DBG_VALUEs before the new insertion position so that we can
305  // sink them.
307  for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
308  if (!DbgVal.isDebugValue())
309  continue;
310  unsigned UseOrder = OrderMap.Orders[&DbgVal];
311  if (UseOrder < FirstOrder)
312  DbgValues.push_back(&DbgVal);
313  }
314 
315  // Sink LocalMI before SinkPos and assign it the same DebugLoc.
316  LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
317  FuncInfo.MBB->remove(&LocalMI);
318  FuncInfo.MBB->insert(SinkPos, &LocalMI);
319  if (SinkPos != FuncInfo.MBB->end())
320  LocalMI.setDebugLoc(SinkPos->getDebugLoc());
321 
322  // Sink any debug values that we've collected.
323  for (MachineInstr *DI : DbgValues) {
324  FuncInfo.MBB->remove(DI);
325  FuncInfo.MBB->insert(SinkPos, DI);
326  }
327 }
328 
330  // Don't consider constants or arguments to have trivial kills.
331  const Instruction *I = dyn_cast<Instruction>(V);
332  if (!I)
333  return false;
334 
335  // No-op casts are trivially coalesced by fast-isel.
336  if (const auto *Cast = dyn_cast<CastInst>(I))
337  if (Cast->isNoopCast(DL) && !hasTrivialKill(Cast->getOperand(0)))
338  return false;
339 
340  // Even the value might have only one use in the LLVM IR, it is possible that
341  // FastISel might fold the use into another instruction and now there is more
342  // than one use at the Machine Instruction level.
343  unsigned Reg = lookUpRegForValue(V);
344  if (Reg && !MRI.use_empty(Reg))
345  return false;
346 
347  // GEPs with all zero indices are trivially coalesced by fast-isel.
348  if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
349  if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0)))
350  return false;
351 
352  // Only instructions with a single use in the same basic block are considered
353  // to have trivial kills.
354  return I->hasOneUse() &&
355  !(I->getOpcode() == Instruction::BitCast ||
356  I->getOpcode() == Instruction::PtrToInt ||
357  I->getOpcode() == Instruction::IntToPtr) &&
358  cast<Instruction>(*I->user_begin())->getParent() == I->getParent();
359 }
360 
361 unsigned FastISel::getRegForValue(const Value *V) {
362  EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
363  // Don't handle non-simple values in FastISel.
364  if (!RealVT.isSimple())
365  return 0;
366 
367  // Ignore illegal types. We must do this before looking up the value
368  // in ValueMap because Arguments are given virtual registers regardless
369  // of whether FastISel can handle them.
370  MVT VT = RealVT.getSimpleVT();
371  if (!TLI.isTypeLegal(VT)) {
372  // Handle integer promotions, though, because they're common and easy.
373  if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
374  VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
375  else
376  return 0;
377  }
378 
379  // Look up the value to see if we already have a register for it.
380  unsigned Reg = lookUpRegForValue(V);
381  if (Reg)
382  return Reg;
383 
384  // In bottom-up mode, just create the virtual register which will be used
385  // to hold the value. It will be materialized later.
386  if (isa<Instruction>(V) &&
387  (!isa<AllocaInst>(V) ||
388  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
390 
391  SavePoint SaveInsertPt = enterLocalValueArea();
392 
393  // Materialize the value in a register. Emit any instructions in the
394  // local value area.
395  Reg = materializeRegForValue(V, VT);
396 
397  leaveLocalValueArea(SaveInsertPt);
398 
399  return Reg;
400 }
401 
402 unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
403  unsigned Reg = 0;
404  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
405  if (CI->getValue().getActiveBits() <= 64)
406  Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
407  } else if (isa<AllocaInst>(V))
408  Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
409  else if (isa<ConstantPointerNull>(V))
410  // Translate this as an integer zero so that it can be
411  // local-CSE'd with actual integer zeros.
412  Reg = getRegForValue(
414  else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
415  if (CF->isNullValue())
416  Reg = fastMaterializeFloatZero(CF);
417  else
418  // Try to emit the constant directly.
419  Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
420 
421  if (!Reg) {
422  // Try to emit the constant by using an integer constant with a cast.
423  const APFloat &Flt = CF->getValueAPF();
424  EVT IntVT = TLI.getPointerTy(DL);
425  uint32_t IntBitWidth = IntVT.getSizeInBits();
426  APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
427  bool isExact;
428  (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
429  if (isExact) {
430  unsigned IntegerReg =
432  if (IntegerReg != 0)
433  Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
434  /*Kill=*/false);
435  }
436  }
437  } else if (const auto *Op = dyn_cast<Operator>(V)) {
438  if (!selectOperator(Op, Op->getOpcode()))
439  if (!isa<Instruction>(Op) ||
440  !fastSelectInstruction(cast<Instruction>(Op)))
441  return 0;
442  Reg = lookUpRegForValue(Op);
443  } else if (isa<UndefValue>(V)) {
446  TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
447  }
448  return Reg;
449 }
450 
451 /// Helper for getRegForValue. This function is called when the value isn't
452 /// already available in a register and must be materialized with new
453 /// instructions.
454 unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
455  unsigned Reg = 0;
456  // Give the target-specific code a try first.
457  if (isa<Constant>(V))
458  Reg = fastMaterializeConstant(cast<Constant>(V));
459 
460  // If target-specific code couldn't or didn't want to handle the value, then
461  // give target-independent code a try.
462  if (!Reg)
463  Reg = materializeConstant(V, VT);
464 
465  // Don't cache constant materializations in the general ValueMap.
466  // To do so would require tracking what uses they dominate.
467  if (Reg) {
468  LocalValueMap[V] = Reg;
470  }
471  return Reg;
472 }
473 
474 unsigned FastISel::lookUpRegForValue(const Value *V) {
475  // Look up the value to see if we already have a register for it. We
476  // cache values defined by Instructions across blocks, and other values
477  // only locally. This is because Instructions already have the SSA
478  // def-dominates-use requirement enforced.
480  if (I != FuncInfo.ValueMap.end())
481  return I->second;
482  return LocalValueMap[V];
483 }
484 
485 void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
486  if (!isa<Instruction>(I)) {
487  LocalValueMap[I] = Reg;
488  return;
489  }
490 
491  unsigned &AssignedReg = FuncInfo.ValueMap[I];
492  if (AssignedReg == 0)
493  // Use the new register.
494  AssignedReg = Reg;
495  else if (Reg != AssignedReg) {
496  // Arrange for uses of AssignedReg to be replaced by uses of Reg.
497  for (unsigned i = 0; i < NumRegs; i++) {
498  FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
499  FuncInfo.RegsWithFixups.insert(Reg + i);
500  }
501 
502  AssignedReg = Reg;
503  }
504 }
505 
506 std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
507  unsigned IdxN = getRegForValue(Idx);
508  if (IdxN == 0)
509  // Unhandled operand. Halt "fast" selection and bail.
510  return std::pair<unsigned, bool>(0, false);
511 
512  bool IdxNIsKill = hasTrivialKill(Idx);
513 
514  // If the index is smaller or larger than intptr_t, truncate or extend it.
515  MVT PtrVT = TLI.getPointerTy(DL);
516  EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
517  if (IdxVT.bitsLT(PtrVT)) {
518  IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
519  IdxNIsKill);
520  IdxNIsKill = true;
521  } else if (IdxVT.bitsGT(PtrVT)) {
522  IdxN =
523  fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
524  IdxNIsKill = true;
525  }
526  return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
527 }
528 
530  if (getLastLocalValue()) {
532  FuncInfo.MBB = FuncInfo.InsertPt->getParent();
533  ++FuncInfo.InsertPt;
534  } else
536 
537  // Now skip past any EH_LABELs, which must remain at the beginning.
538  while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
539  FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL)
540  ++FuncInfo.InsertPt;
541 }
542 
545  assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
546  "Invalid iterator!");
547  while (I != E) {
548  MachineInstr *Dead = &*I;
549  ++I;
550  Dead->eraseFromParent();
551  ++NumFastIselDead;
552  }
554 }
555 
558  DebugLoc OldDL = DbgLoc;
560  DbgLoc = DebugLoc();
561  SavePoint SP = {OldInsertPt, OldDL};
562  return SP;
563 }
564 
566  if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
567  LastLocalValue = &*std::prev(FuncInfo.InsertPt);
568 
569  // Restore the previous insert position.
570  FuncInfo.InsertPt = OldInsertPt.InsertPt;
571  DbgLoc = OldInsertPt.DL;
572 }
573 
574 bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
575  EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
576  if (VT == MVT::Other || !VT.isSimple())
577  // Unhandled type. Halt "fast" selection and bail.
578  return false;
579 
580  // We only handle legal types. For example, on x86-32 the instruction
581  // selector contains all of the 64-bit instructions from x86-64,
582  // under the assumption that i64 won't be used if the target doesn't
583  // support it.
584  if (!TLI.isTypeLegal(VT)) {
585  // MVT::i1 is special. Allow AND, OR, or XOR because they
586  // don't require additional zeroing, which makes them easy.
587  if (VT == MVT::i1 && (ISDOpcode == ISD::AND || ISDOpcode == ISD::OR ||
588  ISDOpcode == ISD::XOR))
589  VT = TLI.getTypeToTransformTo(I->getContext(), VT);
590  else
591  return false;
592  }
593 
594  // Check if the first operand is a constant, and handle it as "ri". At -O0,
595  // we don't have anything that canonicalizes operand order.
596  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
597  if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
598  unsigned Op1 = getRegForValue(I->getOperand(1));
599  if (!Op1)
600  return false;
601  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
602 
603  unsigned ResultReg =
604  fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
605  CI->getZExtValue(), VT.getSimpleVT());
606  if (!ResultReg)
607  return false;
608 
609  // We successfully emitted code for the given LLVM Instruction.
610  updateValueMap(I, ResultReg);
611  return true;
612  }
613 
614  unsigned Op0 = getRegForValue(I->getOperand(0));
615  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
616  return false;
617  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
618 
619  // Check if the second operand is a constant and handle it appropriately.
620  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
621  uint64_t Imm = CI->getSExtValue();
622 
623  // Transform "sdiv exact X, 8" -> "sra X, 3".
624  if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
625  cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
626  Imm = Log2_64(Imm);
627  ISDOpcode = ISD::SRA;
628  }
629 
630  // Transform "urem x, pow2" -> "and x, pow2-1".
631  if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
632  isPowerOf2_64(Imm)) {
633  --Imm;
634  ISDOpcode = ISD::AND;
635  }
636 
637  unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
638  Op0IsKill, Imm, VT.getSimpleVT());
639  if (!ResultReg)
640  return false;
641 
642  // We successfully emitted code for the given LLVM Instruction.
643  updateValueMap(I, ResultReg);
644  return true;
645  }
646 
647  unsigned Op1 = getRegForValue(I->getOperand(1));
648  if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
649  return false;
650  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
651 
652  // Now we have both operands in registers. Emit the instruction.
653  unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
654  ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
655  if (!ResultReg)
656  // Target-specific code wasn't able to find a machine opcode for
657  // the given ISD opcode and type. Halt "fast" selection and bail.
658  return false;
659 
660  // We successfully emitted code for the given LLVM Instruction.
661  updateValueMap(I, ResultReg);
662  return true;
663 }
664 
666  unsigned N = getRegForValue(I->getOperand(0));
667  if (!N) // Unhandled operand. Halt "fast" selection and bail.
668  return false;
669  bool NIsKill = hasTrivialKill(I->getOperand(0));
670 
671  // Keep a running tab of the total offset to coalesce multiple N = N + Offset
672  // into a single N = N + TotalOffset.
673  uint64_t TotalOffs = 0;
674  // FIXME: What's a good SWAG number for MaxOffs?
675  uint64_t MaxOffs = 2048;
676  MVT VT = TLI.getPointerTy(DL);
677  for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
678  GTI != E; ++GTI) {
679  const Value *Idx = GTI.getOperand();
680  if (StructType *StTy = GTI.getStructTypeOrNull()) {
681  uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
682  if (Field) {
683  // N = N + Offset
684  TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
685  if (TotalOffs >= MaxOffs) {
686  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
687  if (!N) // Unhandled operand. Halt "fast" selection and bail.
688  return false;
689  NIsKill = true;
690  TotalOffs = 0;
691  }
692  }
693  } else {
694  Type *Ty = GTI.getIndexedType();
695 
696  // If this is a constant subscript, handle it quickly.
697  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
698  if (CI->isZero())
699  continue;
700  // N = N + Offset
701  uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
702  TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
703  if (TotalOffs >= MaxOffs) {
704  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
705  if (!N) // Unhandled operand. Halt "fast" selection and bail.
706  return false;
707  NIsKill = true;
708  TotalOffs = 0;
709  }
710  continue;
711  }
712  if (TotalOffs) {
713  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
714  if (!N) // Unhandled operand. Halt "fast" selection and bail.
715  return false;
716  NIsKill = true;
717  TotalOffs = 0;
718  }
719 
720  // N = N + Idx * ElementSize;
721  uint64_t ElementSize = DL.getTypeAllocSize(Ty);
722  std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
723  unsigned IdxN = Pair.first;
724  bool IdxNIsKill = Pair.second;
725  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
726  return false;
727 
728  if (ElementSize != 1) {
729  IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
730  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
731  return false;
732  IdxNIsKill = true;
733  }
734  N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
735  if (!N) // Unhandled operand. Halt "fast" selection and bail.
736  return false;
737  }
738  }
739  if (TotalOffs) {
740  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
741  if (!N) // Unhandled operand. Halt "fast" selection and bail.
742  return false;
743  }
744 
745  // We successfully emitted code for the given LLVM Instruction.
746  updateValueMap(I, N);
747  return true;
748 }
749 
750 bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
751  const CallInst *CI, unsigned StartIdx) {
752  for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) {
753  Value *Val = CI->getArgOperand(i);
754  // Check for constants and encode them with a StackMaps::ConstantOp prefix.
755  if (const auto *C = dyn_cast<ConstantInt>(Val)) {
756  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
757  Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
758  } else if (isa<ConstantPointerNull>(Val)) {
759  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
761  } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
762  // Values coming from a stack location also require a special encoding,
763  // but that is added later on by the target specific frame index
764  // elimination implementation.
765  auto SI = FuncInfo.StaticAllocaMap.find(AI);
766  if (SI != FuncInfo.StaticAllocaMap.end())
767  Ops.push_back(MachineOperand::CreateFI(SI->second));
768  else
769  return false;
770  } else {
771  unsigned Reg = getRegForValue(Val);
772  if (!Reg)
773  return false;
774  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
775  }
776  }
777  return true;
778 }
779 
781  // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
782  // [live variables...])
784  "Stackmap cannot return a value.");
785 
786  // The stackmap intrinsic only records the live variables (the arguments
787  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
788  // intrinsic, this won't be lowered to a function call. This means we don't
789  // have to worry about calling conventions and target-specific lowering code.
790  // Instead we perform the call lowering right here.
791  //
792  // CALLSEQ_START(0, 0...)
793  // STACKMAP(id, nbytes, ...)
794  // CALLSEQ_END(0, 0)
795  //
797 
798  // Add the <id> and <numBytes> constants.
799  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
800  "Expected a constant integer.");
801  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
802  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
803 
804  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
805  "Expected a constant integer.");
806  const auto *NumBytes =
807  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
808  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
809 
810  // Push live variables for the stack map (skipping the first two arguments
811  // <id> and <numBytes>).
812  if (!addStackMapLiveVars(Ops, I, 2))
813  return false;
814 
815  // We are not adding any register mask info here, because the stackmap doesn't
816  // clobber anything.
817 
818  // Add scratch registers as implicit def and early clobber.
820  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
821  for (unsigned i = 0; ScratchRegs[i]; ++i)
822  Ops.push_back(MachineOperand::CreateReg(
823  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
824  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
825 
826  // Issue CALLSEQ_START
827  unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
828  auto Builder =
829  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
830  const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
831  for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
832  Builder.addImm(0);
833 
834  // Issue STACKMAP.
836  TII.get(TargetOpcode::STACKMAP));
837  for (auto const &MO : Ops)
838  MIB.add(MO);
839 
840  // Issue CALLSEQ_END
841  unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
842  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
843  .addImm(0)
844  .addImm(0);
845 
846  // Inform the Frame Information that we have a stackmap in this function.
848 
849  return true;
850 }
851 
852 /// Lower an argument list according to the target calling convention.
853 ///
854 /// This is a helper for lowering intrinsics that follow a target calling
855 /// convention or require stack pointer adjustment. Only a subset of the
856 /// intrinsic's operands need to participate in the calling convention.
857 bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
858  unsigned NumArgs, const Value *Callee,
859  bool ForceRetVoidTy, CallLoweringInfo &CLI) {
860  ArgListTy Args;
861  Args.reserve(NumArgs);
862 
863  // Populate the argument list.
864  ImmutableCallSite CS(CI);
865  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
866  Value *V = CI->getOperand(ArgI);
867 
868  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
869 
870  ArgListEntry Entry;
871  Entry.Val = V;
872  Entry.Ty = V->getType();
873  Entry.setAttributes(&CS, ArgI);
874  Args.push_back(Entry);
875  }
876 
877  Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
878  : CI->getType();
879  CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
880 
881  return lowerCallTo(CLI);
882 }
883 
885  const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
886  StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
887  SmallString<32> MangledName;
888  Mangler::getNameWithPrefix(MangledName, Target, DL);
889  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
890  return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
891 }
892 
894  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
895  // i32 <numBytes>,
896  // i8* <target>,
897  // i32 <numArgs>,
898  // [Args...],
899  // [live variables...])
901  bool IsAnyRegCC = CC == CallingConv::AnyReg;
902  bool HasDef = !I->getType()->isVoidTy();
904 
905  // Get the real number of arguments participating in the call <numArgs>
906  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
907  "Expected a constant integer.");
908  const auto *NumArgsVal =
909  cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
910  unsigned NumArgs = NumArgsVal->getZExtValue();
911 
912  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
913  // This includes all meta-operands up to but not including CC.
914  unsigned NumMetaOpers = PatchPointOpers::CCPos;
915  assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
916  "Not enough arguments provided to the patchpoint intrinsic");
917 
918  // For AnyRegCC the arguments are lowered later on manually.
919  unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
920  CallLoweringInfo CLI;
921  CLI.setIsPatchPoint();
922  if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
923  return false;
924 
925  assert(CLI.Call && "No call instruction specified.");
926 
928 
929  // Add an explicit result reg if we use the anyreg calling convention.
930  if (IsAnyRegCC && HasDef) {
931  assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
933  CLI.NumResultRegs = 1;
934  Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*IsDef=*/true));
935  }
936 
937  // Add the <id> and <numBytes> constants.
938  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
939  "Expected a constant integer.");
940  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
941  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
942 
943  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
944  "Expected a constant integer.");
945  const auto *NumBytes =
946  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
947  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
948 
949  // Add the call target.
950  if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
951  uint64_t CalleeConstAddr =
952  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
953  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
954  } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
955  if (C->getOpcode() == Instruction::IntToPtr) {
956  uint64_t CalleeConstAddr =
957  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
958  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
959  } else
960  llvm_unreachable("Unsupported ConstantExpr.");
961  } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
963  } else if (isa<ConstantPointerNull>(Callee))
965  else
966  llvm_unreachable("Unsupported callee address.");
967 
968  // Adjust <numArgs> to account for any arguments that have been passed on
969  // the stack instead.
970  unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
971  Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
972 
973  // Add the calling convention
974  Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
975 
976  // Add the arguments we omitted previously. The register allocator should
977  // place these in any free register.
978  if (IsAnyRegCC) {
979  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
980  unsigned Reg = getRegForValue(I->getArgOperand(i));
981  if (!Reg)
982  return false;
983  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
984  }
985  }
986 
987  // Push the arguments from the call instruction.
988  for (auto Reg : CLI.OutRegs)
989  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
990 
991  // Push live variables for the stack map.
992  if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
993  return false;
994 
995  // Push the register mask info.
998 
999  // Add scratch registers as implicit def and early clobber.
1000  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
1001  for (unsigned i = 0; ScratchRegs[i]; ++i)
1003  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
1004  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
1005 
1006  // Add implicit defs (return values).
1007  for (auto Reg : CLI.InRegs)
1008  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/true,
1009  /*IsImpl=*/true));
1010 
1011  // Insert the patchpoint instruction before the call generated by the target.
1013  TII.get(TargetOpcode::PATCHPOINT));
1014 
1015  for (auto &MO : Ops)
1016  MIB.add(MO);
1017 
1018  MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1019 
1020  // Delete the original call instruction.
1021  CLI.Call->eraseFromParent();
1022 
1023  // Inform the Frame Information that we have a patchpoint in this function.
1025 
1026  if (CLI.NumResultRegs)
1027  updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
1028  return true;
1029 }
1030 
1032  const auto &Triple = TM.getTargetTriple();
1033  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1034  return true; // don't do anything to this instruction.
1037  /*IsDef=*/false));
1039  /*IsDef=*/false));
1040  MachineInstrBuilder MIB =
1042  TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
1043  for (auto &MO : Ops)
1044  MIB.add(MO);
1045 
1046  // Insert the Patchable Event Call instruction, that gets lowered properly.
1047  return true;
1048 }
1049 
1051  const auto &Triple = TM.getTargetTriple();
1052  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1053  return true; // don't do anything to this instruction.
1056  /*IsDef=*/false));
1058  /*IsDef=*/false));
1060  /*IsDef=*/false));
1061  MachineInstrBuilder MIB =
1063  TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
1064  for (auto &MO : Ops)
1065  MIB.add(MO);
1066 
1067  // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
1068  return true;
1069 }
1070 
1071 /// Returns an AttributeList representing the attributes applied to the return
1072 /// value of the given call.
1075  if (CLI.RetSExt)
1076  Attrs.push_back(Attribute::SExt);
1077  if (CLI.RetZExt)
1078  Attrs.push_back(Attribute::ZExt);
1079  if (CLI.IsInReg)
1080  Attrs.push_back(Attribute::InReg);
1081 
1083  Attrs);
1084 }
1085 
1086 bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
1087  unsigned NumArgs) {
1088  MCContext &Ctx = MF->getContext();
1089  SmallString<32> MangledName;
1090  Mangler::getNameWithPrefix(MangledName, SymName, DL);
1091  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
1092  return lowerCallTo(CI, Sym, NumArgs);
1093 }
1094 
1096  unsigned NumArgs) {
1097  ImmutableCallSite CS(CI);
1098 
1099  FunctionType *FTy = CS.getFunctionType();
1100  Type *RetTy = CS.getType();
1101 
1102  ArgListTy Args;
1103  Args.reserve(NumArgs);
1104 
1105  // Populate the argument list.
1106  // Attributes for args start at offset 1, after the return attribute.
1107  for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
1108  Value *V = CI->getOperand(ArgI);
1109 
1110  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
1111 
1112  ArgListEntry Entry;
1113  Entry.Val = V;
1114  Entry.Ty = V->getType();
1115  Entry.setAttributes(&CS, ArgI);
1116  Args.push_back(Entry);
1117  }
1119 
1120  CallLoweringInfo CLI;
1121  CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
1122 
1123  return lowerCallTo(CLI);
1124 }
1125 
1127  // Handle the incoming return values from the call.
1128  CLI.clearIns();
1129  SmallVector<EVT, 4> RetTys;
1130  ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
1131 
1133  GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
1134 
1135  bool CanLowerReturn = TLI.CanLowerReturn(
1136  CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1137 
1138  // FIXME: sret demotion isn't supported yet - bail out.
1139  if (!CanLowerReturn)
1140  return false;
1141 
1142  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
1143  EVT VT = RetTys[I];
1144  MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1145  unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1146  for (unsigned i = 0; i != NumRegs; ++i) {
1147  ISD::InputArg MyFlags;
1148  MyFlags.VT = RegisterVT;
1149  MyFlags.ArgVT = VT;
1150  MyFlags.Used = CLI.IsReturnValueUsed;
1151  if (CLI.RetSExt)
1152  MyFlags.Flags.setSExt();
1153  if (CLI.RetZExt)
1154  MyFlags.Flags.setZExt();
1155  if (CLI.IsInReg)
1156  MyFlags.Flags.setInReg();
1157  CLI.Ins.push_back(MyFlags);
1158  }
1159  }
1160 
1161  // Handle all of the outgoing arguments.
1162  CLI.clearOuts();
1163  for (auto &Arg : CLI.getArgs()) {
1164  Type *FinalType = Arg.Ty;
1165  if (Arg.IsByVal)
1166  FinalType = cast<PointerType>(Arg.Ty)->getElementType();
1167  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1168  FinalType, CLI.CallConv, CLI.IsVarArg);
1169 
1170  ISD::ArgFlagsTy Flags;
1171  if (Arg.IsZExt)
1172  Flags.setZExt();
1173  if (Arg.IsSExt)
1174  Flags.setSExt();
1175  if (Arg.IsInReg)
1176  Flags.setInReg();
1177  if (Arg.IsSRet)
1178  Flags.setSRet();
1179  if (Arg.IsSwiftSelf)
1180  Flags.setSwiftSelf();
1181  if (Arg.IsSwiftError)
1182  Flags.setSwiftError();
1183  if (Arg.IsByVal)
1184  Flags.setByVal();
1185  if (Arg.IsInAlloca) {
1186  Flags.setInAlloca();
1187  // Set the byval flag for CCAssignFn callbacks that don't know about
1188  // inalloca. This way we can know how many bytes we should've allocated
1189  // and how many bytes a callee cleanup function will pop. If we port
1190  // inalloca to more targets, we'll have to add custom inalloca handling in
1191  // the various CC lowering callbacks.
1192  Flags.setByVal();
1193  }
1194  if (Arg.IsByVal || Arg.IsInAlloca) {
1195  PointerType *Ty = cast<PointerType>(Arg.Ty);
1196  Type *ElementTy = Ty->getElementType();
1197  unsigned FrameSize = DL.getTypeAllocSize(ElementTy);
1198  // For ByVal, alignment should come from FE. BE will guess if this info is
1199  // not there, but there are cases it cannot get right.
1200  unsigned FrameAlign = Arg.Alignment;
1201  if (!FrameAlign)
1202  FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
1203  Flags.setByValSize(FrameSize);
1204  Flags.setByValAlign(FrameAlign);
1205  }
1206  if (Arg.IsNest)
1207  Flags.setNest();
1208  if (NeedsRegBlock)
1209  Flags.setInConsecutiveRegs();
1210  unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
1211  Flags.setOrigAlign(OriginalAlignment);
1212 
1213  CLI.OutVals.push_back(Arg.Val);
1214  CLI.OutFlags.push_back(Flags);
1215  }
1216 
1217  if (!fastLowerCall(CLI))
1218  return false;
1219 
1220  // Set all unused physreg defs as dead.
1221  assert(CLI.Call && "No call instruction specified.");
1222  CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1223 
1224  if (CLI.NumResultRegs && CLI.CS)
1226 
1227  return true;
1228 }
1229 
1231  ImmutableCallSite CS(CI);
1232 
1233  FunctionType *FuncTy = CS.getFunctionType();
1234  Type *RetTy = CS.getType();
1235 
1236  ArgListTy Args;
1237  ArgListEntry Entry;
1238  Args.reserve(CS.arg_size());
1239 
1240  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
1241  i != e; ++i) {
1242  Value *V = *i;
1243 
1244  // Skip empty types
1245  if (V->getType()->isEmptyTy())
1246  continue;
1247 
1248  Entry.Val = V;
1249  Entry.Ty = V->getType();
1250 
1251  // Skip the first return-type Attribute to get to params.
1252  Entry.setAttributes(&CS, i - CS.arg_begin());
1253  Args.push_back(Entry);
1254  }
1255 
1256  // Check if target-independent constraints permit a tail call here.
1257  // Target-dependent constraints are checked within fastLowerCall.
1258  bool IsTailCall = CI->isTailCall();
1259  if (IsTailCall && !isInTailCallPosition(CS, TM))
1260  IsTailCall = false;
1261 
1262  CallLoweringInfo CLI;
1263  CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
1264  .setTailCall(IsTailCall);
1265 
1266  return lowerCallTo(CLI);
1267 }
1268 
1270  const CallInst *Call = cast<CallInst>(I);
1271 
1272  // Handle simple inline asms.
1273  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
1274  // If the inline asm has side effects, then make sure that no local value
1275  // lives across by flushing the local value map.
1276  if (IA->hasSideEffects())
1277  flushLocalValueMap();
1278 
1279  // Don't attempt to handle constraints.
1280  if (!IA->getConstraintString().empty())
1281  return false;
1282 
1283  unsigned ExtraInfo = 0;
1284  if (IA->hasSideEffects())
1285  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1286  if (IA->isAlignStack())
1287  ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1288 
1291  .addExternalSymbol(IA->getAsmString().c_str())
1292  .addImm(ExtraInfo);
1293  return true;
1294  }
1295 
1296  MachineModuleInfo &MMI = FuncInfo.MF->getMMI();
1297  computeUsesVAFloatArgument(*Call, MMI);
1298 
1299  // Handle intrinsic function calls.
1300  if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1301  return selectIntrinsicCall(II);
1302 
1303  // Usually, it does not make sense to initialize a value,
1304  // make an unrelated function call and use the value, because
1305  // it tends to be spilled on the stack. So, we move the pointer
1306  // to the last local value to the beginning of the block, so that
1307  // all the values which have already been materialized,
1308  // appear after the call. It also makes sense to skip intrinsics
1309  // since they tend to be inlined.
1310  flushLocalValueMap();
1311 
1312  return lowerCall(Call);
1313 }
1314 
1316  switch (II->getIntrinsicID()) {
1317  default:
1318  break;
1319  // At -O0 we don't care about the lifetime intrinsics.
1320  case Intrinsic::lifetime_start:
1321  case Intrinsic::lifetime_end:
1322  // The donothing intrinsic does, well, nothing.
1323  case Intrinsic::donothing:
1324  // Neither does the sideeffect intrinsic.
1325  case Intrinsic::sideeffect:
1326  // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1327  case Intrinsic::assume:
1328  return true;
1329  case Intrinsic::dbg_declare: {
1330  const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1331  assert(DI->getVariable() && "Missing variable");
1332  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1333  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1334  return true;
1335  }
1336 
1337  const Value *Address = DI->getAddress();
1338  if (!Address || isa<UndefValue>(Address)) {
1339  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1340  return true;
1341  }
1342 
1343  // Byval arguments with frame indices were already handled after argument
1344  // lowering and before isel.
1345  const auto *Arg =
1347  if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
1348  return true;
1349 
1351  if (unsigned Reg = lookUpRegForValue(Address))
1352  Op = MachineOperand::CreateReg(Reg, false);
1353 
1354  // If we have a VLA that has a "use" in a metadata node that's then used
1355  // here but it has no other uses, then we have a problem. E.g.,
1356  //
1357  // int foo (const int *x) {
1358  // char a[*x];
1359  // return 0;
1360  // }
1361  //
1362  // If we assign 'a' a vreg and fast isel later on has to use the selection
1363  // DAG isel, it will want to copy the value to the vreg. However, there are
1364  // no uses, which goes counter to what selection DAG isel expects.
1365  if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
1366  (!isa<AllocaInst>(Address) ||
1367  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
1369  false);
1370 
1371  if (Op) {
1373  "Expected inlined-at fields to agree");
1374  if (Op->isReg()) {
1375  Op->setIsDebug(true);
1376  // A dbg.declare describes the address of a source variable, so lower it
1377  // into an indirect DBG_VALUE.
1379  TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true,
1380  Op->getReg(), DI->getVariable(), DI->getExpression());
1381  } else
1383  TII.get(TargetOpcode::DBG_VALUE))
1384  .add(*Op)
1385  .addImm(0)
1386  .addMetadata(DI->getVariable())
1387  .addMetadata(DI->getExpression());
1388  } else {
1389  // We can't yet handle anything else here because it would require
1390  // generating code, thus altering codegen because of debug info.
1391  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1392  }
1393  return true;
1394  }
1395  case Intrinsic::dbg_value: {
1396  // This form of DBG_VALUE is target-independent.
1397  const DbgValueInst *DI = cast<DbgValueInst>(II);
1398  const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1399  const Value *V = DI->getValue();
1401  "Expected inlined-at fields to agree");
1402  if (!V) {
1403  // Currently the optimizer can produce this; insert an undef to
1404  // help debugging. Probably the optimizer should not do this.
1405  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
1406  DI->getVariable(), DI->getExpression());
1407  } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1408  if (CI->getBitWidth() > 64)
1410  .addCImm(CI)
1411  .addImm(0U)
1412  .addMetadata(DI->getVariable())
1413  .addMetadata(DI->getExpression());
1414  else
1416  .addImm(CI->getZExtValue())
1417  .addImm(0U)
1418  .addMetadata(DI->getVariable())
1419  .addMetadata(DI->getExpression());
1420  } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1422  .addFPImm(CF)
1423  .addImm(0U)
1424  .addMetadata(DI->getVariable())
1425  .addMetadata(DI->getExpression());
1426  } else if (unsigned Reg = lookUpRegForValue(V)) {
1427  // FIXME: This does not handle register-indirect values at offset 0.
1428  bool IsIndirect = false;
1429  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
1430  DI->getVariable(), DI->getExpression());
1431  } else {
1432  // We can't yet handle anything else here because it would require
1433  // generating code, thus altering codegen because of debug info.
1434  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1435  }
1436  return true;
1437  }
1438  case Intrinsic::objectsize: {
1439  ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
1440  unsigned long long Res = CI->isZero() ? -1ULL : 0;
1441  Constant *ResCI = ConstantInt::get(II->getType(), Res);
1442  unsigned ResultReg = getRegForValue(ResCI);
1443  if (!ResultReg)
1444  return false;
1445  updateValueMap(II, ResultReg);
1446  return true;
1447  }
1448  case Intrinsic::launder_invariant_group:
1449  case Intrinsic::expect: {
1450  unsigned ResultReg = getRegForValue(II->getArgOperand(0));
1451  if (!ResultReg)
1452  return false;
1453  updateValueMap(II, ResultReg);
1454  return true;
1455  }
1456  case Intrinsic::experimental_stackmap:
1457  return selectStackmap(II);
1458  case Intrinsic::experimental_patchpoint_void:
1459  case Intrinsic::experimental_patchpoint_i64:
1460  return selectPatchpoint(II);
1461 
1462  case Intrinsic::xray_customevent:
1463  return selectXRayCustomEvent(II);
1464  case Intrinsic::xray_typedevent:
1465  return selectXRayTypedEvent(II);
1466  }
1467 
1468  return fastLowerIntrinsicCall(II);
1469 }
1470 
1471 bool FastISel::selectCast(const User *I, unsigned Opcode) {
1472  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1473  EVT DstVT = TLI.getValueType(DL, I->getType());
1474 
1475  if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1476  !DstVT.isSimple())
1477  // Unhandled type. Halt "fast" selection and bail.
1478  return false;
1479 
1480  // Check if the destination type is legal.
1481  if (!TLI.isTypeLegal(DstVT))
1482  return false;
1483 
1484  // Check if the source operand is legal.
1485  if (!TLI.isTypeLegal(SrcVT))
1486  return false;
1487 
1488  unsigned InputReg = getRegForValue(I->getOperand(0));
1489  if (!InputReg)
1490  // Unhandled operand. Halt "fast" selection and bail.
1491  return false;
1492 
1493  bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
1494 
1495  unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1496  Opcode, InputReg, InputRegIsKill);
1497  if (!ResultReg)
1498  return false;
1499 
1500  updateValueMap(I, ResultReg);
1501  return true;
1502 }
1503 
1505  // If the bitcast doesn't change the type, just use the operand value.
1506  if (I->getType() == I->getOperand(0)->getType()) {
1507  unsigned Reg = getRegForValue(I->getOperand(0));
1508  if (!Reg)
1509  return false;
1510  updateValueMap(I, Reg);
1511  return true;
1512  }
1513 
1514  // Bitcasts of other values become reg-reg copies or BITCAST operators.
1515  EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1516  EVT DstEVT = TLI.getValueType(DL, I->getType());
1517  if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1518  !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
1519  // Unhandled type. Halt "fast" selection and bail.
1520  return false;
1521 
1522  MVT SrcVT = SrcEVT.getSimpleVT();
1523  MVT DstVT = DstEVT.getSimpleVT();
1524  unsigned Op0 = getRegForValue(I->getOperand(0));
1525  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1526  return false;
1527  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
1528 
1529  // First, try to perform the bitcast by inserting a reg-reg copy.
1530  unsigned ResultReg = 0;
1531  if (SrcVT == DstVT) {
1532  const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
1533  const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
1534  // Don't attempt a cross-class copy. It will likely fail.
1535  if (SrcClass == DstClass) {
1536  ResultReg = createResultReg(DstClass);
1538  TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
1539  }
1540  }
1541 
1542  // If the reg-reg copy failed, select a BITCAST opcode.
1543  if (!ResultReg)
1544  ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
1545 
1546  if (!ResultReg)
1547  return false;
1548 
1549  updateValueMap(I, ResultReg);
1550  return true;
1551 }
1552 
1553 // Remove local value instructions starting from the instruction after
1554 // SavedLastLocalValue to the current function insert point.
1555 void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1556 {
1557  MachineInstr *CurLastLocalValue = getLastLocalValue();
1558  if (CurLastLocalValue != SavedLastLocalValue) {
1559  // Find the first local value instruction to be deleted.
1560  // This is the instruction after SavedLastLocalValue if it is non-NULL.
1561  // Otherwise it's the first instruction in the block.
1562  MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1563  if (SavedLastLocalValue)
1564  ++FirstDeadInst;
1565  else
1566  FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1567  setLastLocalValue(SavedLastLocalValue);
1568  removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1569  }
1570 }
1571 
1573  MachineInstr *SavedLastLocalValue = getLastLocalValue();
1574  // Just before the terminator instruction, insert instructions to
1575  // feed PHI nodes in successor blocks.
1576  if (isa<TerminatorInst>(I)) {
1577  if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1578  // PHI node handling may have generated local value instructions,
1579  // even though it failed to handle all PHI nodes.
1580  // We remove these instructions because SelectionDAGISel will generate
1581  // them again.
1582  removeDeadLocalValueCode(SavedLastLocalValue);
1583  return false;
1584  }
1585  }
1586 
1587  // FastISel does not handle any operand bundles except OB_funclet.
1589  for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
1590  if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1591  return false;
1592 
1593  DbgLoc = I->getDebugLoc();
1594 
1595  SavedInsertPt = FuncInfo.InsertPt;
1596 
1597  if (const auto *Call = dyn_cast<CallInst>(I)) {
1598  const Function *F = Call->getCalledFunction();
1599  LibFunc Func;
1600 
1601  // As a special case, don't handle calls to builtin library functions that
1602  // may be translated directly to target instructions.
1603  if (F && !F->hasLocalLinkage() && F->hasName() &&
1604  LibInfo->getLibFunc(F->getName(), Func) &&
1606  return false;
1607 
1608  // Don't handle Intrinsic::trap if a trap function is specified.
1609  if (F && F->getIntrinsicID() == Intrinsic::trap &&
1610  Call->hasFnAttr("trap-func-name"))
1611  return false;
1612  }
1613 
1614  // First, try doing target-independent selection.
1616  if (selectOperator(I, I->getOpcode())) {
1617  ++NumFastIselSuccessIndependent;
1618  DbgLoc = DebugLoc();
1619  return true;
1620  }
1621  // Remove dead code.
1623  if (SavedInsertPt != FuncInfo.InsertPt)
1624  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1625  SavedInsertPt = FuncInfo.InsertPt;
1626  }
1627  // Next, try calling the target to attempt to handle the instruction.
1628  if (fastSelectInstruction(I)) {
1629  ++NumFastIselSuccessTarget;
1630  DbgLoc = DebugLoc();
1631  return true;
1632  }
1633  // Remove dead code.
1635  if (SavedInsertPt != FuncInfo.InsertPt)
1636  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1637 
1638  DbgLoc = DebugLoc();
1639  // Undo phi node updates, because they will be added again by SelectionDAG.
1640  if (isa<TerminatorInst>(I)) {
1641  // PHI node handling may have generated local value instructions.
1642  // We remove them because SelectionDAGISel will generate them again.
1643  removeDeadLocalValueCode(SavedLastLocalValue);
1645  }
1646  return false;
1647 }
1648 
1649 /// Emit an unconditional branch to the given block, unless it is the immediate
1650 /// (fall-through) successor, and update the CFG.
1652  const DebugLoc &DbgLoc) {
1653  if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
1654  FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
1655  // For more accurate line information if this is the only instruction
1656  // in the block then emit it, otherwise we have the unconditional
1657  // fall-through case, which needs no instructions.
1658  } else {
1659  // The unconditional branch case.
1660  TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1661  SmallVector<MachineOperand, 0>(), DbgLoc);
1662  }
1663  if (FuncInfo.BPI) {
1665  FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
1667  } else
1669 }
1670 
1672  MachineBasicBlock *TrueMBB,
1673  MachineBasicBlock *FalseMBB) {
1674  // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1675  // happen in degenerate IR and MachineIR forbids to have a block twice in the
1676  // successor/predecessor lists.
1677  if (TrueMBB != FalseMBB) {
1678  if (FuncInfo.BPI) {
1679  auto BranchProbability =
1680  FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1682  } else
1684  }
1685 
1686  fastEmitBranch(FalseMBB, DbgLoc);
1687 }
1688 
1689 /// Emit an FNeg operation.
1691  unsigned OpReg = getRegForValue(BinaryOperator::getFNegArgument(I));
1692  if (!OpReg)
1693  return false;
1694  bool OpRegIsKill = hasTrivialKill(I);
1695 
1696  // If the target has ISD::FNEG, use it.
1697  EVT VT = TLI.getValueType(DL, I->getType());
1698  unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1699  OpReg, OpRegIsKill);
1700  if (ResultReg) {
1701  updateValueMap(I, ResultReg);
1702  return true;
1703  }
1704 
1705  // Bitcast the value to integer, twiddle the sign bit with xor,
1706  // and then bitcast it back to floating-point.
1707  if (VT.getSizeInBits() > 64)
1708  return false;
1709  EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1710  if (!TLI.isTypeLegal(IntVT))
1711  return false;
1712 
1713  unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1714  ISD::BITCAST, OpReg, OpRegIsKill);
1715  if (!IntReg)
1716  return false;
1717 
1718  unsigned IntResultReg = fastEmit_ri_(
1719  IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
1720  UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1721  if (!IntResultReg)
1722  return false;
1723 
1724  ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1725  IntResultReg, /*IsKill=*/true);
1726  if (!ResultReg)
1727  return false;
1728 
1729  updateValueMap(I, ResultReg);
1730  return true;
1731 }
1732 
1734  const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1735  if (!EVI)
1736  return false;
1737 
1738  // Make sure we only try to handle extracts with a legal result. But also
1739  // allow i1 because it's easy.
1740  EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1741  if (!RealVT.isSimple())
1742  return false;
1743  MVT VT = RealVT.getSimpleVT();
1744  if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1745  return false;
1746 
1747  const Value *Op0 = EVI->getOperand(0);
1748  Type *AggTy = Op0->getType();
1749 
1750  // Get the base result register.
1751  unsigned ResultReg;
1753  if (I != FuncInfo.ValueMap.end())
1754  ResultReg = I->second;
1755  else if (isa<Instruction>(Op0))
1756  ResultReg = FuncInfo.InitializeRegForValue(Op0);
1757  else
1758  return false; // fast-isel can't handle aggregate constants at the moment
1759 
1760  // Get the actual result register, which is an offset from the base register.
1761  unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1762 
1763  SmallVector<EVT, 4> AggValueVTs;
1764  ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1765 
1766  for (unsigned i = 0; i < VTIndex; i++)
1767  ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1768 
1769  updateValueMap(EVI, ResultReg);
1770  return true;
1771 }
1772 
1773 bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1774  switch (Opcode) {
1775  case Instruction::Add:
1776  return selectBinaryOp(I, ISD::ADD);
1777  case Instruction::FAdd:
1778  return selectBinaryOp(I, ISD::FADD);
1779  case Instruction::Sub:
1780  return selectBinaryOp(I, ISD::SUB);
1781  case Instruction::FSub:
1782  // FNeg is currently represented in LLVM IR as a special case of FSub.
1783  if (BinaryOperator::isFNeg(I))
1784  return selectFNeg(I);
1785  return selectBinaryOp(I, ISD::FSUB);
1786  case Instruction::Mul:
1787  return selectBinaryOp(I, ISD::MUL);
1788  case Instruction::FMul:
1789  return selectBinaryOp(I, ISD::FMUL);
1790  case Instruction::SDiv:
1791  return selectBinaryOp(I, ISD::SDIV);
1792  case Instruction::UDiv:
1793  return selectBinaryOp(I, ISD::UDIV);
1794  case Instruction::FDiv:
1795  return selectBinaryOp(I, ISD::FDIV);
1796  case Instruction::SRem:
1797  return selectBinaryOp(I, ISD::SREM);
1798  case Instruction::URem:
1799  return selectBinaryOp(I, ISD::UREM);
1800  case Instruction::FRem:
1801  return selectBinaryOp(I, ISD::FREM);
1802  case Instruction::Shl:
1803  return selectBinaryOp(I, ISD::SHL);
1804  case Instruction::LShr:
1805  return selectBinaryOp(I, ISD::SRL);
1806  case Instruction::AShr:
1807  return selectBinaryOp(I, ISD::SRA);
1808  case Instruction::And:
1809  return selectBinaryOp(I, ISD::AND);
1810  case Instruction::Or:
1811  return selectBinaryOp(I, ISD::OR);
1812  case Instruction::Xor:
1813  return selectBinaryOp(I, ISD::XOR);
1814 
1815  case Instruction::GetElementPtr:
1816  return selectGetElementPtr(I);
1817 
1818  case Instruction::Br: {
1819  const BranchInst *BI = cast<BranchInst>(I);
1820 
1821  if (BI->isUnconditional()) {
1822  const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1823  MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1824  fastEmitBranch(MSucc, BI->getDebugLoc());
1825  return true;
1826  }
1827 
1828  // Conditional branches are not handed yet.
1829  // Halt "fast" selection and bail.
1830  return false;
1831  }
1832 
1833  case Instruction::Unreachable:
1835  return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
1836  else
1837  return true;
1838 
1839  case Instruction::Alloca:
1840  // FunctionLowering has the static-sized case covered.
1841  if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1842  return true;
1843 
1844  // Dynamic-sized alloca is not handled yet.
1845  return false;
1846 
1847  case Instruction::Call:
1848  return selectCall(I);
1849 
1850  case Instruction::BitCast:
1851  return selectBitCast(I);
1852 
1853  case Instruction::FPToSI:
1854  return selectCast(I, ISD::FP_TO_SINT);
1855  case Instruction::ZExt:
1856  return selectCast(I, ISD::ZERO_EXTEND);
1857  case Instruction::SExt:
1858  return selectCast(I, ISD::SIGN_EXTEND);
1859  case Instruction::Trunc:
1860  return selectCast(I, ISD::TRUNCATE);
1861  case Instruction::SIToFP:
1862  return selectCast(I, ISD::SINT_TO_FP);
1863 
1864  case Instruction::IntToPtr: // Deliberate fall-through.
1865  case Instruction::PtrToInt: {
1866  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1867  EVT DstVT = TLI.getValueType(DL, I->getType());
1868  if (DstVT.bitsGT(SrcVT))
1869  return selectCast(I, ISD::ZERO_EXTEND);
1870  if (DstVT.bitsLT(SrcVT))
1871  return selectCast(I, ISD::TRUNCATE);
1872  unsigned Reg = getRegForValue(I->getOperand(0));
1873  if (!Reg)
1874  return false;
1875  updateValueMap(I, Reg);
1876  return true;
1877  }
1878 
1879  case Instruction::ExtractValue:
1880  return selectExtractValue(I);
1881 
1882  case Instruction::PHI:
1883  llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1884 
1885  default:
1886  // Unhandled instruction. Halt "fast" selection and bail.
1887  return false;
1888  }
1889 }
1890 
1892  const TargetLibraryInfo *LibInfo,
1894  : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1895  MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1896  TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1897  TII(*MF->getSubtarget().getInstrInfo()),
1898  TLI(*MF->getSubtarget().getTargetLowering()),
1899  TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1900  SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1901 
1902 FastISel::~FastISel() = default;
1903 
1904 bool FastISel::fastLowerArguments() { return false; }
1905 
1906 bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1907 
1909  return false;
1910 }
1911 
1912 unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1913 
1914 unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
1915  bool /*Op0IsKill*/) {
1916  return 0;
1917 }
1918 
1919 unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1920  bool /*Op0IsKill*/, unsigned /*Op1*/,
1921  bool /*Op1IsKill*/) {
1922  return 0;
1923 }
1924 
1925 unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1926  return 0;
1927 }
1928 
1929 unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1930  const ConstantFP * /*FPImm*/) {
1931  return 0;
1932 }
1933 
1934 unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1935  bool /*Op0IsKill*/, uint64_t /*Imm*/) {
1936  return 0;
1937 }
1938 
1939 /// This method is a wrapper of fastEmit_ri. It first tries to emit an
1940 /// instruction with an immediate operand using fastEmit_ri.
1941 /// If that fails, it materializes the immediate into a register and try
1942 /// fastEmit_rr instead.
1943 unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1944  bool Op0IsKill, uint64_t Imm, MVT ImmType) {
1945  // If this is a multiply by a power of two, emit this as a shift left.
1946  if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
1947  Opcode = ISD::SHL;
1948  Imm = Log2_64(Imm);
1949  } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
1950  // div x, 8 -> srl x, 3
1951  Opcode = ISD::SRL;
1952  Imm = Log2_64(Imm);
1953  }
1954 
1955  // Horrible hack (to be removed), check to make sure shift amounts are
1956  // in-range.
1957  if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1958  Imm >= VT.getSizeInBits())
1959  return 0;
1960 
1961  // First check if immediate type is legal. If not, we can't use the ri form.
1962  unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
1963  if (ResultReg)
1964  return ResultReg;
1965  unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1966  bool IsImmKill = true;
1967  if (!MaterialReg) {
1968  // This is a bit ugly/slow, but failing here means falling out of
1969  // fast-isel, which would be very slow.
1970  IntegerType *ITy =
1972  MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
1973  if (!MaterialReg)
1974  return 0;
1975  // FIXME: If the materialized register here has no uses yet then this
1976  // will be the first use and we should be able to mark it as killed.
1977  // However, the local value area for materialising constant expressions
1978  // grows down, not up, which means that any constant expressions we generate
1979  // later which also use 'Imm' could be after this instruction and therefore
1980  // after this kill.
1981  IsImmKill = false;
1982  }
1983  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
1984 }
1985 
1987  return MRI.createVirtualRegister(RC);
1988 }
1989 
1991  unsigned OpNum) {
1993  const TargetRegisterClass *RegClass =
1994  TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
1995  if (!MRI.constrainRegClass(Op, RegClass)) {
1996  // If it's not legal to COPY between the register classes, something
1997  // has gone very wrong before we got here.
1998  unsigned NewOp = createResultReg(RegClass);
2000  TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
2001  return NewOp;
2002  }
2003  }
2004  return Op;
2005 }
2006 
2007 unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2008  const TargetRegisterClass *RC) {
2009  unsigned ResultReg = createResultReg(RC);
2010  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2011 
2012  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
2013  return ResultReg;
2014 }
2015 
2016 unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2017  const TargetRegisterClass *RC, unsigned Op0,
2018  bool Op0IsKill) {
2019  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2020 
2021  unsigned ResultReg = createResultReg(RC);
2022  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2023 
2024  if (II.getNumDefs() >= 1)
2025  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2026  .addReg(Op0, getKillRegState(Op0IsKill));
2027  else {
2029  .addReg(Op0, getKillRegState(Op0IsKill));
2031  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2032  }
2033 
2034  return ResultReg;
2035 }
2036 
2037 unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2038  const TargetRegisterClass *RC, unsigned Op0,
2039  bool Op0IsKill, unsigned Op1,
2040  bool Op1IsKill) {
2041  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2042 
2043  unsigned ResultReg = createResultReg(RC);
2044  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2045  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2046 
2047  if (II.getNumDefs() >= 1)
2048  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2049  .addReg(Op0, getKillRegState(Op0IsKill))
2050  .addReg(Op1, getKillRegState(Op1IsKill));
2051  else {
2053  .addReg(Op0, getKillRegState(Op0IsKill))
2054  .addReg(Op1, getKillRegState(Op1IsKill));
2056  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2057  }
2058  return ResultReg;
2059 }
2060 
2061 unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2062  const TargetRegisterClass *RC, unsigned Op0,
2063  bool Op0IsKill, unsigned Op1,
2064  bool Op1IsKill, unsigned Op2,
2065  bool Op2IsKill) {
2066  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2067 
2068  unsigned ResultReg = createResultReg(RC);
2069  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2070  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2071  Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
2072 
2073  if (II.getNumDefs() >= 1)
2074  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2075  .addReg(Op0, getKillRegState(Op0IsKill))
2076  .addReg(Op1, getKillRegState(Op1IsKill))
2077  .addReg(Op2, getKillRegState(Op2IsKill));
2078  else {
2080  .addReg(Op0, getKillRegState(Op0IsKill))
2081  .addReg(Op1, getKillRegState(Op1IsKill))
2082  .addReg(Op2, getKillRegState(Op2IsKill));
2084  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2085  }
2086  return ResultReg;
2087 }
2088 
2089 unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2090  const TargetRegisterClass *RC, unsigned Op0,
2091  bool Op0IsKill, uint64_t Imm) {
2092  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2093 
2094  unsigned ResultReg = createResultReg(RC);
2095  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2096 
2097  if (II.getNumDefs() >= 1)
2098  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2099  .addReg(Op0, getKillRegState(Op0IsKill))
2100  .addImm(Imm);
2101  else {
2103  .addReg(Op0, getKillRegState(Op0IsKill))
2104  .addImm(Imm);
2106  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2107  }
2108  return ResultReg;
2109 }
2110 
2111 unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2112  const TargetRegisterClass *RC, unsigned Op0,
2113  bool Op0IsKill, uint64_t Imm1,
2114  uint64_t Imm2) {
2115  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2116 
2117  unsigned ResultReg = createResultReg(RC);
2118  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2119 
2120  if (II.getNumDefs() >= 1)
2121  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2122  .addReg(Op0, getKillRegState(Op0IsKill))
2123  .addImm(Imm1)
2124  .addImm(Imm2);
2125  else {
2127  .addReg(Op0, getKillRegState(Op0IsKill))
2128  .addImm(Imm1)
2129  .addImm(Imm2);
2131  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2132  }
2133  return ResultReg;
2134 }
2135 
2136 unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2137  const TargetRegisterClass *RC,
2138  const ConstantFP *FPImm) {
2139  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2140 
2141  unsigned ResultReg = createResultReg(RC);
2142 
2143  if (II.getNumDefs() >= 1)
2144  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2145  .addFPImm(FPImm);
2146  else {
2148  .addFPImm(FPImm);
2150  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2151  }
2152  return ResultReg;
2153 }
2154 
2155 unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2156  const TargetRegisterClass *RC, unsigned Op0,
2157  bool Op0IsKill, unsigned Op1,
2158  bool Op1IsKill, uint64_t Imm) {
2159  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2160 
2161  unsigned ResultReg = createResultReg(RC);
2162  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2163  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2164 
2165  if (II.getNumDefs() >= 1)
2166  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2167  .addReg(Op0, getKillRegState(Op0IsKill))
2168  .addReg(Op1, getKillRegState(Op1IsKill))
2169  .addImm(Imm);
2170  else {
2172  .addReg(Op0, getKillRegState(Op0IsKill))
2173  .addReg(Op1, getKillRegState(Op1IsKill))
2174  .addImm(Imm);
2176  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2177  }
2178  return ResultReg;
2179 }
2180 
2181 unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2182  const TargetRegisterClass *RC, uint64_t Imm) {
2183  unsigned ResultReg = createResultReg(RC);
2184  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2185 
2186  if (II.getNumDefs() >= 1)
2187  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2188  .addImm(Imm);
2189  else {
2192  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2193  }
2194  return ResultReg;
2195 }
2196 
2197 unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2198  bool Op0IsKill, uint32_t Idx) {
2199  unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2201  "Cannot yet extract from physregs");
2202  const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2204  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
2205  ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
2206  return ResultReg;
2207 }
2208 
2209 /// Emit MachineInstrs to compute the value of Op with all but the least
2210 /// significant bit set to zero.
2211 unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
2212  return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
2213 }
2214 
2215 /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2216 /// Emit code to ensure constants are copied into registers when needed.
2217 /// Remember the virtual registers that need to be added to the Machine PHI
2218 /// nodes as input. We cannot just directly add them, because expansion
2219 /// might result in multiple MBB's for one BB. As such, the start of the
2220 /// BB might correspond to a different MBB than the end.
2221 bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2222  const TerminatorInst *TI = LLVMBB->getTerminator();
2223 
2226 
2227  // Check successor nodes' PHI nodes that expect a constant to be available
2228  // from this block.
2229  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
2230  const BasicBlock *SuccBB = TI->getSuccessor(succ);
2231  if (!isa<PHINode>(SuccBB->begin()))
2232  continue;
2233  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2234 
2235  // If this terminator has multiple identical successors (common for
2236  // switches), only handle each succ once.
2237  if (!SuccsHandled.insert(SuccMBB).second)
2238  continue;
2239 
2240  MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2241 
2242  // At this point we know that there is a 1-1 correspondence between LLVM PHI
2243  // nodes and Machine PHI nodes, but the incoming operands have not been
2244  // emitted yet.
2245  for (const PHINode &PN : SuccBB->phis()) {
2246  // Ignore dead phi's.
2247  if (PN.use_empty())
2248  continue;
2249 
2250  // Only handle legal types. Two interesting things to note here. First,
2251  // by bailing out early, we may leave behind some dead instructions,
2252  // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2253  // own moves. Second, this check is necessary because FastISel doesn't
2254  // use CreateRegs to create registers, so it always creates
2255  // exactly one register for each non-void instruction.
2256  EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2257  if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2258  // Handle integer promotions, though, because they're common and easy.
2259  if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2261  return false;
2262  }
2263  }
2264 
2265  const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2266 
2267  // Set the DebugLoc for the copy. Prefer the location of the operand
2268  // if there is one; use the location of the PHI otherwise.
2269  DbgLoc = PN.getDebugLoc();
2270  if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2271  DbgLoc = Inst->getDebugLoc();
2272 
2273  unsigned Reg = getRegForValue(PHIOp);
2274  if (!Reg) {
2276  return false;
2277  }
2278  FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2279  DbgLoc = DebugLoc();
2280  }
2281  }
2282 
2283  return true;
2284 }
2285 
2286 bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2287  assert(LI->hasOneUse() &&
2288  "tryToFoldLoad expected a LoadInst with a single use");
2289  // We know that the load has a single use, but don't know what it is. If it
2290  // isn't one of the folded instructions, then we can't succeed here. Handle
2291  // this by scanning the single-use users of the load until we get to FoldInst.
2292  unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2293 
2294  const Instruction *TheUser = LI->user_back();
2295  while (TheUser != FoldInst && // Scan up until we find FoldInst.
2296  // Stay in the right block.
2297  TheUser->getParent() == FoldInst->getParent() &&
2298  --MaxUsers) { // Don't scan too far.
2299  // If there are multiple or no uses of this instruction, then bail out.
2300  if (!TheUser->hasOneUse())
2301  return false;
2302 
2303  TheUser = TheUser->user_back();
2304  }
2305 
2306  // If we didn't find the fold instruction, then we failed to collapse the
2307  // sequence.
2308  if (TheUser != FoldInst)
2309  return false;
2310 
2311  // Don't try to fold volatile loads. Target has to deal with alignment
2312  // constraints.
2313  if (LI->isVolatile())
2314  return false;
2315 
2316  // Figure out which vreg this is going into. If there is no assigned vreg yet
2317  // then there actually was no reference to it. Perhaps the load is referenced
2318  // by a dead instruction.
2319  unsigned LoadReg = getRegForValue(LI);
2320  if (!LoadReg)
2321  return false;
2322 
2323  // We can't fold if this vreg has no uses or more than one use. Multiple uses
2324  // may mean that the instruction got lowered to multiple MIs, or the use of
2325  // the loaded value ended up being multiple operands of the result.
2326  if (!MRI.hasOneUse(LoadReg))
2327  return false;
2328 
2330  MachineInstr *User = RI->getParent();
2331 
2332  // Set the insertion point properly. Folding the load can cause generation of
2333  // other random instructions (like sign extends) for addressing modes; make
2334  // sure they get inserted in a logical place before the new instruction.
2335  FuncInfo.InsertPt = User;
2336  FuncInfo.MBB = User->getParent();
2337 
2338  // Ask the target to try folding the load.
2339  return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2340 }
2341 
2343  // Must be an add.
2344  if (!isa<AddOperator>(Add))
2345  return false;
2346  // Type size needs to match.
2347  if (DL.getTypeSizeInBits(GEP->getType()) !=
2348  DL.getTypeSizeInBits(Add->getType()))
2349  return false;
2350  // Must be in the same basic block.
2351  if (isa<Instruction>(Add) &&
2352  FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2353  return false;
2354  // Must have a constant operand.
2355  return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2356 }
2357 
2360  const Value *Ptr;
2361  Type *ValTy;
2362  unsigned Alignment;
2364  bool IsVolatile;
2365 
2366  if (const auto *LI = dyn_cast<LoadInst>(I)) {
2367  Alignment = LI->getAlignment();
2368  IsVolatile = LI->isVolatile();
2369  Flags = MachineMemOperand::MOLoad;
2370  Ptr = LI->getPointerOperand();
2371  ValTy = LI->getType();
2372  } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2373  Alignment = SI->getAlignment();
2374  IsVolatile = SI->isVolatile();
2376  Ptr = SI->getPointerOperand();
2377  ValTy = SI->getValueOperand()->getType();
2378  } else
2379  return nullptr;
2380 
2381  bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
2382  bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
2383  bool IsDereferenceable =
2385  const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2386 
2387  AAMDNodes AAInfo;
2388  I->getAAMetadata(AAInfo);
2389 
2390  if (Alignment == 0) // Ensure that codegen never sees alignment 0.
2391  Alignment = DL.getABITypeAlignment(ValTy);
2392 
2393  unsigned Size = DL.getTypeStoreSize(ValTy);
2394 
2395  if (IsVolatile)
2397  if (IsNonTemporal)
2399  if (IsDereferenceable)
2401  if (IsInvariant)
2403 
2404  return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
2405  Alignment, AAInfo, Ranges);
2406 }
2407 
2409  // If both operands are the same, then try to optimize or fold the cmp.
2411  if (CI->getOperand(0) != CI->getOperand(1))
2412  return Predicate;
2413 
2414  switch (Predicate) {
2415  default: llvm_unreachable("Invalid predicate!");
2416  case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2417  case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2418  case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2419  case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2420  case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2421  case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2422  case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2423  case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2424  case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2425  case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2426  case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2427  case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2428  case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2429  case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2430  case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2431  case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2432 
2433  case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2434  case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2435  case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2436  case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2437  case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2438  case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2439  case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2440  case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2441  case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2442  case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2443  }
2444 
2445  return Predicate;
2446 }
void setHasStackMap(bool s=true)
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:539
unsigned fastEmitInst_rrr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, unsigned Op2, bool Op2IsKill)
Emit a MachineInstr with three register operands and a result register in the given register class...
Definition: FastISel.cpp:2061
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
void setByValAlign(unsigned A)
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
Function * getCalledFunction() const
Return the function called, or null if this is an indirect function invocation.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:875
unsigned fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill)
Emit MachineInstrs to compute the value of Op with all but the least significant bit set to zero...
Definition: FastISel.cpp:2211
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
bool hasLocalLinkage() const
Definition: GlobalValue.h:435
This instruction extracts a struct member or array element value from an aggregate value...
bool use_nodbg_empty(unsigned RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register...
MachineConstantPool & MCP
Definition: FastISel.h:208
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:30
instr_iterator instr_end()
bool lowerCall(const CallInst *I)
Definition: FastISel.cpp:1230
unsigned arg_size() const
Definition: CallSite.h:219
static const Value * getFNegArgument(const Value *BinOp)
bool hasDebugInfo() const
Returns true if valid debug info is present.
CallingConv::ID getCallingConv() const
Get the calling convention of the call.
Definition: CallSite.h:312
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
virtual unsigned fastMaterializeConstant(const Constant *C)
Emit a constant in a register using target-specific logic, such as constant pool loads.
Definition: FastISel.h:482
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:42
BasicBlock * getSuccessor(unsigned idx) const
Return the specified successor.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:137
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
bool selectXRayTypedEvent(const CallInst *II)
Definition: FastISel.cpp:1050
ImmutableCallSite * CS
Definition: FastISel.h:90
bool selectGetElementPtr(const User *I)
Definition: FastISel.cpp:665
void leaveLocalValueArea(SavePoint Old)
Reset InsertPt to the given old insert position.
Definition: FastISel.cpp:565
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:588
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:161
unsigned getReg() const
getReg - Returns the register number.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
bool selectStackmap(const CallInst *I)
Definition: FastISel.cpp:780
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:305
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:911
unsigned less than
Definition: InstrTypes.h:910
virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const
Insert branch code into the end of the specified MachineBasicBlock.
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:891
MachineMemOperand * createMachineMemOperandFor(const Instruction *I) const
Create a machine mem operand from the given instruction.
Definition: FastISel.cpp:2359
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:713
virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC, ArgListTy &Args) const
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:901
void addSuccessorWithoutProb(MachineBasicBlock *Succ)
Add Succ as a successor of this MachineBasicBlock.
virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode)
This method is called by target-independent code to request that an instruction with the given type a...
Definition: FastISel.cpp:1912
BasicBlock * getSuccessor(unsigned i) const
arg_iterator arg_end()
Definition: Function.h:666
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
STATISTIC(NumFunctions, "Total number of functions")
A debug info location.
Definition: DebugLoc.h:34
Metadata node.
Definition: Metadata.h:862
F(f)
MachineModuleInfo & getMMI() const
SmallVector< unsigned, 4 > InRegs
Definition: FastISel.h:99
unsigned getCallFrameDestroyOpcode() const
An instruction for reading from memory.
Definition: Instructions.h:164
Hexagon Common GEP
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - Get or set the calling convention of this function call...
bool CanLowerReturn
CanLowerReturn - true iff the function&#39;s return value can be lowered to registers.
virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1925
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:361
virtual unsigned fastMaterializeFloatZero(const ConstantFP *CF)
Emit the floating-point constant +0.0 in a register using target- specific logic. ...
Definition: FastISel.h:489
void setPhysRegsDeadExcept(ArrayRef< unsigned > UsedRegs, const TargetRegisterInfo &TRI)
Mark every physreg used by this instruction as dead except those in the UsedRegs list.
virtual unsigned getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Return the desired alignment for ByVal or InAlloca aggregate function arguments in the caller paramet...
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
void computeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo &MMI)
Determine if any floating-point values are being passed to this variadic function, and set the MachineModuleInfo&#39;s usesVAFloatArgument flag if so.
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1906
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:130
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:258
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:264
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1572
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
unsigned fastEmitInst_rii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm1, uint64_t Imm2)
Emit a MachineInstr with one register operand and two immediate operands.
Definition: FastISel.cpp:2111
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1069
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:896
MachineFunction * MF
Definition: FastISel.h:205
DenseMap< const Value *, unsigned > LocalValueMap
Definition: FastISel.h:203
unsigned fastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm)
Emit a MachineInstr with a register operand, an immediate, and a result register in the given registe...
Definition: FastISel.cpp:2089
ArrayRef< unsigned > getIndices() const
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:242
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:895
unsigned fastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm)
Emit a MachineInstr with two register operands, an immediate, and a result register in the given regi...
Definition: FastISel.cpp:2155
CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const
Definition: FastISel.cpp:2408
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:217
A description of a memory reference used in the backend.
void setHasPatchPoint(bool s=true)
TargetLoweringBase::ArgListTy ArgListTy
Definition: FastISel.h:70
Shift and rotation operations.
Definition: ISDOpcodes.h:380
Class to represent struct types.
Definition: DerivedTypes.h:201
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
unsigned fastEmitInst_i(unsigned MachineInstrOpcode, const TargetRegisterClass *RC, uint64_t Imm)
Emit a MachineInstr with a single immediate operand, and a result register in the given register clas...
Definition: FastISel.cpp:2181
bool canFoldAddIntoGEP(const User *GEP, const Value *Add)
Check if Add is an add that can be safely folded into GEP.
Definition: FastISel.cpp:2342
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i...
IterTy arg_end() const
Definition: CallSite.h:575
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned fastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill)
Emit a MachineInstr with one register operand and a result register in the given register class...
Definition: FastISel.cpp:2016
MachineInstr * EmitStartPt
The top most instruction in the current block that is allowed for emitting local variables.
Definition: FastISel.h:227
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:892
This file contains the simple types necessary to represent the attributes associated with functions a...
InstrTy * getInstruction() const
Definition: CallSite.h:92
The memory access is dereferenceable (i.e., doesn&#39;t trap).
static MachineOperand CreateRegMask(const uint32_t *Mask)
CreateRegMask - Creates a register mask operand referencing Mask.
virtual const TargetRegisterClass * getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const
Returns the largest legal sub-class of RC that supports the sub-register index Idx.
void setByValSize(unsigned S)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock::iterator LastFlushPoint
Last local value flush point.
Definition: FastISel.h:231
const TargetMachine & TM
Definition: FastISel.h:210
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:628
bool selectIntrinsicCall(const IntrinsicInst *II)
Definition: FastISel.cpp:1315
bool selectCast(const User *I, unsigned Opcode)
Definition: FastISel.cpp:1471
unsigned getSizeInBits() const
MachineInstr * getVRegDef(unsigned Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Context object for machine code objects.
Definition: MCContext.h:63
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
SmallVector< ISD::InputArg, 4 > Ins
Definition: FastISel.h:98
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
unsigned constrainOperandRegClass(const MCInstrDesc &II, unsigned Op, unsigned OpNum)
Try to constrain Op so that it is usable by argument OpNum of the provided MCInstrDesc.
Definition: FastISel.cpp:1990
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:448
bool selectOperator(const User *I, unsigned Opcode)
Do "fast" instruction selection for the given LLVM IR operator (Instruction or ConstantExpr), and append generated machine instructions to the current block.
Definition: FastISel.cpp:1773
ArchType getArch() const
getArch - Get the parsed architecture type of this triple.
Definition: Triple.h:283
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
unsigned getRegForValue(const Value *V)
Create a virtual register and arrange for it to be assigned the value for the given LLVM value...
Definition: FastISel.cpp:361
const TargetRegisterClass * getRegClass(const MCInstrDesc &TID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
DenseSet< unsigned > RegsWithFixups
unsigned fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC)
Emit a MachineInstr with no operands and a result register in the given register class.
Definition: FastISel.cpp:2007
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:195
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:126
bool hasTrivialKill(const Value *V)
Test whether the given value has exactly one use.
Definition: FastISel.cpp:329
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void setOrigAlign(unsigned A)
reverse_iterator rend()
amdgpu Simplify well known AMD library false Value * Callee
MachineInstr * getLastLocalValue()
Return the position of the last instruction emitted for materializing constants for use in the curren...
Definition: FastISel.h:238
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:84
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
Value * getOperand(unsigned i) const
Definition: User.h:170
Class to represent pointers.
Definition: DerivedTypes.h:467
unsigned getKillRegState(bool B)
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:492
unsigned lookUpRegForValue(const Value *V)
Look up the value to see if its value is already cached in a register.
Definition: FastISel.cpp:474
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
void finishBasicBlock()
Flush the local value map and sink local values if possible.
Definition: FastISel.cpp:143
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:146
Value * getOperand(unsigned i_nocapture) const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:229
MCContext & getContext() const
void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:141
The memory access is volatile.
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
Definition: DataLayout.cpp:742
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:410
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function...
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:155
virtual ~FastISel()
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:55
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
CallLoweringInfo & setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, ImmutableCallSite &Call)
Definition: FastISel.h:105
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Machine Value Type.
bool hasName() const
Definition: Value.h:251
LLVM Basic Block Representation.
Definition: BasicBlock.h:59
const MachineInstrBuilder & addCImm(const ConstantInt *Val) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:260
Conditional or Unconditional Branch instruction.
Value * getAddress() const
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:42
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:543
Value * getValue() const
SmallVector< ISD::ArgFlagsTy, 16 > OutFlags
Definition: FastISel.h:96
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:187
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:264
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:371
const MCPhysReg * ImplicitDefs
Definition: MCInstrDesc.h:171
size_t size() const
Definition: BasicBlock.h:274
MachineFrameInfo & MFI
Definition: FastISel.h:207
virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1914
bool SkipTargetIndependentISel
Definition: FastISel.h:216
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
unsigned getCallFrameSetupOpcode() const
These methods return the opcode of the frame setup/destroy instructions if they exist (-1 otherwise)...
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:426
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:161
DILocalVariable * getVariable() const
Definition: IntrinsicInst.h:80
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We&#39;re checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2286
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:145
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:885
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned char TargetFlags=0)
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:728
const Triple & getTargetTriple() const
DIExpression * getExpression() const
Definition: IntrinsicInst.h:84
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:894
arg_iterator arg_begin()
Definition: Function.h:657
self_iterator getIterator()
Definition: ilist_node.h:82
The memory access is non-temporal.
Class to represent integer types.
Definition: DerivedTypes.h:40
bool selectXRayCustomEvent(const CallInst *II)
Definition: FastISel.cpp:1031
const TargetRegisterInfo & TRI
Definition: FastISel.h:214
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:902
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:193
static unsigned findSinkableLocalRegDef(MachineInstr &MI)
Return the defined register if this instruction defines exactly one virtual register and uses no othe...
Definition: FastISel.cpp:167
Extended Value Type.
Definition: ValueTypes.h:34
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg) const
For some targets, an LLVM struct type must be broken down into multiple simple types, but the calling convention specifies that the entire struct must be passed in a block of consecutive registers.
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:538
bool selectFNeg(const User *I)
Emit an FNeg operation.
Definition: FastISel.cpp:1690
This class contains a discriminated union of information about pointers in memory operands...
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:900
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
SmallVector< Value *, 16 > OutVals
Definition: FastISel.h:95
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call...
Definition: FastISel.cpp:1073
const TargetInstrInfo & TII
Definition: FastISel.h:212
MachineBasicBlock * MBB
MBB - The current block.
bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:471
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
signed greater than
Definition: InstrTypes.h:912
MachineInstr * LastLocalValue
The position of the last instruction for materializing constants for use in the current block...
Definition: FastISel.h:222
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:633
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge&#39;s probability, relative to other out-edges of the Src.
virtual const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const
Returns a 0 terminated array of registers that can be safely used as scratch registers.
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:529
The memory access writes data.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:51
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:889
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:240
virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1919
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:418
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
This is the shared class of boolean and integer constants.
Definition: Constants.h:84
virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1934
DenseMap< unsigned, unsigned > RegFixups
RegFixups - Registers which need to be replaced after isel is done.
IterTy arg_begin() const
Definition: CallSite.h:571
MachineOperand class - Representation of each machine instruction operand.
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:899
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:64
Provides information about what library functions are available for the current target.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:27
void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB)
Emit an unconditional branch to FalseMBB, obtains the branch weight and adds TrueMBB and FalseMBB to ...
Definition: FastISel.cpp:1671
const TargetLibraryInfo * LibInfo
Definition: FastISel.h:215
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:722
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:570
signed less than
Definition: InstrTypes.h:914
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:642
reg_iterator reg_begin(unsigned RegNo) const
unsigned fastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
Emit a MachineInstr with two register operands and a result register in the given register class...
Definition: FastISel.cpp:2037
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:611
void updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs=1)
Update the value map to include the new mapping for this instruction, or insert an extra copy to get ...
Definition: FastISel.cpp:485
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:223
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:180
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
Definition: FastISel.cpp:129
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
signed less or equal
Definition: InstrTypes.h:915
bool selectBitCast(const User *I)
Definition: FastISel.cpp:1504
Target - Wrapper for Target specific information.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1929
SmallVector< unsigned, 16 > OutRegs
Definition: FastISel.h:97
const DataLayout & DL
Definition: FastISel.h:211
bool selectBinaryOp(const User *I, unsigned ISDOpcode)
Select and emit code for a binary operator instruction, which has an opcode which directly correspond...
Definition: FastISel.cpp:574
void setDebugLoc(DebugLoc dl)
Replace current source information with new such.
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringRef > StandardNames)
Initialize the set of available library functions based on the specified target triple.
BranchProbabilityInfo * BPI
This file defines the FastISel class.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:438
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
bool use_empty(unsigned RegNo) const
use_empty - Return true if there are no instructions using the specified register.
bool isTailCall() const
DebugLoc DbgLoc
Definition: FastISel.h:209
bool selectCall(const User *Call)
Definition: FastISel.cpp:1269
Flags
Flags values. These may be or&#39;d together.
amdgpu Simplify well known AMD library false Value Value * Arg
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:156
The memory access reads data.
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:560
SavePoint enterLocalValueArea()
Prepare InsertPt to begin inserting instructions into the local value area and return the old insert ...
Definition: FastISel.cpp:556
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:428
Representation of each machine instruction.
Definition: MachineInstr.h:60
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:959
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering...
Definition: FastISel.cpp:1908
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
bool selectPatchpoint(const CallInst *I)
Definition: FastISel.cpp:893
bool selectExtractValue(const User *I)
Definition: FastISel.cpp:1733
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:285
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:363
MachineRegisterInfo & MRI
Definition: FastISel.h:206
bool hasOneUse(unsigned RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
uint64_t getElementOffset(unsigned Idx) const
Definition: DataLayout.h:543
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:123
unsigned getNumArgOperands() const
Return the number of call arguments.
unsigned greater or equal
Definition: InstrTypes.h:909
This represents the llvm.dbg.value instruction.
bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs)
Definition: FastISel.cpp:1095
static bool isFNeg(const Value *V, bool IgnoreZeroSign=false)
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:224
TargetOptions Options
Definition: TargetMachine.h:98
Establish a view to a call site for examination.
Definition: CallSite.h:713
static MachineOperand CreateImm(int64_t Val)
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
The memory access always returns the same value (or traps).
virtual unsigned fastMaterializeAlloca(const AllocaInst *C)
Emit an alloca address in a register using target-specific logic.
Definition: FastISel.h:485
iterator end()
Definition: DenseMap.h:79
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:193
static bool isRegUsedByPhiNodes(unsigned DefReg, FunctionLoweringInfo &FuncInfo)
Definition: FastISel.cpp:219
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:893
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:320
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:91
bool isUnconditional() const
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block...
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:897
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:309
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
LLVM_NODISCARD bool empty() const
Definition: DenseMap.h:93
Type * getType() const
Return the type of the instruction that generated this call site.
Definition: CallSite.h:264
const TargetLowering & TLI
Definition: FastISel.h:213
bool isReg() const
isReg - Tests if this is a MO_Register operand.
unsigned createResultReg(const TargetRegisterClass *RC)
Definition: FastISel.cpp:1986
CallLoweringInfo & setIsPatchPoint(bool Value=true)
Definition: FastISel.h:183
unsigned fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm, MVT ImmType)
This method is a wrapper of fastEmit_ri.
Definition: FastISel.cpp:1943
iterator_range< use_instr_iterator > use_instructions(unsigned Reg) const
unsigned fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, bool Op0IsKill, uint32_t Idx)
Emit a MachineInstr for an extract_subreg from a specified index of a superregister to a specified ty...
Definition: FastISel.cpp:2197
MachineBasicBlock::iterator InsertPt
Definition: FastISel.h:319
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void GetReturnInfo(Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
user_iterator user_begin()
Definition: Value.h:375
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, bool SkipTargetIndependentISel=false)
Definition: FastISel.cpp:1891
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th call argument.
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1904
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:888
LLVM Value Representation.
Definition: Value.h:73
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:898
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:411
FunctionType * getFunctionType() const
Definition: CallSite.h:320
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
static const Function * getParent(const Value *V)
const Value * getCalledValue() const
Get a pointer to the function that is invoked by this instruction.
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable&#39;s name.
Definition: Mangler.cpp:112
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
IRTranslator LLVM IR MI
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:412
unsigned greater than
Definition: InstrTypes.h:908
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
unsigned TrapUnreachable
Emit target-specific trap instruction for &#39;unreachable&#39; IR instructions.
virtual bool fastSelectInstruction(const Instruction *I)=0
This method is called by target-independent code when the normal FastISel process fails to select an ...
static cl::opt< bool > SinkLocalValues("fast-isel-sink-local-values", cl::init(true), cl::Hidden, cl::desc("Sink local values in FastISel"))
unsigned fastEmitInst_f(unsigned MachineInstOpcode, const TargetRegisterClass *RC, const ConstantFP *FPImm)
Emit a MachineInstr with a floating point immediate, and a result register in the given register clas...
Definition: FastISel.cpp:2136
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty...
Definition: Type.cpp:98
Conversion operators.
Definition: ISDOpcodes.h:435
const TerminatorInst * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:138
FunctionLoweringInfo & FuncInfo
Definition: FastISel.h:204
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
Definition: Value.cpp:546
void setIsDebug(bool Val=true)
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:444
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:126
#define LLVM_DEBUG(X)
Definition: Debug.h:119
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(unsigned Reg) const
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:890
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:36
void fastEmitBranch(MachineBasicBlock *MBB, const DebugLoc &DL)
Emit an unconditional branch to the given block, unless it is the immediate (fall-through) successor...
Definition: FastISel.cpp:1651
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
This represents the llvm.dbg.declare instruction.
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:555
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
static MachineOperand CreateFI(int Idx)
bool use_empty() const
Definition: Value.h:322
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:537
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Type * getElementType() const
Definition: DerivedTypes.h:486
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:871
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:887
signed greater or equal
Definition: InstrTypes.h:913
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
This class contains meta information specific to a module.
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:67
unsigned InitializeRegForValue(const Value *V)
gep_type_iterator gep_type_begin(const User *GEP)
std::pair< unsigned, bool > getRegForGEPIndex(const Value *V)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
Definition: FastISel.cpp:506