LLVM  8.0.0svn
FastISel.cpp
Go to the documentation of this file.
1 //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the implementation of the FastISel class.
11 //
12 // "Fast" instruction selection is designed to emit very poor code quickly.
13 // Also, it is not designed to be able to do much lowering, so most illegal
14 // types (e.g. i64 on 32-bit targets) and operations are not supported. It is
15 // also not intended to be able to do much optimization, except in a few cases
16 // where doing optimizations reduces overall compile time. For example, folding
17 // constants into immediate fields is often done, because it's cheap and it
18 // reduces the number of instructions later phases have to examine.
19 //
20 // "Fast" instruction selection is able to fail gracefully and transfer
21 // control to the SelectionDAG selector for operations that it doesn't
22 // support. In many cases, this allows us to avoid duplicating a lot of
23 // the complicated lowering logic that SelectionDAG currently has.
24 //
25 // The intended use for "fast" instruction selection is "-O0" mode
26 // compilation, where the quality of the generated code is irrelevant when
27 // weighed against the speed at which the code can be generated. Also,
28 // at -O0, the LLVM optimizers are not running, and this makes the
29 // compile time of codegen a much higher portion of the overall compile
30 // time. Despite its limitations, "fast" instruction selection is able to
31 // handle enough code on its own to provide noticeable overall speedups
32 // in -O0 compiles.
33 //
34 // Basic operations are supported in a target-independent way, by reading
35 // the same instruction descriptions that the SelectionDAG selector reads,
36 // and identifying simple arithmetic operations that can be directly selected
37 // from simple operators. More complicated operations currently require
38 // target-specific code.
39 //
40 //===----------------------------------------------------------------------===//
41 
42 #include "llvm/CodeGen/FastISel.h"
43 #include "llvm/ADT/APFloat.h"
44 #include "llvm/ADT/APSInt.h"
45 #include "llvm/ADT/DenseMap.h"
46 #include "llvm/ADT/Optional.h"
47 #include "llvm/ADT/SmallPtrSet.h"
48 #include "llvm/ADT/SmallString.h"
49 #include "llvm/ADT/SmallVector.h"
50 #include "llvm/ADT/Statistic.h"
53 #include "llvm/CodeGen/Analysis.h"
64 #include "llvm/CodeGen/StackMaps.h"
69 #include "llvm/IR/Argument.h"
70 #include "llvm/IR/Attributes.h"
71 #include "llvm/IR/BasicBlock.h"
72 #include "llvm/IR/CallSite.h"
73 #include "llvm/IR/CallingConv.h"
74 #include "llvm/IR/Constant.h"
75 #include "llvm/IR/Constants.h"
76 #include "llvm/IR/DataLayout.h"
77 #include "llvm/IR/DebugInfo.h"
78 #include "llvm/IR/DebugLoc.h"
79 #include "llvm/IR/DerivedTypes.h"
80 #include "llvm/IR/Function.h"
82 #include "llvm/IR/GlobalValue.h"
83 #include "llvm/IR/InlineAsm.h"
84 #include "llvm/IR/InstrTypes.h"
85 #include "llvm/IR/Instruction.h"
86 #include "llvm/IR/Instructions.h"
87 #include "llvm/IR/IntrinsicInst.h"
88 #include "llvm/IR/LLVMContext.h"
89 #include "llvm/IR/Mangler.h"
90 #include "llvm/IR/Metadata.h"
91 #include "llvm/IR/Operator.h"
92 #include "llvm/IR/PatternMatch.h"
93 #include "llvm/IR/Type.h"
94 #include "llvm/IR/User.h"
95 #include "llvm/IR/Value.h"
96 #include "llvm/MC/MCContext.h"
97 #include "llvm/MC/MCInstrDesc.h"
98 #include "llvm/MC/MCRegisterInfo.h"
99 #include "llvm/Support/Casting.h"
100 #include "llvm/Support/Debug.h"
103 #include "llvm/Support/MathExtras.h"
107 #include <algorithm>
108 #include <cassert>
109 #include <cstdint>
110 #include <iterator>
111 #include <utility>
112 
113 using namespace llvm;
114 using namespace PatternMatch;
115 
116 #define DEBUG_TYPE "isel"
117 
118 // FIXME: Remove this after the feature has proven reliable.
119 static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values",
120  cl::init(true), cl::Hidden,
121  cl::desc("Sink local values in FastISel"));
122 
123 STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
124  "target-independent selector");
125 STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
126  "target-specific selector");
127 STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
128 
129 /// Set the current block to which generated machine instructions will be
130 /// appended.
132  assert(LocalValueMap.empty() &&
133  "local values should be cleared after finishing a BB");
134 
135  // Instructions are appended to FuncInfo.MBB. If the basic block already
136  // contains labels or copies, use the last instruction as the last local
137  // value.
138  EmitStartPt = nullptr;
139  if (!FuncInfo.MBB->empty())
140  EmitStartPt = &FuncInfo.MBB->back();
141  LastLocalValue = EmitStartPt;
142 }
143 
144 /// Flush the local CSE map and sink anything we can.
145 void FastISel::finishBasicBlock() { flushLocalValueMap(); }
146 
148  if (!FuncInfo.CanLowerReturn)
149  // Fallback to SDISel argument lowering code to deal with sret pointer
150  // parameter.
151  return false;
152 
153  if (!fastLowerArguments())
154  return false;
155 
156  // Enter arguments into ValueMap for uses in non-entry BBs.
157  for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
158  E = FuncInfo.Fn->arg_end();
159  I != E; ++I) {
160  DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(&*I);
161  assert(VI != LocalValueMap.end() && "Missed an argument?");
162  FuncInfo.ValueMap[&*I] = VI->second;
163  }
164  return true;
165 }
166 
167 /// Return the defined register if this instruction defines exactly one
168 /// virtual register and uses no other virtual registers. Otherwise return 0.
170  unsigned RegDef = 0;
171  for (const MachineOperand &MO : MI.operands()) {
172  if (!MO.isReg())
173  continue;
174  if (MO.isDef()) {
175  if (RegDef)
176  return 0;
177  RegDef = MO.getReg();
178  } else if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
179  // This is another use of a vreg. Don't try to sink it.
180  return 0;
181  }
182  }
183  return RegDef;
184 }
185 
186 void FastISel::flushLocalValueMap() {
187  // Try to sink local values down to their first use so that we can give them a
188  // better debug location. This has the side effect of shrinking local value
189  // live ranges, which helps out fast regalloc.
190  if (SinkLocalValues && LastLocalValue != EmitStartPt) {
191  // Sink local value materialization instructions between EmitStartPt and
192  // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
193  // avoid inserting into the range that we're iterating over.
195  EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
196  : FuncInfo.MBB->rend();
197  MachineBasicBlock::reverse_iterator RI(LastLocalValue);
198 
199  InstOrderMap OrderMap;
200  for (; RI != RE;) {
201  MachineInstr &LocalMI = *RI;
202  ++RI;
203  bool Store = true;
204  if (!LocalMI.isSafeToMove(nullptr, Store))
205  continue;
206  unsigned DefReg = findSinkableLocalRegDef(LocalMI);
207  if (DefReg == 0)
208  continue;
209 
210  sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
211  }
212  }
213 
214  LocalValueMap.clear();
215  LastLocalValue = EmitStartPt;
216  recomputeInsertPt();
217  SavedInsertPt = FuncInfo.InsertPt;
218  LastFlushPoint = FuncInfo.InsertPt;
219 }
220 
221 static bool isRegUsedByPhiNodes(unsigned DefReg,
222  FunctionLoweringInfo &FuncInfo) {
223  for (auto &P : FuncInfo.PHINodesToUpdate)
224  if (P.second == DefReg)
225  return true;
226  return false;
227 }
228 
229 /// Build a map of instruction orders. Return the first terminator and its
230 /// order. Consider EH_LABEL instructions to be terminators as well, since local
231 /// values for phis after invokes must be materialized before the call.
233  MachineBasicBlock *MBB, MachineBasicBlock::iterator LastFlushPoint) {
234  unsigned Order = 0;
235  for (MachineInstr &I : *MBB) {
236  if (!FirstTerminator &&
237  (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) {
238  FirstTerminator = &I;
239  FirstTerminatorOrder = Order;
240  }
241  Orders[&I] = Order++;
242 
243  // We don't need to order instructions past the last flush point.
244  if (I.getIterator() == LastFlushPoint)
245  break;
246  }
247 }
248 
249 void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
250  unsigned DefReg,
251  InstOrderMap &OrderMap) {
252  // If this register is used by a register fixup, MRI will not contain all
253  // the uses until after register fixups, so don't attempt to sink or DCE
254  // this instruction. Register fixups typically come from no-op cast
255  // instructions, which replace the cast instruction vreg with the local
256  // value vreg.
257  if (FuncInfo.RegsWithFixups.count(DefReg))
258  return;
259 
260  // We can DCE this instruction if there are no uses and it wasn't a
261  // materialized for a successor PHI node.
262  bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
263  if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
264  if (EmitStartPt == &LocalMI)
265  EmitStartPt = EmitStartPt->getPrevNode();
266  LLVM_DEBUG(dbgs() << "removing dead local value materialization "
267  << LocalMI);
268  OrderMap.Orders.erase(&LocalMI);
269  LocalMI.eraseFromParent();
270  return;
271  }
272 
273  // Number the instructions if we haven't yet so we can efficiently find the
274  // earliest use.
275  if (OrderMap.Orders.empty())
276  OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
277 
278  // Find the first user in the BB.
279  MachineInstr *FirstUser = nullptr;
280  unsigned FirstOrder = std::numeric_limits<unsigned>::max();
281  for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
282  auto I = OrderMap.Orders.find(&UseInst);
283  assert(I != OrderMap.Orders.end() &&
284  "local value used by instruction outside local region");
285  unsigned UseOrder = I->second;
286  if (UseOrder < FirstOrder) {
287  FirstOrder = UseOrder;
288  FirstUser = &UseInst;
289  }
290  }
291 
292  // The insertion point will be the first terminator or the first user,
293  // whichever came first. If there was no terminator, this must be a
294  // fallthrough block and the insertion point is the end of the block.
296  if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder) {
297  FirstOrder = OrderMap.FirstTerminatorOrder;
298  SinkPos = OrderMap.FirstTerminator->getIterator();
299  } else if (FirstUser) {
300  SinkPos = FirstUser->getIterator();
301  } else {
302  assert(UsedByPHI && "must be users if not used by a phi");
303  SinkPos = FuncInfo.MBB->instr_end();
304  }
305 
306  // Collect all DBG_VALUEs before the new insertion position so that we can
307  // sink them.
309  for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
310  if (!DbgVal.isDebugValue())
311  continue;
312  unsigned UseOrder = OrderMap.Orders[&DbgVal];
313  if (UseOrder < FirstOrder)
314  DbgValues.push_back(&DbgVal);
315  }
316 
317  // Sink LocalMI before SinkPos and assign it the same DebugLoc.
318  LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
319  FuncInfo.MBB->remove(&LocalMI);
320  FuncInfo.MBB->insert(SinkPos, &LocalMI);
321  if (SinkPos != FuncInfo.MBB->end())
322  LocalMI.setDebugLoc(SinkPos->getDebugLoc());
323 
324  // Sink any debug values that we've collected.
325  for (MachineInstr *DI : DbgValues) {
326  FuncInfo.MBB->remove(DI);
327  FuncInfo.MBB->insert(SinkPos, DI);
328  }
329 }
330 
332  // Don't consider constants or arguments to have trivial kills.
333  const Instruction *I = dyn_cast<Instruction>(V);
334  if (!I)
335  return false;
336 
337  // No-op casts are trivially coalesced by fast-isel.
338  if (const auto *Cast = dyn_cast<CastInst>(I))
339  if (Cast->isNoopCast(DL) && !hasTrivialKill(Cast->getOperand(0)))
340  return false;
341 
342  // Even the value might have only one use in the LLVM IR, it is possible that
343  // FastISel might fold the use into another instruction and now there is more
344  // than one use at the Machine Instruction level.
345  unsigned Reg = lookUpRegForValue(V);
346  if (Reg && !MRI.use_empty(Reg))
347  return false;
348 
349  // GEPs with all zero indices are trivially coalesced by fast-isel.
350  if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
351  if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0)))
352  return false;
353 
354  // Only instructions with a single use in the same basic block are considered
355  // to have trivial kills.
356  return I->hasOneUse() &&
357  !(I->getOpcode() == Instruction::BitCast ||
358  I->getOpcode() == Instruction::PtrToInt ||
359  I->getOpcode() == Instruction::IntToPtr) &&
360  cast<Instruction>(*I->user_begin())->getParent() == I->getParent();
361 }
362 
363 unsigned FastISel::getRegForValue(const Value *V) {
364  EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
365  // Don't handle non-simple values in FastISel.
366  if (!RealVT.isSimple())
367  return 0;
368 
369  // Ignore illegal types. We must do this before looking up the value
370  // in ValueMap because Arguments are given virtual registers regardless
371  // of whether FastISel can handle them.
372  MVT VT = RealVT.getSimpleVT();
373  if (!TLI.isTypeLegal(VT)) {
374  // Handle integer promotions, though, because they're common and easy.
375  if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
376  VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
377  else
378  return 0;
379  }
380 
381  // Look up the value to see if we already have a register for it.
382  unsigned Reg = lookUpRegForValue(V);
383  if (Reg)
384  return Reg;
385 
386  // In bottom-up mode, just create the virtual register which will be used
387  // to hold the value. It will be materialized later.
388  if (isa<Instruction>(V) &&
389  (!isa<AllocaInst>(V) ||
390  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
391  return FuncInfo.InitializeRegForValue(V);
392 
393  SavePoint SaveInsertPt = enterLocalValueArea();
394 
395  // Materialize the value in a register. Emit any instructions in the
396  // local value area.
397  Reg = materializeRegForValue(V, VT);
398 
399  leaveLocalValueArea(SaveInsertPt);
400 
401  return Reg;
402 }
403 
404 unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
405  unsigned Reg = 0;
406  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
407  if (CI->getValue().getActiveBits() <= 64)
408  Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
409  } else if (isa<AllocaInst>(V))
410  Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
411  else if (isa<ConstantPointerNull>(V))
412  // Translate this as an integer zero so that it can be
413  // local-CSE'd with actual integer zeros.
414  Reg = getRegForValue(
415  Constant::getNullValue(DL.getIntPtrType(V->getContext())));
416  else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
417  if (CF->isNullValue())
418  Reg = fastMaterializeFloatZero(CF);
419  else
420  // Try to emit the constant directly.
421  Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
422 
423  if (!Reg) {
424  // Try to emit the constant by using an integer constant with a cast.
425  const APFloat &Flt = CF->getValueAPF();
426  EVT IntVT = TLI.getPointerTy(DL);
427  uint32_t IntBitWidth = IntVT.getSizeInBits();
428  APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
429  bool isExact;
430  (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
431  if (isExact) {
432  unsigned IntegerReg =
433  getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
434  if (IntegerReg != 0)
435  Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
436  /*Kill=*/false);
437  }
438  }
439  } else if (const auto *Op = dyn_cast<Operator>(V)) {
440  if (!selectOperator(Op, Op->getOpcode()))
441  if (!isa<Instruction>(Op) ||
442  !fastSelectInstruction(cast<Instruction>(Op)))
443  return 0;
444  Reg = lookUpRegForValue(Op);
445  } else if (isa<UndefValue>(V)) {
446  Reg = createResultReg(TLI.getRegClassFor(VT));
447  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
448  TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
449  }
450  return Reg;
451 }
452 
453 /// Helper for getRegForValue. This function is called when the value isn't
454 /// already available in a register and must be materialized with new
455 /// instructions.
456 unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
457  unsigned Reg = 0;
458  // Give the target-specific code a try first.
459  if (isa<Constant>(V))
460  Reg = fastMaterializeConstant(cast<Constant>(V));
461 
462  // If target-specific code couldn't or didn't want to handle the value, then
463  // give target-independent code a try.
464  if (!Reg)
465  Reg = materializeConstant(V, VT);
466 
467  // Don't cache constant materializations in the general ValueMap.
468  // To do so would require tracking what uses they dominate.
469  if (Reg) {
470  LocalValueMap[V] = Reg;
471  LastLocalValue = MRI.getVRegDef(Reg);
472  }
473  return Reg;
474 }
475 
476 unsigned FastISel::lookUpRegForValue(const Value *V) {
477  // Look up the value to see if we already have a register for it. We
478  // cache values defined by Instructions across blocks, and other values
479  // only locally. This is because Instructions already have the SSA
480  // def-dominates-use requirement enforced.
481  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(V);
482  if (I != FuncInfo.ValueMap.end())
483  return I->second;
484  return LocalValueMap[V];
485 }
486 
487 void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
488  if (!isa<Instruction>(I)) {
489  LocalValueMap[I] = Reg;
490  return;
491  }
492 
493  unsigned &AssignedReg = FuncInfo.ValueMap[I];
494  if (AssignedReg == 0)
495  // Use the new register.
496  AssignedReg = Reg;
497  else if (Reg != AssignedReg) {
498  // Arrange for uses of AssignedReg to be replaced by uses of Reg.
499  for (unsigned i = 0; i < NumRegs; i++) {
500  FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
501  FuncInfo.RegsWithFixups.insert(Reg + i);
502  }
503 
504  AssignedReg = Reg;
505  }
506 }
507 
508 std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
509  unsigned IdxN = getRegForValue(Idx);
510  if (IdxN == 0)
511  // Unhandled operand. Halt "fast" selection and bail.
512  return std::pair<unsigned, bool>(0, false);
513 
514  bool IdxNIsKill = hasTrivialKill(Idx);
515 
516  // If the index is smaller or larger than intptr_t, truncate or extend it.
517  MVT PtrVT = TLI.getPointerTy(DL);
518  EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
519  if (IdxVT.bitsLT(PtrVT)) {
520  IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
521  IdxNIsKill);
522  IdxNIsKill = true;
523  } else if (IdxVT.bitsGT(PtrVT)) {
524  IdxN =
525  fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
526  IdxNIsKill = true;
527  }
528  return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
529 }
530 
532  if (getLastLocalValue()) {
533  FuncInfo.InsertPt = getLastLocalValue();
534  FuncInfo.MBB = FuncInfo.InsertPt->getParent();
535  ++FuncInfo.InsertPt;
536  } else
537  FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
538 
539  // Now skip past any EH_LABELs, which must remain at the beginning.
540  while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
541  FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL)
542  ++FuncInfo.InsertPt;
543 }
544 
547  assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
548  "Invalid iterator!");
549  while (I != E) {
550  MachineInstr *Dead = &*I;
551  ++I;
552  Dead->eraseFromParent();
553  ++NumFastIselDead;
554  }
555  recomputeInsertPt();
556 }
557 
559  MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt;
560  DebugLoc OldDL = DbgLoc;
561  recomputeInsertPt();
562  DbgLoc = DebugLoc();
563  SavePoint SP = {OldInsertPt, OldDL};
564  return SP;
565 }
566 
568  if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
569  LastLocalValue = &*std::prev(FuncInfo.InsertPt);
570 
571  // Restore the previous insert position.
572  FuncInfo.InsertPt = OldInsertPt.InsertPt;
573  DbgLoc = OldInsertPt.DL;
574 }
575 
576 bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
577  EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
578  if (VT == MVT::Other || !VT.isSimple())
579  // Unhandled type. Halt "fast" selection and bail.
580  return false;
581 
582  // We only handle legal types. For example, on x86-32 the instruction
583  // selector contains all of the 64-bit instructions from x86-64,
584  // under the assumption that i64 won't be used if the target doesn't
585  // support it.
586  if (!TLI.isTypeLegal(VT)) {
587  // MVT::i1 is special. Allow AND, OR, or XOR because they
588  // don't require additional zeroing, which makes them easy.
589  if (VT == MVT::i1 && (ISDOpcode == ISD::AND || ISDOpcode == ISD::OR ||
590  ISDOpcode == ISD::XOR))
591  VT = TLI.getTypeToTransformTo(I->getContext(), VT);
592  else
593  return false;
594  }
595 
596  // Check if the first operand is a constant, and handle it as "ri". At -O0,
597  // we don't have anything that canonicalizes operand order.
598  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
599  if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
600  unsigned Op1 = getRegForValue(I->getOperand(1));
601  if (!Op1)
602  return false;
603  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
604 
605  unsigned ResultReg =
606  fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
607  CI->getZExtValue(), VT.getSimpleVT());
608  if (!ResultReg)
609  return false;
610 
611  // We successfully emitted code for the given LLVM Instruction.
612  updateValueMap(I, ResultReg);
613  return true;
614  }
615 
616  unsigned Op0 = getRegForValue(I->getOperand(0));
617  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
618  return false;
619  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
620 
621  // Check if the second operand is a constant and handle it appropriately.
622  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
623  uint64_t Imm = CI->getSExtValue();
624 
625  // Transform "sdiv exact X, 8" -> "sra X, 3".
626  if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
627  cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
628  Imm = Log2_64(Imm);
629  ISDOpcode = ISD::SRA;
630  }
631 
632  // Transform "urem x, pow2" -> "and x, pow2-1".
633  if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
634  isPowerOf2_64(Imm)) {
635  --Imm;
636  ISDOpcode = ISD::AND;
637  }
638 
639  unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
640  Op0IsKill, Imm, VT.getSimpleVT());
641  if (!ResultReg)
642  return false;
643 
644  // We successfully emitted code for the given LLVM Instruction.
645  updateValueMap(I, ResultReg);
646  return true;
647  }
648 
649  unsigned Op1 = getRegForValue(I->getOperand(1));
650  if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
651  return false;
652  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
653 
654  // Now we have both operands in registers. Emit the instruction.
655  unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
656  ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
657  if (!ResultReg)
658  // Target-specific code wasn't able to find a machine opcode for
659  // the given ISD opcode and type. Halt "fast" selection and bail.
660  return false;
661 
662  // We successfully emitted code for the given LLVM Instruction.
663  updateValueMap(I, ResultReg);
664  return true;
665 }
666 
668  unsigned N = getRegForValue(I->getOperand(0));
669  if (!N) // Unhandled operand. Halt "fast" selection and bail.
670  return false;
671  bool NIsKill = hasTrivialKill(I->getOperand(0));
672 
673  // Keep a running tab of the total offset to coalesce multiple N = N + Offset
674  // into a single N = N + TotalOffset.
675  uint64_t TotalOffs = 0;
676  // FIXME: What's a good SWAG number for MaxOffs?
677  uint64_t MaxOffs = 2048;
678  MVT VT = TLI.getPointerTy(DL);
679  for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
680  GTI != E; ++GTI) {
681  const Value *Idx = GTI.getOperand();
682  if (StructType *StTy = GTI.getStructTypeOrNull()) {
683  uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
684  if (Field) {
685  // N = N + Offset
686  TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
687  if (TotalOffs >= MaxOffs) {
688  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
689  if (!N) // Unhandled operand. Halt "fast" selection and bail.
690  return false;
691  NIsKill = true;
692  TotalOffs = 0;
693  }
694  }
695  } else {
696  Type *Ty = GTI.getIndexedType();
697 
698  // If this is a constant subscript, handle it quickly.
699  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
700  if (CI->isZero())
701  continue;
702  // N = N + Offset
703  uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
704  TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
705  if (TotalOffs >= MaxOffs) {
706  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
707  if (!N) // Unhandled operand. Halt "fast" selection and bail.
708  return false;
709  NIsKill = true;
710  TotalOffs = 0;
711  }
712  continue;
713  }
714  if (TotalOffs) {
715  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
716  if (!N) // Unhandled operand. Halt "fast" selection and bail.
717  return false;
718  NIsKill = true;
719  TotalOffs = 0;
720  }
721 
722  // N = N + Idx * ElementSize;
723  uint64_t ElementSize = DL.getTypeAllocSize(Ty);
724  std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
725  unsigned IdxN = Pair.first;
726  bool IdxNIsKill = Pair.second;
727  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
728  return false;
729 
730  if (ElementSize != 1) {
731  IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
732  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
733  return false;
734  IdxNIsKill = true;
735  }
736  N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
737  if (!N) // Unhandled operand. Halt "fast" selection and bail.
738  return false;
739  }
740  }
741  if (TotalOffs) {
742  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
743  if (!N) // Unhandled operand. Halt "fast" selection and bail.
744  return false;
745  }
746 
747  // We successfully emitted code for the given LLVM Instruction.
748  updateValueMap(I, N);
749  return true;
750 }
751 
752 bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
753  const CallInst *CI, unsigned StartIdx) {
754  for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) {
755  Value *Val = CI->getArgOperand(i);
756  // Check for constants and encode them with a StackMaps::ConstantOp prefix.
757  if (const auto *C = dyn_cast<ConstantInt>(Val)) {
758  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
759  Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
760  } else if (isa<ConstantPointerNull>(Val)) {
761  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
763  } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
764  // Values coming from a stack location also require a special encoding,
765  // but that is added later on by the target specific frame index
766  // elimination implementation.
767  auto SI = FuncInfo.StaticAllocaMap.find(AI);
768  if (SI != FuncInfo.StaticAllocaMap.end())
769  Ops.push_back(MachineOperand::CreateFI(SI->second));
770  else
771  return false;
772  } else {
773  unsigned Reg = getRegForValue(Val);
774  if (!Reg)
775  return false;
776  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
777  }
778  }
779  return true;
780 }
781 
783  // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
784  // [live variables...])
786  "Stackmap cannot return a value.");
787 
788  // The stackmap intrinsic only records the live variables (the arguments
789  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
790  // intrinsic, this won't be lowered to a function call. This means we don't
791  // have to worry about calling conventions and target-specific lowering code.
792  // Instead we perform the call lowering right here.
793  //
794  // CALLSEQ_START(0, 0...)
795  // STACKMAP(id, nbytes, ...)
796  // CALLSEQ_END(0, 0)
797  //
799 
800  // Add the <id> and <numBytes> constants.
801  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
802  "Expected a constant integer.");
803  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
804  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
805 
806  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
807  "Expected a constant integer.");
808  const auto *NumBytes =
809  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
810  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
811 
812  // Push live variables for the stack map (skipping the first two arguments
813  // <id> and <numBytes>).
814  if (!addStackMapLiveVars(Ops, I, 2))
815  return false;
816 
817  // We are not adding any register mask info here, because the stackmap doesn't
818  // clobber anything.
819 
820  // Add scratch registers as implicit def and early clobber.
822  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
823  for (unsigned i = 0; ScratchRegs[i]; ++i)
824  Ops.push_back(MachineOperand::CreateReg(
825  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
826  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
827 
828  // Issue CALLSEQ_START
829  unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
830  auto Builder =
831  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
832  const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
833  for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
834  Builder.addImm(0);
835 
836  // Issue STACKMAP.
837  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
838  TII.get(TargetOpcode::STACKMAP));
839  for (auto const &MO : Ops)
840  MIB.add(MO);
841 
842  // Issue CALLSEQ_END
843  unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
844  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
845  .addImm(0)
846  .addImm(0);
847 
848  // Inform the Frame Information that we have a stackmap in this function.
849  FuncInfo.MF->getFrameInfo().setHasStackMap();
850 
851  return true;
852 }
853 
854 /// Lower an argument list according to the target calling convention.
855 ///
856 /// This is a helper for lowering intrinsics that follow a target calling
857 /// convention or require stack pointer adjustment. Only a subset of the
858 /// intrinsic's operands need to participate in the calling convention.
859 bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
860  unsigned NumArgs, const Value *Callee,
861  bool ForceRetVoidTy, CallLoweringInfo &CLI) {
862  ArgListTy Args;
863  Args.reserve(NumArgs);
864 
865  // Populate the argument list.
866  ImmutableCallSite CS(CI);
867  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
868  Value *V = CI->getOperand(ArgI);
869 
870  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
871 
872  ArgListEntry Entry;
873  Entry.Val = V;
874  Entry.Ty = V->getType();
875  Entry.setAttributes(&CS, ArgI);
876  Args.push_back(Entry);
877  }
878 
879  Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
880  : CI->getType();
881  CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
882 
883  return lowerCallTo(CLI);
884 }
885 
887  const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
888  StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
889  SmallString<32> MangledName;
890  Mangler::getNameWithPrefix(MangledName, Target, DL);
891  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
892  return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
893 }
894 
896  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
897  // i32 <numBytes>,
898  // i8* <target>,
899  // i32 <numArgs>,
900  // [Args...],
901  // [live variables...])
903  bool IsAnyRegCC = CC == CallingConv::AnyReg;
904  bool HasDef = !I->getType()->isVoidTy();
906 
907  // Get the real number of arguments participating in the call <numArgs>
908  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
909  "Expected a constant integer.");
910  const auto *NumArgsVal =
911  cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
912  unsigned NumArgs = NumArgsVal->getZExtValue();
913 
914  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
915  // This includes all meta-operands up to but not including CC.
916  unsigned NumMetaOpers = PatchPointOpers::CCPos;
917  assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
918  "Not enough arguments provided to the patchpoint intrinsic");
919 
920  // For AnyRegCC the arguments are lowered later on manually.
921  unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
922  CallLoweringInfo CLI;
923  CLI.setIsPatchPoint();
924  if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
925  return false;
926 
927  assert(CLI.Call && "No call instruction specified.");
928 
930 
931  // Add an explicit result reg if we use the anyreg calling convention.
932  if (IsAnyRegCC && HasDef) {
933  assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
934  CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64));
935  CLI.NumResultRegs = 1;
936  Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*IsDef=*/true));
937  }
938 
939  // Add the <id> and <numBytes> constants.
940  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
941  "Expected a constant integer.");
942  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
943  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
944 
945  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
946  "Expected a constant integer.");
947  const auto *NumBytes =
948  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
949  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
950 
951  // Add the call target.
952  if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
953  uint64_t CalleeConstAddr =
954  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
955  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
956  } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
957  if (C->getOpcode() == Instruction::IntToPtr) {
958  uint64_t CalleeConstAddr =
959  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
960  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
961  } else
962  llvm_unreachable("Unsupported ConstantExpr.");
963  } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
965  } else if (isa<ConstantPointerNull>(Callee))
967  else
968  llvm_unreachable("Unsupported callee address.");
969 
970  // Adjust <numArgs> to account for any arguments that have been passed on
971  // the stack instead.
972  unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
973  Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
974 
975  // Add the calling convention
976  Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
977 
978  // Add the arguments we omitted previously. The register allocator should
979  // place these in any free register.
980  if (IsAnyRegCC) {
981  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
982  unsigned Reg = getRegForValue(I->getArgOperand(i));
983  if (!Reg)
984  return false;
985  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
986  }
987  }
988 
989  // Push the arguments from the call instruction.
990  for (auto Reg : CLI.OutRegs)
991  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
992 
993  // Push live variables for the stack map.
994  if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
995  return false;
996 
997  // Push the register mask info.
999  TRI.getCallPreservedMask(*FuncInfo.MF, CC)));
1000 
1001  // Add scratch registers as implicit def and early clobber.
1002  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
1003  for (unsigned i = 0; ScratchRegs[i]; ++i)
1005  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
1006  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
1007 
1008  // Add implicit defs (return values).
1009  for (auto Reg : CLI.InRegs)
1010  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/true,
1011  /*IsImpl=*/true));
1012 
1013  // Insert the patchpoint instruction before the call generated by the target.
1014  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc,
1015  TII.get(TargetOpcode::PATCHPOINT));
1016 
1017  for (auto &MO : Ops)
1018  MIB.add(MO);
1019 
1020  MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1021 
1022  // Delete the original call instruction.
1023  CLI.Call->eraseFromParent();
1024 
1025  // Inform the Frame Information that we have a patchpoint in this function.
1026  FuncInfo.MF->getFrameInfo().setHasPatchPoint();
1027 
1028  if (CLI.NumResultRegs)
1029  updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
1030  return true;
1031 }
1032 
1034  const auto &Triple = TM.getTargetTriple();
1035  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1036  return true; // don't do anything to this instruction.
1038  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1039  /*IsDef=*/false));
1040  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1041  /*IsDef=*/false));
1042  MachineInstrBuilder MIB =
1043  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1044  TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
1045  for (auto &MO : Ops)
1046  MIB.add(MO);
1047 
1048  // Insert the Patchable Event Call instruction, that gets lowered properly.
1049  return true;
1050 }
1051 
1053  const auto &Triple = TM.getTargetTriple();
1054  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1055  return true; // don't do anything to this instruction.
1057  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1058  /*IsDef=*/false));
1059  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1060  /*IsDef=*/false));
1061  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(2)),
1062  /*IsDef=*/false));
1063  MachineInstrBuilder MIB =
1064  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1065  TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
1066  for (auto &MO : Ops)
1067  MIB.add(MO);
1068 
1069  // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
1070  return true;
1071 }
1072 
1073 /// Returns an AttributeList representing the attributes applied to the return
1074 /// value of the given call.
1077  if (CLI.RetSExt)
1078  Attrs.push_back(Attribute::SExt);
1079  if (CLI.RetZExt)
1080  Attrs.push_back(Attribute::ZExt);
1081  if (CLI.IsInReg)
1082  Attrs.push_back(Attribute::InReg);
1083 
1085  Attrs);
1086 }
1087 
1088 bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
1089  unsigned NumArgs) {
1090  MCContext &Ctx = MF->getContext();
1091  SmallString<32> MangledName;
1092  Mangler::getNameWithPrefix(MangledName, SymName, DL);
1093  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
1094  return lowerCallTo(CI, Sym, NumArgs);
1095 }
1096 
1098  unsigned NumArgs) {
1099  ImmutableCallSite CS(CI);
1100 
1101  FunctionType *FTy = CS.getFunctionType();
1102  Type *RetTy = CS.getType();
1103 
1104  ArgListTy Args;
1105  Args.reserve(NumArgs);
1106 
1107  // Populate the argument list.
1108  // Attributes for args start at offset 1, after the return attribute.
1109  for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
1110  Value *V = CI->getOperand(ArgI);
1111 
1112  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
1113 
1114  ArgListEntry Entry;
1115  Entry.Val = V;
1116  Entry.Ty = V->getType();
1117  Entry.setAttributes(&CS, ArgI);
1118  Args.push_back(Entry);
1119  }
1120  TLI.markLibCallAttributes(MF, CS.getCallingConv(), Args);
1121 
1122  CallLoweringInfo CLI;
1123  CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
1124 
1125  return lowerCallTo(CLI);
1126 }
1127 
1129  // Handle the incoming return values from the call.
1130  CLI.clearIns();
1131  SmallVector<EVT, 4> RetTys;
1132  ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
1133 
1135  GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
1136 
1137  bool CanLowerReturn = TLI.CanLowerReturn(
1138  CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1139 
1140  // FIXME: sret demotion isn't supported yet - bail out.
1141  if (!CanLowerReturn)
1142  return false;
1143 
1144  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
1145  EVT VT = RetTys[I];
1146  MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1147  unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1148  for (unsigned i = 0; i != NumRegs; ++i) {
1149  ISD::InputArg MyFlags;
1150  MyFlags.VT = RegisterVT;
1151  MyFlags.ArgVT = VT;
1152  MyFlags.Used = CLI.IsReturnValueUsed;
1153  if (CLI.RetSExt)
1154  MyFlags.Flags.setSExt();
1155  if (CLI.RetZExt)
1156  MyFlags.Flags.setZExt();
1157  if (CLI.IsInReg)
1158  MyFlags.Flags.setInReg();
1159  CLI.Ins.push_back(MyFlags);
1160  }
1161  }
1162 
1163  // Handle all of the outgoing arguments.
1164  CLI.clearOuts();
1165  for (auto &Arg : CLI.getArgs()) {
1166  Type *FinalType = Arg.Ty;
1167  if (Arg.IsByVal)
1168  FinalType = cast<PointerType>(Arg.Ty)->getElementType();
1169  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1170  FinalType, CLI.CallConv, CLI.IsVarArg);
1171 
1172  ISD::ArgFlagsTy Flags;
1173  if (Arg.IsZExt)
1174  Flags.setZExt();
1175  if (Arg.IsSExt)
1176  Flags.setSExt();
1177  if (Arg.IsInReg)
1178  Flags.setInReg();
1179  if (Arg.IsSRet)
1180  Flags.setSRet();
1181  if (Arg.IsSwiftSelf)
1182  Flags.setSwiftSelf();
1183  if (Arg.IsSwiftError)
1184  Flags.setSwiftError();
1185  if (Arg.IsByVal)
1186  Flags.setByVal();
1187  if (Arg.IsInAlloca) {
1188  Flags.setInAlloca();
1189  // Set the byval flag for CCAssignFn callbacks that don't know about
1190  // inalloca. This way we can know how many bytes we should've allocated
1191  // and how many bytes a callee cleanup function will pop. If we port
1192  // inalloca to more targets, we'll have to add custom inalloca handling in
1193  // the various CC lowering callbacks.
1194  Flags.setByVal();
1195  }
1196  if (Arg.IsByVal || Arg.IsInAlloca) {
1197  PointerType *Ty = cast<PointerType>(Arg.Ty);
1198  Type *ElementTy = Ty->getElementType();
1199  unsigned FrameSize = DL.getTypeAllocSize(ElementTy);
1200  // For ByVal, alignment should come from FE. BE will guess if this info is
1201  // not there, but there are cases it cannot get right.
1202  unsigned FrameAlign = Arg.Alignment;
1203  if (!FrameAlign)
1204  FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
1205  Flags.setByValSize(FrameSize);
1206  Flags.setByValAlign(FrameAlign);
1207  }
1208  if (Arg.IsNest)
1209  Flags.setNest();
1210  if (NeedsRegBlock)
1211  Flags.setInConsecutiveRegs();
1212  unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
1213  Flags.setOrigAlign(OriginalAlignment);
1214 
1215  CLI.OutVals.push_back(Arg.Val);
1216  CLI.OutFlags.push_back(Flags);
1217  }
1218 
1219  if (!fastLowerCall(CLI))
1220  return false;
1221 
1222  // Set all unused physreg defs as dead.
1223  assert(CLI.Call && "No call instruction specified.");
1224  CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1225 
1226  if (CLI.NumResultRegs && CLI.CS)
1227  updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs);
1228 
1229  return true;
1230 }
1231 
1233  ImmutableCallSite CS(CI);
1234 
1235  FunctionType *FuncTy = CS.getFunctionType();
1236  Type *RetTy = CS.getType();
1237 
1238  ArgListTy Args;
1239  ArgListEntry Entry;
1240  Args.reserve(CS.arg_size());
1241 
1242  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
1243  i != e; ++i) {
1244  Value *V = *i;
1245 
1246  // Skip empty types
1247  if (V->getType()->isEmptyTy())
1248  continue;
1249 
1250  Entry.Val = V;
1251  Entry.Ty = V->getType();
1252 
1253  // Skip the first return-type Attribute to get to params.
1254  Entry.setAttributes(&CS, i - CS.arg_begin());
1255  Args.push_back(Entry);
1256  }
1257 
1258  // Check if target-independent constraints permit a tail call here.
1259  // Target-dependent constraints are checked within fastLowerCall.
1260  bool IsTailCall = CI->isTailCall();
1261  if (IsTailCall && !isInTailCallPosition(CS, TM))
1262  IsTailCall = false;
1263 
1264  CallLoweringInfo CLI;
1265  CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
1266  .setTailCall(IsTailCall);
1267 
1268  return lowerCallTo(CLI);
1269 }
1270 
1272  const CallInst *Call = cast<CallInst>(I);
1273 
1274  // Handle simple inline asms.
1275  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
1276  // If the inline asm has side effects, then make sure that no local value
1277  // lives across by flushing the local value map.
1278  if (IA->hasSideEffects())
1279  flushLocalValueMap();
1280 
1281  // Don't attempt to handle constraints.
1282  if (!IA->getConstraintString().empty())
1283  return false;
1284 
1285  unsigned ExtraInfo = 0;
1286  if (IA->hasSideEffects())
1287  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1288  if (IA->isAlignStack())
1289  ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1290 
1291  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1293  .addExternalSymbol(IA->getAsmString().c_str())
1294  .addImm(ExtraInfo);
1295  return true;
1296  }
1297 
1298  MachineModuleInfo &MMI = FuncInfo.MF->getMMI();
1299  computeUsesVAFloatArgument(*Call, MMI);
1300 
1301  // Handle intrinsic function calls.
1302  if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1303  return selectIntrinsicCall(II);
1304 
1305  // Usually, it does not make sense to initialize a value,
1306  // make an unrelated function call and use the value, because
1307  // it tends to be spilled on the stack. So, we move the pointer
1308  // to the last local value to the beginning of the block, so that
1309  // all the values which have already been materialized,
1310  // appear after the call. It also makes sense to skip intrinsics
1311  // since they tend to be inlined.
1312  flushLocalValueMap();
1313 
1314  return lowerCall(Call);
1315 }
1316 
1318  switch (II->getIntrinsicID()) {
1319  default:
1320  break;
1321  // At -O0 we don't care about the lifetime intrinsics.
1322  case Intrinsic::lifetime_start:
1323  case Intrinsic::lifetime_end:
1324  // The donothing intrinsic does, well, nothing.
1325  case Intrinsic::donothing:
1326  // Neither does the sideeffect intrinsic.
1327  case Intrinsic::sideeffect:
1328  // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1329  case Intrinsic::assume:
1330  return true;
1331  case Intrinsic::dbg_declare: {
1332  const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1333  assert(DI->getVariable() && "Missing variable");
1334  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1335  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1336  return true;
1337  }
1338 
1339  const Value *Address = DI->getAddress();
1340  if (!Address || isa<UndefValue>(Address)) {
1341  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1342  return true;
1343  }
1344 
1345  // Byval arguments with frame indices were already handled after argument
1346  // lowering and before isel.
1347  const auto *Arg =
1349  if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
1350  return true;
1351 
1353  if (unsigned Reg = lookUpRegForValue(Address))
1354  Op = MachineOperand::CreateReg(Reg, false);
1355 
1356  // If we have a VLA that has a "use" in a metadata node that's then used
1357  // here but it has no other uses, then we have a problem. E.g.,
1358  //
1359  // int foo (const int *x) {
1360  // char a[*x];
1361  // return 0;
1362  // }
1363  //
1364  // If we assign 'a' a vreg and fast isel later on has to use the selection
1365  // DAG isel, it will want to copy the value to the vreg. However, there are
1366  // no uses, which goes counter to what selection DAG isel expects.
1367  if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
1368  (!isa<AllocaInst>(Address) ||
1369  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
1370  Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address),
1371  false);
1372 
1373  if (Op) {
1375  "Expected inlined-at fields to agree");
1376  // A dbg.declare describes the address of a source variable, so lower it
1377  // into an indirect DBG_VALUE.
1378  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1379  TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true,
1380  *Op, DI->getVariable(), DI->getExpression());
1381  } else {
1382  // We can't yet handle anything else here because it would require
1383  // generating code, thus altering codegen because of debug info.
1384  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1385  }
1386  return true;
1387  }
1388  case Intrinsic::dbg_value: {
1389  // This form of DBG_VALUE is target-independent.
1390  const DbgValueInst *DI = cast<DbgValueInst>(II);
1391  const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1392  const Value *V = DI->getValue();
1394  "Expected inlined-at fields to agree");
1395  if (!V) {
1396  // Currently the optimizer can produce this; insert an undef to
1397  // help debugging. Probably the optimizer should not do this.
1398  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
1399  DI->getVariable(), DI->getExpression());
1400  } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1401  if (CI->getBitWidth() > 64)
1402  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1403  .addCImm(CI)
1404  .addImm(0U)
1405  .addMetadata(DI->getVariable())
1406  .addMetadata(DI->getExpression());
1407  else
1408  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1409  .addImm(CI->getZExtValue())
1410  .addImm(0U)
1411  .addMetadata(DI->getVariable())
1412  .addMetadata(DI->getExpression());
1413  } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1414  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1415  .addFPImm(CF)
1416  .addImm(0U)
1417  .addMetadata(DI->getVariable())
1418  .addMetadata(DI->getExpression());
1419  } else if (unsigned Reg = lookUpRegForValue(V)) {
1420  // FIXME: This does not handle register-indirect values at offset 0.
1421  bool IsIndirect = false;
1422  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
1423  DI->getVariable(), DI->getExpression());
1424  } else {
1425  // We can't yet handle anything else here because it would require
1426  // generating code, thus altering codegen because of debug info.
1427  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1428  }
1429  return true;
1430  }
1431  case Intrinsic::dbg_label: {
1432  const DbgLabelInst *DI = cast<DbgLabelInst>(II);
1433  assert(DI->getLabel() && "Missing label");
1434  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1435  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1436  return true;
1437  }
1438 
1439  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1440  TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
1441  return true;
1442  }
1443  case Intrinsic::objectsize: {
1444  ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
1445  unsigned long long Res = CI->isZero() ? -1ULL : 0;
1446  Constant *ResCI = ConstantInt::get(II->getType(), Res);
1447  unsigned ResultReg = getRegForValue(ResCI);
1448  if (!ResultReg)
1449  return false;
1450  updateValueMap(II, ResultReg);
1451  return true;
1452  }
1453  case Intrinsic::is_constant: {
1454  Constant *ResCI = ConstantInt::get(II->getType(), 0);
1455  unsigned ResultReg = getRegForValue(ResCI);
1456  if (!ResultReg)
1457  return false;
1458  updateValueMap(II, ResultReg);
1459  return true;
1460  }
1461  case Intrinsic::launder_invariant_group:
1462  case Intrinsic::strip_invariant_group:
1463  case Intrinsic::expect: {
1464  unsigned ResultReg = getRegForValue(II->getArgOperand(0));
1465  if (!ResultReg)
1466  return false;
1467  updateValueMap(II, ResultReg);
1468  return true;
1469  }
1470  case Intrinsic::experimental_stackmap:
1471  return selectStackmap(II);
1472  case Intrinsic::experimental_patchpoint_void:
1473  case Intrinsic::experimental_patchpoint_i64:
1474  return selectPatchpoint(II);
1475 
1476  case Intrinsic::xray_customevent:
1477  return selectXRayCustomEvent(II);
1478  case Intrinsic::xray_typedevent:
1479  return selectXRayTypedEvent(II);
1480  }
1481 
1482  return fastLowerIntrinsicCall(II);
1483 }
1484 
1485 bool FastISel::selectCast(const User *I, unsigned Opcode) {
1486  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1487  EVT DstVT = TLI.getValueType(DL, I->getType());
1488 
1489  if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1490  !DstVT.isSimple())
1491  // Unhandled type. Halt "fast" selection and bail.
1492  return false;
1493 
1494  // Check if the destination type is legal.
1495  if (!TLI.isTypeLegal(DstVT))
1496  return false;
1497 
1498  // Check if the source operand is legal.
1499  if (!TLI.isTypeLegal(SrcVT))
1500  return false;
1501 
1502  unsigned InputReg = getRegForValue(I->getOperand(0));
1503  if (!InputReg)
1504  // Unhandled operand. Halt "fast" selection and bail.
1505  return false;
1506 
1507  bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
1508 
1509  unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1510  Opcode, InputReg, InputRegIsKill);
1511  if (!ResultReg)
1512  return false;
1513 
1514  updateValueMap(I, ResultReg);
1515  return true;
1516 }
1517 
1519  // If the bitcast doesn't change the type, just use the operand value.
1520  if (I->getType() == I->getOperand(0)->getType()) {
1521  unsigned Reg = getRegForValue(I->getOperand(0));
1522  if (!Reg)
1523  return false;
1524  updateValueMap(I, Reg);
1525  return true;
1526  }
1527 
1528  // Bitcasts of other values become reg-reg copies or BITCAST operators.
1529  EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1530  EVT DstEVT = TLI.getValueType(DL, I->getType());
1531  if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1532  !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
1533  // Unhandled type. Halt "fast" selection and bail.
1534  return false;
1535 
1536  MVT SrcVT = SrcEVT.getSimpleVT();
1537  MVT DstVT = DstEVT.getSimpleVT();
1538  unsigned Op0 = getRegForValue(I->getOperand(0));
1539  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1540  return false;
1541  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
1542 
1543  // First, try to perform the bitcast by inserting a reg-reg copy.
1544  unsigned ResultReg = 0;
1545  if (SrcVT == DstVT) {
1546  const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
1547  const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
1548  // Don't attempt a cross-class copy. It will likely fail.
1549  if (SrcClass == DstClass) {
1550  ResultReg = createResultReg(DstClass);
1551  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1552  TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
1553  }
1554  }
1555 
1556  // If the reg-reg copy failed, select a BITCAST opcode.
1557  if (!ResultReg)
1558  ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
1559 
1560  if (!ResultReg)
1561  return false;
1562 
1563  updateValueMap(I, ResultReg);
1564  return true;
1565 }
1566 
1567 // Remove local value instructions starting from the instruction after
1568 // SavedLastLocalValue to the current function insert point.
1569 void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1570 {
1571  MachineInstr *CurLastLocalValue = getLastLocalValue();
1572  if (CurLastLocalValue != SavedLastLocalValue) {
1573  // Find the first local value instruction to be deleted.
1574  // This is the instruction after SavedLastLocalValue if it is non-NULL.
1575  // Otherwise it's the first instruction in the block.
1576  MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1577  if (SavedLastLocalValue)
1578  ++FirstDeadInst;
1579  else
1580  FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1581  setLastLocalValue(SavedLastLocalValue);
1582  removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1583  }
1584 }
1585 
1587  MachineInstr *SavedLastLocalValue = getLastLocalValue();
1588  // Just before the terminator instruction, insert instructions to
1589  // feed PHI nodes in successor blocks.
1590  if (I->isTerminator()) {
1591  if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1592  // PHI node handling may have generated local value instructions,
1593  // even though it failed to handle all PHI nodes.
1594  // We remove these instructions because SelectionDAGISel will generate
1595  // them again.
1596  removeDeadLocalValueCode(SavedLastLocalValue);
1597  return false;
1598  }
1599  }
1600 
1601  // FastISel does not handle any operand bundles except OB_funclet.
1603  for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
1604  if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1605  return false;
1606 
1607  DbgLoc = I->getDebugLoc();
1608 
1609  SavedInsertPt = FuncInfo.InsertPt;
1610 
1611  if (const auto *Call = dyn_cast<CallInst>(I)) {
1612  const Function *F = Call->getCalledFunction();
1613  LibFunc Func;
1614 
1615  // As a special case, don't handle calls to builtin library functions that
1616  // may be translated directly to target instructions.
1617  if (F && !F->hasLocalLinkage() && F->hasName() &&
1618  LibInfo->getLibFunc(F->getName(), Func) &&
1619  LibInfo->hasOptimizedCodeGen(Func))
1620  return false;
1621 
1622  // Don't handle Intrinsic::trap if a trap function is specified.
1623  if (F && F->getIntrinsicID() == Intrinsic::trap &&
1624  Call->hasFnAttr("trap-func-name"))
1625  return false;
1626  }
1627 
1628  // First, try doing target-independent selection.
1629  if (!SkipTargetIndependentISel) {
1630  if (selectOperator(I, I->getOpcode())) {
1631  ++NumFastIselSuccessIndependent;
1632  DbgLoc = DebugLoc();
1633  return true;
1634  }
1635  // Remove dead code.
1636  recomputeInsertPt();
1637  if (SavedInsertPt != FuncInfo.InsertPt)
1638  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1639  SavedInsertPt = FuncInfo.InsertPt;
1640  }
1641  // Next, try calling the target to attempt to handle the instruction.
1642  if (fastSelectInstruction(I)) {
1643  ++NumFastIselSuccessTarget;
1644  DbgLoc = DebugLoc();
1645  return true;
1646  }
1647  // Remove dead code.
1648  recomputeInsertPt();
1649  if (SavedInsertPt != FuncInfo.InsertPt)
1650  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1651 
1652  DbgLoc = DebugLoc();
1653  // Undo phi node updates, because they will be added again by SelectionDAG.
1654  if (I->isTerminator()) {
1655  // PHI node handling may have generated local value instructions.
1656  // We remove them because SelectionDAGISel will generate them again.
1657  removeDeadLocalValueCode(SavedLastLocalValue);
1658  FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
1659  }
1660  return false;
1661 }
1662 
1663 /// Emit an unconditional branch to the given block, unless it is the immediate
1664 /// (fall-through) successor, and update the CFG.
1666  const DebugLoc &DbgLoc) {
1667  if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
1668  FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
1669  // For more accurate line information if this is the only instruction
1670  // in the block then emit it, otherwise we have the unconditional
1671  // fall-through case, which needs no instructions.
1672  } else {
1673  // The unconditional branch case.
1674  TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1675  SmallVector<MachineOperand, 0>(), DbgLoc);
1676  }
1677  if (FuncInfo.BPI) {
1678  auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1679  FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
1680  FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
1681  } else
1682  FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
1683 }
1684 
1686  MachineBasicBlock *TrueMBB,
1687  MachineBasicBlock *FalseMBB) {
1688  // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1689  // happen in degenerate IR and MachineIR forbids to have a block twice in the
1690  // successor/predecessor lists.
1691  if (TrueMBB != FalseMBB) {
1692  if (FuncInfo.BPI) {
1693  auto BranchProbability =
1694  FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1695  FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
1696  } else
1697  FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
1698  }
1699 
1700  fastEmitBranch(FalseMBB, DbgLoc);
1701 }
1702 
1703 /// Emit an FNeg operation.
1705  Value *X;
1706  if (!match(I, m_FNeg(m_Value(X))))
1707  return false;
1708  unsigned OpReg = getRegForValue(X);
1709  if (!OpReg)
1710  return false;
1711  bool OpRegIsKill = hasTrivialKill(I);
1712 
1713  // If the target has ISD::FNEG, use it.
1714  EVT VT = TLI.getValueType(DL, I->getType());
1715  unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1716  OpReg, OpRegIsKill);
1717  if (ResultReg) {
1718  updateValueMap(I, ResultReg);
1719  return true;
1720  }
1721 
1722  // Bitcast the value to integer, twiddle the sign bit with xor,
1723  // and then bitcast it back to floating-point.
1724  if (VT.getSizeInBits() > 64)
1725  return false;
1726  EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1727  if (!TLI.isTypeLegal(IntVT))
1728  return false;
1729 
1730  unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1731  ISD::BITCAST, OpReg, OpRegIsKill);
1732  if (!IntReg)
1733  return false;
1734 
1735  unsigned IntResultReg = fastEmit_ri_(
1736  IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
1737  UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1738  if (!IntResultReg)
1739  return false;
1740 
1741  ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1742  IntResultReg, /*IsKill=*/true);
1743  if (!ResultReg)
1744  return false;
1745 
1746  updateValueMap(I, ResultReg);
1747  return true;
1748 }
1749 
1751  const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1752  if (!EVI)
1753  return false;
1754 
1755  // Make sure we only try to handle extracts with a legal result. But also
1756  // allow i1 because it's easy.
1757  EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1758  if (!RealVT.isSimple())
1759  return false;
1760  MVT VT = RealVT.getSimpleVT();
1761  if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1762  return false;
1763 
1764  const Value *Op0 = EVI->getOperand(0);
1765  Type *AggTy = Op0->getType();
1766 
1767  // Get the base result register.
1768  unsigned ResultReg;
1769  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(Op0);
1770  if (I != FuncInfo.ValueMap.end())
1771  ResultReg = I->second;
1772  else if (isa<Instruction>(Op0))
1773  ResultReg = FuncInfo.InitializeRegForValue(Op0);
1774  else
1775  return false; // fast-isel can't handle aggregate constants at the moment
1776 
1777  // Get the actual result register, which is an offset from the base register.
1778  unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1779 
1780  SmallVector<EVT, 4> AggValueVTs;
1781  ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1782 
1783  for (unsigned i = 0; i < VTIndex; i++)
1784  ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1785 
1786  updateValueMap(EVI, ResultReg);
1787  return true;
1788 }
1789 
1790 bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1791  switch (Opcode) {
1792  case Instruction::Add:
1793  return selectBinaryOp(I, ISD::ADD);
1794  case Instruction::FAdd:
1795  return selectBinaryOp(I, ISD::FADD);
1796  case Instruction::Sub:
1797  return selectBinaryOp(I, ISD::SUB);
1798  case Instruction::FSub:
1799  // FNeg is currently represented in LLVM IR as a special case of FSub.
1800  return selectFNeg(I) || selectBinaryOp(I, ISD::FSUB);
1801  case Instruction::Mul:
1802  return selectBinaryOp(I, ISD::MUL);
1803  case Instruction::FMul:
1804  return selectBinaryOp(I, ISD::FMUL);
1805  case Instruction::SDiv:
1806  return selectBinaryOp(I, ISD::SDIV);
1807  case Instruction::UDiv:
1808  return selectBinaryOp(I, ISD::UDIV);
1809  case Instruction::FDiv:
1810  return selectBinaryOp(I, ISD::FDIV);
1811  case Instruction::SRem:
1812  return selectBinaryOp(I, ISD::SREM);
1813  case Instruction::URem:
1814  return selectBinaryOp(I, ISD::UREM);
1815  case Instruction::FRem:
1816  return selectBinaryOp(I, ISD::FREM);
1817  case Instruction::Shl:
1818  return selectBinaryOp(I, ISD::SHL);
1819  case Instruction::LShr:
1820  return selectBinaryOp(I, ISD::SRL);
1821  case Instruction::AShr:
1822  return selectBinaryOp(I, ISD::SRA);
1823  case Instruction::And:
1824  return selectBinaryOp(I, ISD::AND);
1825  case Instruction::Or:
1826  return selectBinaryOp(I, ISD::OR);
1827  case Instruction::Xor:
1828  return selectBinaryOp(I, ISD::XOR);
1829 
1830  case Instruction::GetElementPtr:
1831  return selectGetElementPtr(I);
1832 
1833  case Instruction::Br: {
1834  const BranchInst *BI = cast<BranchInst>(I);
1835 
1836  if (BI->isUnconditional()) {
1837  const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1838  MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1839  fastEmitBranch(MSucc, BI->getDebugLoc());
1840  return true;
1841  }
1842 
1843  // Conditional branches are not handed yet.
1844  // Halt "fast" selection and bail.
1845  return false;
1846  }
1847 
1848  case Instruction::Unreachable:
1849  if (TM.Options.TrapUnreachable)
1850  return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
1851  else
1852  return true;
1853 
1854  case Instruction::Alloca:
1855  // FunctionLowering has the static-sized case covered.
1856  if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1857  return true;
1858 
1859  // Dynamic-sized alloca is not handled yet.
1860  return false;
1861 
1862  case Instruction::Call:
1863  return selectCall(I);
1864 
1865  case Instruction::BitCast:
1866  return selectBitCast(I);
1867 
1868  case Instruction::FPToSI:
1869  return selectCast(I, ISD::FP_TO_SINT);
1870  case Instruction::ZExt:
1871  return selectCast(I, ISD::ZERO_EXTEND);
1872  case Instruction::SExt:
1873  return selectCast(I, ISD::SIGN_EXTEND);
1874  case Instruction::Trunc:
1875  return selectCast(I, ISD::TRUNCATE);
1876  case Instruction::SIToFP:
1877  return selectCast(I, ISD::SINT_TO_FP);
1878 
1879  case Instruction::IntToPtr: // Deliberate fall-through.
1880  case Instruction::PtrToInt: {
1881  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1882  EVT DstVT = TLI.getValueType(DL, I->getType());
1883  if (DstVT.bitsGT(SrcVT))
1884  return selectCast(I, ISD::ZERO_EXTEND);
1885  if (DstVT.bitsLT(SrcVT))
1886  return selectCast(I, ISD::TRUNCATE);
1887  unsigned Reg = getRegForValue(I->getOperand(0));
1888  if (!Reg)
1889  return false;
1890  updateValueMap(I, Reg);
1891  return true;
1892  }
1893 
1894  case Instruction::ExtractValue:
1895  return selectExtractValue(I);
1896 
1897  case Instruction::PHI:
1898  llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1899 
1900  default:
1901  // Unhandled instruction. Halt "fast" selection and bail.
1902  return false;
1903  }
1904 }
1905 
1907  const TargetLibraryInfo *LibInfo,
1908  bool SkipTargetIndependentISel)
1909  : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1910  MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1911  TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1912  TII(*MF->getSubtarget().getInstrInfo()),
1913  TLI(*MF->getSubtarget().getTargetLowering()),
1914  TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1915  SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1916 
1917 FastISel::~FastISel() = default;
1918 
1919 bool FastISel::fastLowerArguments() { return false; }
1920 
1921 bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1922 
1924  return false;
1925 }
1926 
1927 unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1928 
1929 unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
1930  bool /*Op0IsKill*/) {
1931  return 0;
1932 }
1933 
1934 unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1935  bool /*Op0IsKill*/, unsigned /*Op1*/,
1936  bool /*Op1IsKill*/) {
1937  return 0;
1938 }
1939 
1940 unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1941  return 0;
1942 }
1943 
1944 unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1945  const ConstantFP * /*FPImm*/) {
1946  return 0;
1947 }
1948 
1949 unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1950  bool /*Op0IsKill*/, uint64_t /*Imm*/) {
1951  return 0;
1952 }
1953 
1954 /// This method is a wrapper of fastEmit_ri. It first tries to emit an
1955 /// instruction with an immediate operand using fastEmit_ri.
1956 /// If that fails, it materializes the immediate into a register and try
1957 /// fastEmit_rr instead.
1958 unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1959  bool Op0IsKill, uint64_t Imm, MVT ImmType) {
1960  // If this is a multiply by a power of two, emit this as a shift left.
1961  if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
1962  Opcode = ISD::SHL;
1963  Imm = Log2_64(Imm);
1964  } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
1965  // div x, 8 -> srl x, 3
1966  Opcode = ISD::SRL;
1967  Imm = Log2_64(Imm);
1968  }
1969 
1970  // Horrible hack (to be removed), check to make sure shift amounts are
1971  // in-range.
1972  if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1973  Imm >= VT.getSizeInBits())
1974  return 0;
1975 
1976  // First check if immediate type is legal. If not, we can't use the ri form.
1977  unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
1978  if (ResultReg)
1979  return ResultReg;
1980  unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1981  bool IsImmKill = true;
1982  if (!MaterialReg) {
1983  // This is a bit ugly/slow, but failing here means falling out of
1984  // fast-isel, which would be very slow.
1985  IntegerType *ITy =
1987  MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
1988  if (!MaterialReg)
1989  return 0;
1990  // FIXME: If the materialized register here has no uses yet then this
1991  // will be the first use and we should be able to mark it as killed.
1992  // However, the local value area for materialising constant expressions
1993  // grows down, not up, which means that any constant expressions we generate
1994  // later which also use 'Imm' could be after this instruction and therefore
1995  // after this kill.
1996  IsImmKill = false;
1997  }
1998  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
1999 }
2000 
2002  return MRI.createVirtualRegister(RC);
2003 }
2004 
2006  unsigned OpNum) {
2008  const TargetRegisterClass *RegClass =
2009  TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
2010  if (!MRI.constrainRegClass(Op, RegClass)) {
2011  // If it's not legal to COPY between the register classes, something
2012  // has gone very wrong before we got here.
2013  unsigned NewOp = createResultReg(RegClass);
2015  TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
2016  return NewOp;
2017  }
2018  }
2019  return Op;
2020 }
2021 
2022 unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2023  const TargetRegisterClass *RC) {
2024  unsigned ResultReg = createResultReg(RC);
2025  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2026 
2027  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
2028  return ResultReg;
2029 }
2030 
2031 unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2032  const TargetRegisterClass *RC, unsigned Op0,
2033  bool Op0IsKill) {
2034  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2035 
2036  unsigned ResultReg = createResultReg(RC);
2037  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2038 
2039  if (II.getNumDefs() >= 1)
2040  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2041  .addReg(Op0, getKillRegState(Op0IsKill));
2042  else {
2044  .addReg(Op0, getKillRegState(Op0IsKill));
2046  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2047  }
2048 
2049  return ResultReg;
2050 }
2051 
2052 unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2053  const TargetRegisterClass *RC, unsigned Op0,
2054  bool Op0IsKill, unsigned Op1,
2055  bool Op1IsKill) {
2056  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2057 
2058  unsigned ResultReg = createResultReg(RC);
2059  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2060  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2061 
2062  if (II.getNumDefs() >= 1)
2063  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2064  .addReg(Op0, getKillRegState(Op0IsKill))
2065  .addReg(Op1, getKillRegState(Op1IsKill));
2066  else {
2068  .addReg(Op0, getKillRegState(Op0IsKill))
2069  .addReg(Op1, getKillRegState(Op1IsKill));
2071  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2072  }
2073  return ResultReg;
2074 }
2075 
2076 unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2077  const TargetRegisterClass *RC, unsigned Op0,
2078  bool Op0IsKill, unsigned Op1,
2079  bool Op1IsKill, unsigned Op2,
2080  bool Op2IsKill) {
2081  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2082 
2083  unsigned ResultReg = createResultReg(RC);
2084  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2085  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2086  Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
2087 
2088  if (II.getNumDefs() >= 1)
2089  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2090  .addReg(Op0, getKillRegState(Op0IsKill))
2091  .addReg(Op1, getKillRegState(Op1IsKill))
2092  .addReg(Op2, getKillRegState(Op2IsKill));
2093  else {
2095  .addReg(Op0, getKillRegState(Op0IsKill))
2096  .addReg(Op1, getKillRegState(Op1IsKill))
2097  .addReg(Op2, getKillRegState(Op2IsKill));
2099  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2100  }
2101  return ResultReg;
2102 }
2103 
2104 unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2105  const TargetRegisterClass *RC, unsigned Op0,
2106  bool Op0IsKill, uint64_t Imm) {
2107  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2108 
2109  unsigned ResultReg = createResultReg(RC);
2110  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2111 
2112  if (II.getNumDefs() >= 1)
2113  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2114  .addReg(Op0, getKillRegState(Op0IsKill))
2115  .addImm(Imm);
2116  else {
2118  .addReg(Op0, getKillRegState(Op0IsKill))
2119  .addImm(Imm);
2121  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2122  }
2123  return ResultReg;
2124 }
2125 
2126 unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2127  const TargetRegisterClass *RC, unsigned Op0,
2128  bool Op0IsKill, uint64_t Imm1,
2129  uint64_t Imm2) {
2130  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2131 
2132  unsigned ResultReg = createResultReg(RC);
2133  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2134 
2135  if (II.getNumDefs() >= 1)
2136  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2137  .addReg(Op0, getKillRegState(Op0IsKill))
2138  .addImm(Imm1)
2139  .addImm(Imm2);
2140  else {
2142  .addReg(Op0, getKillRegState(Op0IsKill))
2143  .addImm(Imm1)
2144  .addImm(Imm2);
2146  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2147  }
2148  return ResultReg;
2149 }
2150 
2151 unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2152  const TargetRegisterClass *RC,
2153  const ConstantFP *FPImm) {
2154  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2155 
2156  unsigned ResultReg = createResultReg(RC);
2157 
2158  if (II.getNumDefs() >= 1)
2159  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2160  .addFPImm(FPImm);
2161  else {
2163  .addFPImm(FPImm);
2165  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2166  }
2167  return ResultReg;
2168 }
2169 
2170 unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2171  const TargetRegisterClass *RC, unsigned Op0,
2172  bool Op0IsKill, unsigned Op1,
2173  bool Op1IsKill, uint64_t Imm) {
2174  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2175 
2176  unsigned ResultReg = createResultReg(RC);
2177  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2178  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2179 
2180  if (II.getNumDefs() >= 1)
2181  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2182  .addReg(Op0, getKillRegState(Op0IsKill))
2183  .addReg(Op1, getKillRegState(Op1IsKill))
2184  .addImm(Imm);
2185  else {
2187  .addReg(Op0, getKillRegState(Op0IsKill))
2188  .addReg(Op1, getKillRegState(Op1IsKill))
2189  .addImm(Imm);
2191  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2192  }
2193  return ResultReg;
2194 }
2195 
2196 unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2197  const TargetRegisterClass *RC, uint64_t Imm) {
2198  unsigned ResultReg = createResultReg(RC);
2199  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2200 
2201  if (II.getNumDefs() >= 1)
2202  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2203  .addImm(Imm);
2204  else {
2207  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2208  }
2209  return ResultReg;
2210 }
2211 
2212 unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2213  bool Op0IsKill, uint32_t Idx) {
2214  unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2216  "Cannot yet extract from physregs");
2217  const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2219  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
2220  ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
2221  return ResultReg;
2222 }
2223 
2224 /// Emit MachineInstrs to compute the value of Op with all but the least
2225 /// significant bit set to zero.
2226 unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
2227  return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
2228 }
2229 
2230 /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2231 /// Emit code to ensure constants are copied into registers when needed.
2232 /// Remember the virtual registers that need to be added to the Machine PHI
2233 /// nodes as input. We cannot just directly add them, because expansion
2234 /// might result in multiple MBB's for one BB. As such, the start of the
2235 /// BB might correspond to a different MBB than the end.
2236 bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2237  const Instruction *TI = LLVMBB->getTerminator();
2238 
2241 
2242  // Check successor nodes' PHI nodes that expect a constant to be available
2243  // from this block.
2244  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
2245  const BasicBlock *SuccBB = TI->getSuccessor(succ);
2246  if (!isa<PHINode>(SuccBB->begin()))
2247  continue;
2248  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2249 
2250  // If this terminator has multiple identical successors (common for
2251  // switches), only handle each succ once.
2252  if (!SuccsHandled.insert(SuccMBB).second)
2253  continue;
2254 
2255  MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2256 
2257  // At this point we know that there is a 1-1 correspondence between LLVM PHI
2258  // nodes and Machine PHI nodes, but the incoming operands have not been
2259  // emitted yet.
2260  for (const PHINode &PN : SuccBB->phis()) {
2261  // Ignore dead phi's.
2262  if (PN.use_empty())
2263  continue;
2264 
2265  // Only handle legal types. Two interesting things to note here. First,
2266  // by bailing out early, we may leave behind some dead instructions,
2267  // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2268  // own moves. Second, this check is necessary because FastISel doesn't
2269  // use CreateRegs to create registers, so it always creates
2270  // exactly one register for each non-void instruction.
2271  EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2272  if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2273  // Handle integer promotions, though, because they're common and easy.
2274  if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2276  return false;
2277  }
2278  }
2279 
2280  const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2281 
2282  // Set the DebugLoc for the copy. Prefer the location of the operand
2283  // if there is one; use the location of the PHI otherwise.
2284  DbgLoc = PN.getDebugLoc();
2285  if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2286  DbgLoc = Inst->getDebugLoc();
2287 
2288  unsigned Reg = getRegForValue(PHIOp);
2289  if (!Reg) {
2291  return false;
2292  }
2293  FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2294  DbgLoc = DebugLoc();
2295  }
2296  }
2297 
2298  return true;
2299 }
2300 
2301 bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2302  assert(LI->hasOneUse() &&
2303  "tryToFoldLoad expected a LoadInst with a single use");
2304  // We know that the load has a single use, but don't know what it is. If it
2305  // isn't one of the folded instructions, then we can't succeed here. Handle
2306  // this by scanning the single-use users of the load until we get to FoldInst.
2307  unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2308 
2309  const Instruction *TheUser = LI->user_back();
2310  while (TheUser != FoldInst && // Scan up until we find FoldInst.
2311  // Stay in the right block.
2312  TheUser->getParent() == FoldInst->getParent() &&
2313  --MaxUsers) { // Don't scan too far.
2314  // If there are multiple or no uses of this instruction, then bail out.
2315  if (!TheUser->hasOneUse())
2316  return false;
2317 
2318  TheUser = TheUser->user_back();
2319  }
2320 
2321  // If we didn't find the fold instruction, then we failed to collapse the
2322  // sequence.
2323  if (TheUser != FoldInst)
2324  return false;
2325 
2326  // Don't try to fold volatile loads. Target has to deal with alignment
2327  // constraints.
2328  if (LI->isVolatile())
2329  return false;
2330 
2331  // Figure out which vreg this is going into. If there is no assigned vreg yet
2332  // then there actually was no reference to it. Perhaps the load is referenced
2333  // by a dead instruction.
2334  unsigned LoadReg = getRegForValue(LI);
2335  if (!LoadReg)
2336  return false;
2337 
2338  // We can't fold if this vreg has no uses or more than one use. Multiple uses
2339  // may mean that the instruction got lowered to multiple MIs, or the use of
2340  // the loaded value ended up being multiple operands of the result.
2341  if (!MRI.hasOneUse(LoadReg))
2342  return false;
2343 
2345  MachineInstr *User = RI->getParent();
2346 
2347  // Set the insertion point properly. Folding the load can cause generation of
2348  // other random instructions (like sign extends) for addressing modes; make
2349  // sure they get inserted in a logical place before the new instruction.
2350  FuncInfo.InsertPt = User;
2351  FuncInfo.MBB = User->getParent();
2352 
2353  // Ask the target to try folding the load.
2354  return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2355 }
2356 
2358  // Must be an add.
2359  if (!isa<AddOperator>(Add))
2360  return false;
2361  // Type size needs to match.
2362  if (DL.getTypeSizeInBits(GEP->getType()) !=
2363  DL.getTypeSizeInBits(Add->getType()))
2364  return false;
2365  // Must be in the same basic block.
2366  if (isa<Instruction>(Add) &&
2367  FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2368  return false;
2369  // Must have a constant operand.
2370  return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2371 }
2372 
2375  const Value *Ptr;
2376  Type *ValTy;
2377  unsigned Alignment;
2379  bool IsVolatile;
2380 
2381  if (const auto *LI = dyn_cast<LoadInst>(I)) {
2382  Alignment = LI->getAlignment();
2383  IsVolatile = LI->isVolatile();
2384  Flags = MachineMemOperand::MOLoad;
2385  Ptr = LI->getPointerOperand();
2386  ValTy = LI->getType();
2387  } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2388  Alignment = SI->getAlignment();
2389  IsVolatile = SI->isVolatile();
2391  Ptr = SI->getPointerOperand();
2392  ValTy = SI->getValueOperand()->getType();
2393  } else
2394  return nullptr;
2395 
2396  bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
2397  bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
2398  bool IsDereferenceable =
2400  const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2401 
2402  AAMDNodes AAInfo;
2403  I->getAAMetadata(AAInfo);
2404 
2405  if (Alignment == 0) // Ensure that codegen never sees alignment 0.
2406  Alignment = DL.getABITypeAlignment(ValTy);
2407 
2408  unsigned Size = DL.getTypeStoreSize(ValTy);
2409 
2410  if (IsVolatile)
2412  if (IsNonTemporal)
2414  if (IsDereferenceable)
2416  if (IsInvariant)
2418 
2419  return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
2420  Alignment, AAInfo, Ranges);
2421 }
2422 
2424  // If both operands are the same, then try to optimize or fold the cmp.
2426  if (CI->getOperand(0) != CI->getOperand(1))
2427  return Predicate;
2428 
2429  switch (Predicate) {
2430  default: llvm_unreachable("Invalid predicate!");
2431  case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2432  case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2433  case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2434  case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2435  case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2436  case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2437  case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2438  case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2439  case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2440  case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2441  case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2442  case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2443  case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2444  case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2445  case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2446  case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2447 
2448  case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2449  case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2450  case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2451  case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2452  case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2453  case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2454  case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2455  case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2456  case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2457  case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2458  }
2459 
2460  return Predicate;
2461 }
void setHasStackMap(bool s=true)
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:564
unsigned fastEmitInst_rrr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, unsigned Op2, bool Op2IsKill)
Emit a MachineInstr with three register operands and a result register in the given register class...
Definition: FastISel.cpp:2076
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
void setByValAlign(unsigned A)
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:72
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:632
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
unsigned fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill)
Emit MachineInstrs to compute the value of Op with all but the least significant bit set to zero...
Definition: FastISel.cpp:2226
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
bool hasLocalLinkage() const
Definition: GlobalValue.h:436
This instruction extracts a struct member or array element value from an aggregate value...
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:30
bool lowerCall(const CallInst *I)
Definition: FastISel.cpp:1232
unsigned arg_size() const
Definition: CallSite.h:219
CallingConv::ID getCallingConv() const
Get the calling convention of the call.
Definition: CallSite.h:312
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
This represents the llvm.dbg.label instruction.
This class represents lattice values for constants.
Definition: AllocatorList.h:24
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:42
BasicBlock * getSuccessor(unsigned Idx) const
Return the specified successor. This instruction must be a terminator.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
bool selectXRayTypedEvent(const CallInst *II)
Definition: FastISel.cpp:1052
ImmutableCallSite * CS
Definition: FastISel.h:90
bool selectGetElementPtr(const User *I)
Definition: FastISel.cpp:667
void leaveLocalValueArea(SavePoint Old)
Reset InsertPt to the given old insert position.
Definition: FastISel.cpp:567
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:164
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
bool selectStackmap(const CallInst *I)
Definition: FastISel.cpp:782
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:305
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:668
unsigned less than
Definition: InstrTypes.h:667
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:648
MachineMemOperand * createMachineMemOperandFor(const Instruction *I) const
Create a machine mem operand from the given instruction.
Definition: FastISel.cpp:2374
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:705
bool isTerminator() const
Definition: Instruction.h:129
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:658
virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode)
This method is called by target-independent code to request that an instruction with the given type a...
Definition: FastISel.cpp:1927
BasicBlock * getSuccessor(unsigned i) const
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:34
Metadata node.
Definition: Metadata.h:864
F(f)
SmallVector< unsigned, 4 > InRegs
Definition: FastISel.h:99
An instruction for reading from memory.
Definition: Instructions.h:168
Hexagon Common GEP
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:138
virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1940
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:459
void setPhysRegsDeadExcept(ArrayRef< unsigned > UsedRegs, const TargetRegisterInfo &TRI)
Mark every physreg used by this instruction as dead except those in the UsedRegs list.
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
void computeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo &MMI)
Determine if any floating-point values are being passed to this variadic function, and set the MachineModuleInfo&#39;s usesVAFloatArgument flag if so.
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1921
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:130
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:265
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:269
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1586
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1079
unsigned fastEmitInst_rii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm1, uint64_t Imm2)
Emit a MachineInstr with one register operand and two immediate operands.
Definition: FastISel.cpp:2126
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1069
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:653
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
unsigned fastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm)
Emit a MachineInstr with a register operand, an immediate, and a result register in the given registe...
Definition: FastISel.cpp:2104
ArrayRef< unsigned > getIndices() const
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:652
unsigned fastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm)
Emit a MachineInstr with two register operands, an immediate, and a result register in the given regi...
Definition: FastISel.cpp:2170
CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const
Definition: FastISel.cpp:2423
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:221
A description of a memory reference used in the backend.
const HexagonInstrInfo * TII
TargetLoweringBase::ArgListTy ArgListTy
Definition: FastISel.h:70
Shift and rotation operations.
Definition: ISDOpcodes.h:403
const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
Class to represent struct types.
Definition: DerivedTypes.h:201
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
DILabel * getLabel() const
bool canFoldAddIntoGEP(const User *GEP, const Value *Add)
Check if Add is an add that can be safely folded into GEP.
Definition: FastISel.cpp:2357
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i...
IterTy arg_end() const
Definition: CallSite.h:575
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned fastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill)
Emit a MachineInstr with one register operand and a result register in the given register class...
Definition: FastISel.cpp:2031
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:649
This file contains the simple types necessary to represent the attributes associated with functions a...
InstrTy * getInstruction() const
Definition: CallSite.h:92
The memory access is dereferenceable (i.e., doesn&#39;t trap).
static MachineOperand CreateRegMask(const uint32_t *Mask)
CreateRegMask - Creates a register mask operand referencing Mask.
virtual const TargetRegisterClass * getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const
Returns the largest legal sub-class of RC that supports the sub-register index Idx.
void setByValSize(unsigned S)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:660
bool selectIntrinsicCall(const IntrinsicInst *II)
Definition: FastISel.cpp:1317
bool selectCast(const User *I, unsigned Opcode)
Definition: FastISel.cpp:1485
unsigned getSizeInBits() const
Context object for machine code objects.
Definition: MCContext.h:63
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
SmallVector< ISD::InputArg, 4 > Ins
Definition: FastISel.h:98
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
unsigned constrainOperandRegClass(const MCInstrDesc &II, unsigned Op, unsigned OpNum)
Try to constrain Op so that it is usable by argument OpNum of the provided MCInstrDesc.
Definition: FastISel.cpp:2005
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:471
bool selectOperator(const User *I, unsigned Opcode)
Do "fast" instruction selection for the given LLVM IR operator (Instruction or ConstantExpr), and append generated machine instructions to the current block.
Definition: FastISel.cpp:1790
ArchType getArch() const
getArch - Get the parsed architecture type of this triple.
Definition: Triple.h:290
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
bool selectExtractValue(const User *U)
Definition: FastISel.cpp:1750
unsigned getRegForValue(const Value *V)
Create a virtual register and arrange for it to be assigned the value for the given LLVM value...
Definition: FastISel.cpp:363
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
unsigned fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC)
Emit a MachineInstr with no operands and a result register in the given register class.
Definition: FastISel.cpp:2022
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:221
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:126
bool hasTrivialKill(const Value *V)
Test whether the given value has exactly one use.
Definition: FastISel.cpp:331
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
void setOrigAlign(unsigned A)
amdgpu Simplify well known AMD library false Value * Callee
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:84
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
unsigned getNumSuccessors() const
Return the number of successors that this instruction has.
Value * getOperand(unsigned i) const
Definition: User.h:170
Class to represent pointers.
Definition: DerivedTypes.h:467
unsigned getKillRegState(bool B)
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:517
unsigned lookUpRegForValue(const Value *V)
Look up the value to see if its value is already cached in a register.
Definition: FastISel.cpp:476
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
void finishBasicBlock()
Flush the local value map and sink local values if possible.
Definition: FastISel.cpp:145
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:229
void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:141
The memory access is volatile.
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc (such as G_OR or G_SDIV)...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:419
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:169
virtual ~FastISel()
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
unsigned const MachineRegisterInfo * MRI
CallLoweringInfo & setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, ImmutableCallSite &Call)
Definition: FastISel.h:105
Machine Value Type.
Value * getCalledValue() const
Definition: InstrTypes.h:1105
bool hasName() const
Definition: Value.h:251
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
const MachineInstrBuilder & addCImm(const ConstantInt *Val) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:276
Conditional or Unconditional Branch instruction.
Value * getAddress() const
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:42
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:545
Value * getValue() const
SmallVector< ISD::ArgFlagsTy, 16 > OutFlags
Definition: FastISel.h:96
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:264
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:371
const MCPhysReg * ImplicitDefs
Definition: MCInstrDesc.h:174
DIExpression * getExpression() const
virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1929
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:434
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:161
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We&#39;re checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2301
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:147
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:642
std::pair< unsigned, bool > getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
Definition: FastISel.cpp:508
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned char TargetFlags=0)
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:760
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:651
self_iterator getIterator()
Definition: ilist_node.h:82
The memory access is non-temporal.
Class to represent integer types.
Definition: DerivedTypes.h:40
bool selectXRayCustomEvent(const CallInst *II)
Definition: FastISel.cpp:1033
const TargetRegisterInfo & TRI
Definition: FastISel.h:214
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:659
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:194
static unsigned findSinkableLocalRegDef(MachineInstr &MI)
Return the defined register if this instruction defines exactly one virtual register and uses no othe...
Definition: FastISel.cpp:169
Extended Value Type.
Definition: ValueTypes.h:34
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:530
bool selectFNeg(const User *I)
Emit an FNeg operation.
Definition: FastISel.cpp:1704
size_t size() const
Definition: SmallVector.h:53
This class contains a discriminated union of information about pointers in memory operands...
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:657
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
SmallVector< Value *, 16 > OutVals
Definition: FastISel.h:95
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call...
Definition: FastISel.cpp:1075
const TargetInstrInfo & TII
Definition: FastISel.h:212
MachineBasicBlock * MBB
MBB - The current block.
bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:471
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
signed greater than
Definition: InstrTypes.h:669
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:665
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:531
The memory access writes data.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:51
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:646
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:240
virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1934
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:418
This is the shared class of boolean and integer constants.
Definition: Constants.h:84
virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1949
IterTy arg_begin() const
Definition: CallSite.h:571
MachineOperand class - Representation of each machine instruction operand.
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:656
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:847
bool selectCall(const User *I)
Definition: FastISel.cpp:1271
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:64
Provides information about what library functions are available for the current target.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:27
void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB)
Emit an unconditional branch to FalseMBB, obtains the branch weight and adds TrueMBB and FalseMBB to ...
Definition: FastISel.cpp:1685
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:722
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:577
signed less than
Definition: InstrTypes.h:671
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:644
reg_iterator reg_begin(unsigned RegNo) const
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
unsigned fastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
Emit a MachineInstr with two register operands and a result register in the given register class...
Definition: FastISel.cpp:2052
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:618
void updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs=1)
Update the value map to include the new mapping for this instruction, or insert an extra copy to get ...
Definition: FastISel.cpp:487
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:226
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:194
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
Definition: FastISel.cpp:131
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
signed less or equal
Definition: InstrTypes.h:672
bool selectBitCast(const User *I)
Definition: FastISel.cpp:1518
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Target - Wrapper for Target specific information.
virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1944
SmallVector< unsigned, 16 > OutRegs
Definition: FastISel.h:97
const DataLayout & DL
Definition: FastISel.h:211
bool selectBinaryOp(const User *I, unsigned ISDOpcode)
Select and emit code for a binary operator instruction, which has an opcode which directly correspond...
Definition: FastISel.cpp:576
void setDebugLoc(DebugLoc dl)
Replace current source information with new such.
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringRef > StandardNames)
Initialize the set of available library functions based on the specified target triple.
This file defines the FastISel class.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:461
bool isTailCall() const
DebugLoc DbgLoc
Definition: FastISel.h:209
Flags
Flags values. These may be or&#39;d together.
amdgpu Simplify well known AMD library false Value Value * Arg
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:254
The memory access reads data.
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:560
SavePoint enterLocalValueArea()
Prepare InsertPt to begin inserting instructions into the local value area and return the old insert ...
Definition: FastISel.cpp:558
Representation of each machine instruction.
Definition: MachineInstr.h:64
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:717
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering...
Definition: FastISel.cpp:1923
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match &#39;fneg X&#39; as &#39;fsub -0.0, X&#39;.
Definition: PatternMatch.h:690
bool selectPatchpoint(const CallInst *I)
Definition: FastISel.cpp:895
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1077
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:311
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:380
MachineRegisterInfo & MRI
Definition: FastISel.h:206
bool hasOneUse(unsigned RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:123
unsigned greater or equal
Definition: InstrTypes.h:666
This represents the llvm.dbg.value instruction.
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1134
bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs)
Definition: FastISel.cpp:1097
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:215
Establish a view to a call site for examination.
Definition: CallSite.h:711
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1112
unsigned fastEmitInst_i(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm)
Emit a MachineInstr with a single immediate operand, and a result register in the given register clas...
Definition: FastISel.cpp:2196
static MachineOperand CreateImm(int64_t Val)
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call&#39;s live variable operands to a stackmap or patchpoint target node&#39;s ope...
The memory access always returns the same value (or traps).
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:193
static bool isRegUsedByPhiNodes(unsigned DefReg, FunctionLoweringInfo &FuncInfo)
Definition: FastISel.cpp:221
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:650
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
uint32_t Size
Definition: Profile.cpp:47
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:325
DILocalVariable * getVariable() const
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool isUnconditional() const
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:654
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:309
Type * getType() const
Return the type of the instruction that generated this call site.
Definition: CallSite.h:264
const TargetLowering & TLI
Definition: FastISel.h:213
unsigned createResultReg(const TargetRegisterClass *RC)
Definition: FastISel.cpp:2001
CallLoweringInfo & setIsPatchPoint(bool Value=true)
Definition: FastISel.h:183
unsigned fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm, MVT ImmType)
This method is a wrapper of fastEmit_ri.
Definition: FastISel.cpp:1958
unsigned fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, bool Op0IsKill, uint32_t Idx)
Emit a MachineInstr for an extract_subreg from a specified index of a superregister to a specified ty...
Definition: FastISel.cpp:2212
MachineBasicBlock::iterator InsertPt
Definition: FastISel.h:319
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
Definition: Value.h:376
FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, bool SkipTargetIndependentISel=false)
Definition: FastISel.cpp:1906
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1919
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:645
LLVM Value Representation.
Definition: Value.h:73
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:655
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:411
FunctionType * getFunctionType() const
Definition: CallSite.h:320
static const Function * getParent(const Value *V)
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable&#39;s name.
Definition: Mangler.cpp:112
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
IRTranslator LLVM IR MI
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:413
void fastEmitBranch(MachineBasicBlock *MSucc, const DebugLoc &DbgLoc)
Emit an unconditional branch to the given block, unless it is the immediate (fall-through) successor...
Definition: FastISel.cpp:1665
unsigned greater than
Definition: InstrTypes.h:665
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
static cl::opt< bool > SinkLocalValues("fast-isel-sink-local-values", cl::init(true), cl::Hidden, cl::desc("Sink local values in FastISel"))
unsigned fastEmitInst_f(unsigned MachineInstOpcode, const TargetRegisterClass *RC, const ConstantFP *FPImm)
Emit a MachineInstr with a floating point immediate, and a result register in the given register clas...
Definition: FastISel.cpp:2151
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty...
Definition: Type.cpp:98
Conversion operators.
Definition: ISDOpcodes.h:458
FunctionLoweringInfo & FuncInfo
Definition: FastISel.h:204
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
Definition: Value.cpp:538
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:467
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:126
#define LLVM_DEBUG(X)
Definition: Debug.h:123
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:647
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:36
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
This represents the llvm.dbg.declare instruction.
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:577
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
static MachineOperand CreateFI(int Idx)
bool use_empty() const
Definition: Value.h:323
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:545
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Type * getElementType() const
Definition: DerivedTypes.h:486
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:873
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:644
signed greater or equal
Definition: InstrTypes.h:670
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
This class contains meta information specific to a module.
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:67
gep_type_iterator gep_type_begin(const User *GEP)