LLVM  9.0.0svn
FastISel.cpp
Go to the documentation of this file.
1 //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the implementation of the FastISel class.
10 //
11 // "Fast" instruction selection is designed to emit very poor code quickly.
12 // Also, it is not designed to be able to do much lowering, so most illegal
13 // types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14 // also not intended to be able to do much optimization, except in a few cases
15 // where doing optimizations reduces overall compile time. For example, folding
16 // constants into immediate fields is often done, because it's cheap and it
17 // reduces the number of instructions later phases have to examine.
18 //
19 // "Fast" instruction selection is able to fail gracefully and transfer
20 // control to the SelectionDAG selector for operations that it doesn't
21 // support. In many cases, this allows us to avoid duplicating a lot of
22 // the complicated lowering logic that SelectionDAG currently has.
23 //
24 // The intended use for "fast" instruction selection is "-O0" mode
25 // compilation, where the quality of the generated code is irrelevant when
26 // weighed against the speed at which the code can be generated. Also,
27 // at -O0, the LLVM optimizers are not running, and this makes the
28 // compile time of codegen a much higher portion of the overall compile
29 // time. Despite its limitations, "fast" instruction selection is able to
30 // handle enough code on its own to provide noticeable overall speedups
31 // in -O0 compiles.
32 //
33 // Basic operations are supported in a target-independent way, by reading
34 // the same instruction descriptions that the SelectionDAG selector reads,
35 // and identifying simple arithmetic operations that can be directly selected
36 // from simple operators. More complicated operations currently require
37 // target-specific code.
38 //
39 //===----------------------------------------------------------------------===//
40 
41 #include "llvm/CodeGen/FastISel.h"
42 #include "llvm/ADT/APFloat.h"
43 #include "llvm/ADT/APSInt.h"
44 #include "llvm/ADT/DenseMap.h"
45 #include "llvm/ADT/Optional.h"
46 #include "llvm/ADT/SmallPtrSet.h"
47 #include "llvm/ADT/SmallString.h"
48 #include "llvm/ADT/SmallVector.h"
49 #include "llvm/ADT/Statistic.h"
52 #include "llvm/CodeGen/Analysis.h"
63 #include "llvm/CodeGen/StackMaps.h"
68 #include "llvm/IR/Argument.h"
69 #include "llvm/IR/Attributes.h"
70 #include "llvm/IR/BasicBlock.h"
71 #include "llvm/IR/CallSite.h"
72 #include "llvm/IR/CallingConv.h"
73 #include "llvm/IR/Constant.h"
74 #include "llvm/IR/Constants.h"
75 #include "llvm/IR/DataLayout.h"
76 #include "llvm/IR/DebugInfo.h"
77 #include "llvm/IR/DebugLoc.h"
78 #include "llvm/IR/DerivedTypes.h"
79 #include "llvm/IR/Function.h"
81 #include "llvm/IR/GlobalValue.h"
82 #include "llvm/IR/InlineAsm.h"
83 #include "llvm/IR/InstrTypes.h"
84 #include "llvm/IR/Instruction.h"
85 #include "llvm/IR/Instructions.h"
86 #include "llvm/IR/IntrinsicInst.h"
87 #include "llvm/IR/LLVMContext.h"
88 #include "llvm/IR/Mangler.h"
89 #include "llvm/IR/Metadata.h"
90 #include "llvm/IR/Operator.h"
91 #include "llvm/IR/PatternMatch.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCInstrDesc.h"
97 #include "llvm/MC/MCRegisterInfo.h"
98 #include "llvm/Support/Casting.h"
99 #include "llvm/Support/Debug.h"
102 #include "llvm/Support/MathExtras.h"
106 #include <algorithm>
107 #include <cassert>
108 #include <cstdint>
109 #include <iterator>
110 #include <utility>
111 
112 using namespace llvm;
113 using namespace PatternMatch;
114 
115 #define DEBUG_TYPE "isel"
116 
117 // FIXME: Remove this after the feature has proven reliable.
118 static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values",
119  cl::init(true), cl::Hidden,
120  cl::desc("Sink local values in FastISel"));
121 
122 STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
123  "target-independent selector");
124 STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
125  "target-specific selector");
126 STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
127 
128 /// Set the current block to which generated machine instructions will be
129 /// appended.
131  assert(LocalValueMap.empty() &&
132  "local values should be cleared after finishing a BB");
133 
134  // Instructions are appended to FuncInfo.MBB. If the basic block already
135  // contains labels or copies, use the last instruction as the last local
136  // value.
137  EmitStartPt = nullptr;
138  if (!FuncInfo.MBB->empty())
139  EmitStartPt = &FuncInfo.MBB->back();
140  LastLocalValue = EmitStartPt;
141 }
142 
143 /// Flush the local CSE map and sink anything we can.
144 void FastISel::finishBasicBlock() { flushLocalValueMap(); }
145 
147  if (!FuncInfo.CanLowerReturn)
148  // Fallback to SDISel argument lowering code to deal with sret pointer
149  // parameter.
150  return false;
151 
152  if (!fastLowerArguments())
153  return false;
154 
155  // Enter arguments into ValueMap for uses in non-entry BBs.
156  for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
157  E = FuncInfo.Fn->arg_end();
158  I != E; ++I) {
159  DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(&*I);
160  assert(VI != LocalValueMap.end() && "Missed an argument?");
161  FuncInfo.ValueMap[&*I] = VI->second;
162  }
163  return true;
164 }
165 
166 /// Return the defined register if this instruction defines exactly one
167 /// virtual register and uses no other virtual registers. Otherwise return 0.
169  unsigned RegDef = 0;
170  for (const MachineOperand &MO : MI.operands()) {
171  if (!MO.isReg())
172  continue;
173  if (MO.isDef()) {
174  if (RegDef)
175  return 0;
176  RegDef = MO.getReg();
177  } else if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
178  // This is another use of a vreg. Don't try to sink it.
179  return 0;
180  }
181  }
182  return RegDef;
183 }
184 
185 void FastISel::flushLocalValueMap() {
186  // Try to sink local values down to their first use so that we can give them a
187  // better debug location. This has the side effect of shrinking local value
188  // live ranges, which helps out fast regalloc.
189  if (SinkLocalValues && LastLocalValue != EmitStartPt) {
190  // Sink local value materialization instructions between EmitStartPt and
191  // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
192  // avoid inserting into the range that we're iterating over.
194  EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
195  : FuncInfo.MBB->rend();
196  MachineBasicBlock::reverse_iterator RI(LastLocalValue);
197 
198  InstOrderMap OrderMap;
199  for (; RI != RE;) {
200  MachineInstr &LocalMI = *RI;
201  ++RI;
202  bool Store = true;
203  if (!LocalMI.isSafeToMove(nullptr, Store))
204  continue;
205  unsigned DefReg = findSinkableLocalRegDef(LocalMI);
206  if (DefReg == 0)
207  continue;
208 
209  sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
210  }
211  }
212 
213  LocalValueMap.clear();
214  LastLocalValue = EmitStartPt;
215  recomputeInsertPt();
216  SavedInsertPt = FuncInfo.InsertPt;
217  LastFlushPoint = FuncInfo.InsertPt;
218 }
219 
220 static bool isRegUsedByPhiNodes(unsigned DefReg,
221  FunctionLoweringInfo &FuncInfo) {
222  for (auto &P : FuncInfo.PHINodesToUpdate)
223  if (P.second == DefReg)
224  return true;
225  return false;
226 }
227 
228 /// Build a map of instruction orders. Return the first terminator and its
229 /// order. Consider EH_LABEL instructions to be terminators as well, since local
230 /// values for phis after invokes must be materialized before the call.
232  MachineBasicBlock *MBB, MachineBasicBlock::iterator LastFlushPoint) {
233  unsigned Order = 0;
234  for (MachineInstr &I : *MBB) {
235  if (!FirstTerminator &&
236  (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) {
237  FirstTerminator = &I;
238  FirstTerminatorOrder = Order;
239  }
240  Orders[&I] = Order++;
241 
242  // We don't need to order instructions past the last flush point.
243  if (I.getIterator() == LastFlushPoint)
244  break;
245  }
246 }
247 
248 void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
249  unsigned DefReg,
250  InstOrderMap &OrderMap) {
251  // If this register is used by a register fixup, MRI will not contain all
252  // the uses until after register fixups, so don't attempt to sink or DCE
253  // this instruction. Register fixups typically come from no-op cast
254  // instructions, which replace the cast instruction vreg with the local
255  // value vreg.
256  if (FuncInfo.RegsWithFixups.count(DefReg))
257  return;
258 
259  // We can DCE this instruction if there are no uses and it wasn't a
260  // materialized for a successor PHI node.
261  bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
262  if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
263  if (EmitStartPt == &LocalMI)
264  EmitStartPt = EmitStartPt->getPrevNode();
265  LLVM_DEBUG(dbgs() << "removing dead local value materialization "
266  << LocalMI);
267  OrderMap.Orders.erase(&LocalMI);
268  LocalMI.eraseFromParent();
269  return;
270  }
271 
272  // Number the instructions if we haven't yet so we can efficiently find the
273  // earliest use.
274  if (OrderMap.Orders.empty())
275  OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
276 
277  // Find the first user in the BB.
278  MachineInstr *FirstUser = nullptr;
279  unsigned FirstOrder = std::numeric_limits<unsigned>::max();
280  for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
281  auto I = OrderMap.Orders.find(&UseInst);
282  assert(I != OrderMap.Orders.end() &&
283  "local value used by instruction outside local region");
284  unsigned UseOrder = I->second;
285  if (UseOrder < FirstOrder) {
286  FirstOrder = UseOrder;
287  FirstUser = &UseInst;
288  }
289  }
290 
291  // The insertion point will be the first terminator or the first user,
292  // whichever came first. If there was no terminator, this must be a
293  // fallthrough block and the insertion point is the end of the block.
295  if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder) {
296  FirstOrder = OrderMap.FirstTerminatorOrder;
297  SinkPos = OrderMap.FirstTerminator->getIterator();
298  } else if (FirstUser) {
299  SinkPos = FirstUser->getIterator();
300  } else {
301  assert(UsedByPHI && "must be users if not used by a phi");
302  SinkPos = FuncInfo.MBB->instr_end();
303  }
304 
305  // Collect all DBG_VALUEs before the new insertion position so that we can
306  // sink them.
308  for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
309  if (!DbgVal.isDebugValue())
310  continue;
311  unsigned UseOrder = OrderMap.Orders[&DbgVal];
312  if (UseOrder < FirstOrder)
313  DbgValues.push_back(&DbgVal);
314  }
315 
316  // Sink LocalMI before SinkPos and assign it the same DebugLoc.
317  LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
318  FuncInfo.MBB->remove(&LocalMI);
319  FuncInfo.MBB->insert(SinkPos, &LocalMI);
320  if (SinkPos != FuncInfo.MBB->end())
321  LocalMI.setDebugLoc(SinkPos->getDebugLoc());
322 
323  // Sink any debug values that we've collected.
324  for (MachineInstr *DI : DbgValues) {
325  FuncInfo.MBB->remove(DI);
326  FuncInfo.MBB->insert(SinkPos, DI);
327  }
328 }
329 
331  // Don't consider constants or arguments to have trivial kills.
332  const Instruction *I = dyn_cast<Instruction>(V);
333  if (!I)
334  return false;
335 
336  // No-op casts are trivially coalesced by fast-isel.
337  if (const auto *Cast = dyn_cast<CastInst>(I))
338  if (Cast->isNoopCast(DL) && !hasTrivialKill(Cast->getOperand(0)))
339  return false;
340 
341  // Even the value might have only one use in the LLVM IR, it is possible that
342  // FastISel might fold the use into another instruction and now there is more
343  // than one use at the Machine Instruction level.
344  unsigned Reg = lookUpRegForValue(V);
345  if (Reg && !MRI.use_empty(Reg))
346  return false;
347 
348  // GEPs with all zero indices are trivially coalesced by fast-isel.
349  if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
350  if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0)))
351  return false;
352 
353  // Only instructions with a single use in the same basic block are considered
354  // to have trivial kills.
355  return I->hasOneUse() &&
356  !(I->getOpcode() == Instruction::BitCast ||
357  I->getOpcode() == Instruction::PtrToInt ||
358  I->getOpcode() == Instruction::IntToPtr) &&
359  cast<Instruction>(*I->user_begin())->getParent() == I->getParent();
360 }
361 
362 unsigned FastISel::getRegForValue(const Value *V) {
363  EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
364  // Don't handle non-simple values in FastISel.
365  if (!RealVT.isSimple())
366  return 0;
367 
368  // Ignore illegal types. We must do this before looking up the value
369  // in ValueMap because Arguments are given virtual registers regardless
370  // of whether FastISel can handle them.
371  MVT VT = RealVT.getSimpleVT();
372  if (!TLI.isTypeLegal(VT)) {
373  // Handle integer promotions, though, because they're common and easy.
374  if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
375  VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
376  else
377  return 0;
378  }
379 
380  // Look up the value to see if we already have a register for it.
381  unsigned Reg = lookUpRegForValue(V);
382  if (Reg)
383  return Reg;
384 
385  // In bottom-up mode, just create the virtual register which will be used
386  // to hold the value. It will be materialized later.
387  if (isa<Instruction>(V) &&
388  (!isa<AllocaInst>(V) ||
389  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
390  return FuncInfo.InitializeRegForValue(V);
391 
392  SavePoint SaveInsertPt = enterLocalValueArea();
393 
394  // Materialize the value in a register. Emit any instructions in the
395  // local value area.
396  Reg = materializeRegForValue(V, VT);
397 
398  leaveLocalValueArea(SaveInsertPt);
399 
400  return Reg;
401 }
402 
403 unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
404  unsigned Reg = 0;
405  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
406  if (CI->getValue().getActiveBits() <= 64)
407  Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
408  } else if (isa<AllocaInst>(V))
409  Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
410  else if (isa<ConstantPointerNull>(V))
411  // Translate this as an integer zero so that it can be
412  // local-CSE'd with actual integer zeros.
413  Reg = getRegForValue(
414  Constant::getNullValue(DL.getIntPtrType(V->getContext())));
415  else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
416  if (CF->isNullValue())
417  Reg = fastMaterializeFloatZero(CF);
418  else
419  // Try to emit the constant directly.
420  Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
421 
422  if (!Reg) {
423  // Try to emit the constant by using an integer constant with a cast.
424  const APFloat &Flt = CF->getValueAPF();
425  EVT IntVT = TLI.getPointerTy(DL);
426  uint32_t IntBitWidth = IntVT.getSizeInBits();
427  APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
428  bool isExact;
429  (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
430  if (isExact) {
431  unsigned IntegerReg =
432  getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
433  if (IntegerReg != 0)
434  Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
435  /*Kill=*/false);
436  }
437  }
438  } else if (const auto *Op = dyn_cast<Operator>(V)) {
439  if (!selectOperator(Op, Op->getOpcode()))
440  if (!isa<Instruction>(Op) ||
441  !fastSelectInstruction(cast<Instruction>(Op)))
442  return 0;
443  Reg = lookUpRegForValue(Op);
444  } else if (isa<UndefValue>(V)) {
445  Reg = createResultReg(TLI.getRegClassFor(VT));
446  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
447  TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
448  }
449  return Reg;
450 }
451 
452 /// Helper for getRegForValue. This function is called when the value isn't
453 /// already available in a register and must be materialized with new
454 /// instructions.
455 unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
456  unsigned Reg = 0;
457  // Give the target-specific code a try first.
458  if (isa<Constant>(V))
459  Reg = fastMaterializeConstant(cast<Constant>(V));
460 
461  // If target-specific code couldn't or didn't want to handle the value, then
462  // give target-independent code a try.
463  if (!Reg)
464  Reg = materializeConstant(V, VT);
465 
466  // Don't cache constant materializations in the general ValueMap.
467  // To do so would require tracking what uses they dominate.
468  if (Reg) {
469  LocalValueMap[V] = Reg;
470  LastLocalValue = MRI.getVRegDef(Reg);
471  }
472  return Reg;
473 }
474 
475 unsigned FastISel::lookUpRegForValue(const Value *V) {
476  // Look up the value to see if we already have a register for it. We
477  // cache values defined by Instructions across blocks, and other values
478  // only locally. This is because Instructions already have the SSA
479  // def-dominates-use requirement enforced.
480  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(V);
481  if (I != FuncInfo.ValueMap.end())
482  return I->second;
483  return LocalValueMap[V];
484 }
485 
486 void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
487  if (!isa<Instruction>(I)) {
488  LocalValueMap[I] = Reg;
489  return;
490  }
491 
492  unsigned &AssignedReg = FuncInfo.ValueMap[I];
493  if (AssignedReg == 0)
494  // Use the new register.
495  AssignedReg = Reg;
496  else if (Reg != AssignedReg) {
497  // Arrange for uses of AssignedReg to be replaced by uses of Reg.
498  for (unsigned i = 0; i < NumRegs; i++) {
499  FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
500  FuncInfo.RegsWithFixups.insert(Reg + i);
501  }
502 
503  AssignedReg = Reg;
504  }
505 }
506 
507 std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
508  unsigned IdxN = getRegForValue(Idx);
509  if (IdxN == 0)
510  // Unhandled operand. Halt "fast" selection and bail.
511  return std::pair<unsigned, bool>(0, false);
512 
513  bool IdxNIsKill = hasTrivialKill(Idx);
514 
515  // If the index is smaller or larger than intptr_t, truncate or extend it.
516  MVT PtrVT = TLI.getPointerTy(DL);
517  EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
518  if (IdxVT.bitsLT(PtrVT)) {
519  IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
520  IdxNIsKill);
521  IdxNIsKill = true;
522  } else if (IdxVT.bitsGT(PtrVT)) {
523  IdxN =
524  fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
525  IdxNIsKill = true;
526  }
527  return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
528 }
529 
531  if (getLastLocalValue()) {
532  FuncInfo.InsertPt = getLastLocalValue();
533  FuncInfo.MBB = FuncInfo.InsertPt->getParent();
534  ++FuncInfo.InsertPt;
535  } else
536  FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
537 
538  // Now skip past any EH_LABELs, which must remain at the beginning.
539  while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
540  FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL)
541  ++FuncInfo.InsertPt;
542 }
543 
546  assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
547  "Invalid iterator!");
548  while (I != E) {
549  if (LastFlushPoint == I)
550  LastFlushPoint = E;
551  if (SavedInsertPt == I)
552  SavedInsertPt = E;
553  if (EmitStartPt == I)
554  EmitStartPt = E.isValid() ? &*E : nullptr;
555  if (LastLocalValue == I)
556  LastLocalValue = E.isValid() ? &*E : nullptr;
557 
558  MachineInstr *Dead = &*I;
559  ++I;
560  Dead->eraseFromParent();
561  ++NumFastIselDead;
562  }
563  recomputeInsertPt();
564 }
565 
567  MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt;
568  DebugLoc OldDL = DbgLoc;
569  recomputeInsertPt();
570  DbgLoc = DebugLoc();
571  SavePoint SP = {OldInsertPt, OldDL};
572  return SP;
573 }
574 
576  if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
577  LastLocalValue = &*std::prev(FuncInfo.InsertPt);
578 
579  // Restore the previous insert position.
580  FuncInfo.InsertPt = OldInsertPt.InsertPt;
581  DbgLoc = OldInsertPt.DL;
582 }
583 
584 bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
585  EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
586  if (VT == MVT::Other || !VT.isSimple())
587  // Unhandled type. Halt "fast" selection and bail.
588  return false;
589 
590  // We only handle legal types. For example, on x86-32 the instruction
591  // selector contains all of the 64-bit instructions from x86-64,
592  // under the assumption that i64 won't be used if the target doesn't
593  // support it.
594  if (!TLI.isTypeLegal(VT)) {
595  // MVT::i1 is special. Allow AND, OR, or XOR because they
596  // don't require additional zeroing, which makes them easy.
597  if (VT == MVT::i1 && (ISDOpcode == ISD::AND || ISDOpcode == ISD::OR ||
598  ISDOpcode == ISD::XOR))
599  VT = TLI.getTypeToTransformTo(I->getContext(), VT);
600  else
601  return false;
602  }
603 
604  // Check if the first operand is a constant, and handle it as "ri". At -O0,
605  // we don't have anything that canonicalizes operand order.
606  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
607  if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
608  unsigned Op1 = getRegForValue(I->getOperand(1));
609  if (!Op1)
610  return false;
611  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
612 
613  unsigned ResultReg =
614  fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
615  CI->getZExtValue(), VT.getSimpleVT());
616  if (!ResultReg)
617  return false;
618 
619  // We successfully emitted code for the given LLVM Instruction.
620  updateValueMap(I, ResultReg);
621  return true;
622  }
623 
624  unsigned Op0 = getRegForValue(I->getOperand(0));
625  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
626  return false;
627  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
628 
629  // Check if the second operand is a constant and handle it appropriately.
630  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
631  uint64_t Imm = CI->getSExtValue();
632 
633  // Transform "sdiv exact X, 8" -> "sra X, 3".
634  if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
635  cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
636  Imm = Log2_64(Imm);
637  ISDOpcode = ISD::SRA;
638  }
639 
640  // Transform "urem x, pow2" -> "and x, pow2-1".
641  if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
642  isPowerOf2_64(Imm)) {
643  --Imm;
644  ISDOpcode = ISD::AND;
645  }
646 
647  unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
648  Op0IsKill, Imm, VT.getSimpleVT());
649  if (!ResultReg)
650  return false;
651 
652  // We successfully emitted code for the given LLVM Instruction.
653  updateValueMap(I, ResultReg);
654  return true;
655  }
656 
657  unsigned Op1 = getRegForValue(I->getOperand(1));
658  if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
659  return false;
660  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
661 
662  // Now we have both operands in registers. Emit the instruction.
663  unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
664  ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
665  if (!ResultReg)
666  // Target-specific code wasn't able to find a machine opcode for
667  // the given ISD opcode and type. Halt "fast" selection and bail.
668  return false;
669 
670  // We successfully emitted code for the given LLVM Instruction.
671  updateValueMap(I, ResultReg);
672  return true;
673 }
674 
676  unsigned N = getRegForValue(I->getOperand(0));
677  if (!N) // Unhandled operand. Halt "fast" selection and bail.
678  return false;
679  bool NIsKill = hasTrivialKill(I->getOperand(0));
680 
681  // Keep a running tab of the total offset to coalesce multiple N = N + Offset
682  // into a single N = N + TotalOffset.
683  uint64_t TotalOffs = 0;
684  // FIXME: What's a good SWAG number for MaxOffs?
685  uint64_t MaxOffs = 2048;
686  MVT VT = TLI.getPointerTy(DL);
687  for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
688  GTI != E; ++GTI) {
689  const Value *Idx = GTI.getOperand();
690  if (StructType *StTy = GTI.getStructTypeOrNull()) {
691  uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
692  if (Field) {
693  // N = N + Offset
694  TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
695  if (TotalOffs >= MaxOffs) {
696  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
697  if (!N) // Unhandled operand. Halt "fast" selection and bail.
698  return false;
699  NIsKill = true;
700  TotalOffs = 0;
701  }
702  }
703  } else {
704  Type *Ty = GTI.getIndexedType();
705 
706  // If this is a constant subscript, handle it quickly.
707  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
708  if (CI->isZero())
709  continue;
710  // N = N + Offset
711  uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
712  TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
713  if (TotalOffs >= MaxOffs) {
714  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
715  if (!N) // Unhandled operand. Halt "fast" selection and bail.
716  return false;
717  NIsKill = true;
718  TotalOffs = 0;
719  }
720  continue;
721  }
722  if (TotalOffs) {
723  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
724  if (!N) // Unhandled operand. Halt "fast" selection and bail.
725  return false;
726  NIsKill = true;
727  TotalOffs = 0;
728  }
729 
730  // N = N + Idx * ElementSize;
731  uint64_t ElementSize = DL.getTypeAllocSize(Ty);
732  std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
733  unsigned IdxN = Pair.first;
734  bool IdxNIsKill = Pair.second;
735  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
736  return false;
737 
738  if (ElementSize != 1) {
739  IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
740  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
741  return false;
742  IdxNIsKill = true;
743  }
744  N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
745  if (!N) // Unhandled operand. Halt "fast" selection and bail.
746  return false;
747  }
748  }
749  if (TotalOffs) {
750  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
751  if (!N) // Unhandled operand. Halt "fast" selection and bail.
752  return false;
753  }
754 
755  // We successfully emitted code for the given LLVM Instruction.
756  updateValueMap(I, N);
757  return true;
758 }
759 
760 bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
761  const CallInst *CI, unsigned StartIdx) {
762  for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) {
763  Value *Val = CI->getArgOperand(i);
764  // Check for constants and encode them with a StackMaps::ConstantOp prefix.
765  if (const auto *C = dyn_cast<ConstantInt>(Val)) {
766  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
767  Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
768  } else if (isa<ConstantPointerNull>(Val)) {
769  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
771  } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
772  // Values coming from a stack location also require a special encoding,
773  // but that is added later on by the target specific frame index
774  // elimination implementation.
775  auto SI = FuncInfo.StaticAllocaMap.find(AI);
776  if (SI != FuncInfo.StaticAllocaMap.end())
777  Ops.push_back(MachineOperand::CreateFI(SI->second));
778  else
779  return false;
780  } else {
781  unsigned Reg = getRegForValue(Val);
782  if (!Reg)
783  return false;
784  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
785  }
786  }
787  return true;
788 }
789 
791  // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
792  // [live variables...])
794  "Stackmap cannot return a value.");
795 
796  // The stackmap intrinsic only records the live variables (the arguments
797  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
798  // intrinsic, this won't be lowered to a function call. This means we don't
799  // have to worry about calling conventions and target-specific lowering code.
800  // Instead we perform the call lowering right here.
801  //
802  // CALLSEQ_START(0, 0...)
803  // STACKMAP(id, nbytes, ...)
804  // CALLSEQ_END(0, 0)
805  //
807 
808  // Add the <id> and <numBytes> constants.
809  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
810  "Expected a constant integer.");
811  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
812  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
813 
814  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
815  "Expected a constant integer.");
816  const auto *NumBytes =
817  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
818  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
819 
820  // Push live variables for the stack map (skipping the first two arguments
821  // <id> and <numBytes>).
822  if (!addStackMapLiveVars(Ops, I, 2))
823  return false;
824 
825  // We are not adding any register mask info here, because the stackmap doesn't
826  // clobber anything.
827 
828  // Add scratch registers as implicit def and early clobber.
830  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
831  for (unsigned i = 0; ScratchRegs[i]; ++i)
832  Ops.push_back(MachineOperand::CreateReg(
833  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
834  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
835 
836  // Issue CALLSEQ_START
837  unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
838  auto Builder =
839  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
840  const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
841  for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
842  Builder.addImm(0);
843 
844  // Issue STACKMAP.
845  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
846  TII.get(TargetOpcode::STACKMAP));
847  for (auto const &MO : Ops)
848  MIB.add(MO);
849 
850  // Issue CALLSEQ_END
851  unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
852  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
853  .addImm(0)
854  .addImm(0);
855 
856  // Inform the Frame Information that we have a stackmap in this function.
857  FuncInfo.MF->getFrameInfo().setHasStackMap();
858 
859  return true;
860 }
861 
862 /// Lower an argument list according to the target calling convention.
863 ///
864 /// This is a helper for lowering intrinsics that follow a target calling
865 /// convention or require stack pointer adjustment. Only a subset of the
866 /// intrinsic's operands need to participate in the calling convention.
867 bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
868  unsigned NumArgs, const Value *Callee,
869  bool ForceRetVoidTy, CallLoweringInfo &CLI) {
870  ArgListTy Args;
871  Args.reserve(NumArgs);
872 
873  // Populate the argument list.
874  ImmutableCallSite CS(CI);
875  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
876  Value *V = CI->getOperand(ArgI);
877 
878  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
879 
881  Entry.Val = V;
882  Entry.Ty = V->getType();
883  Entry.setAttributes(&CS, ArgI);
884  Args.push_back(Entry);
885  }
886 
887  Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
888  : CI->getType();
889  CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
890 
891  return lowerCallTo(CLI);
892 }
893 
895  const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
896  StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
897  SmallString<32> MangledName;
898  Mangler::getNameWithPrefix(MangledName, Target, DL);
899  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
900  return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
901 }
902 
904  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
905  // i32 <numBytes>,
906  // i8* <target>,
907  // i32 <numArgs>,
908  // [Args...],
909  // [live variables...])
911  bool IsAnyRegCC = CC == CallingConv::AnyReg;
912  bool HasDef = !I->getType()->isVoidTy();
914 
915  // Get the real number of arguments participating in the call <numArgs>
916  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
917  "Expected a constant integer.");
918  const auto *NumArgsVal =
919  cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
920  unsigned NumArgs = NumArgsVal->getZExtValue();
921 
922  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
923  // This includes all meta-operands up to but not including CC.
924  unsigned NumMetaOpers = PatchPointOpers::CCPos;
925  assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
926  "Not enough arguments provided to the patchpoint intrinsic");
927 
928  // For AnyRegCC the arguments are lowered later on manually.
929  unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
930  CallLoweringInfo CLI;
931  CLI.setIsPatchPoint();
932  if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
933  return false;
934 
935  assert(CLI.Call && "No call instruction specified.");
936 
938 
939  // Add an explicit result reg if we use the anyreg calling convention.
940  if (IsAnyRegCC && HasDef) {
941  assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
942  CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64));
943  CLI.NumResultRegs = 1;
944  Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*IsDef=*/true));
945  }
946 
947  // Add the <id> and <numBytes> constants.
948  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
949  "Expected a constant integer.");
950  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
951  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
952 
953  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
954  "Expected a constant integer.");
955  const auto *NumBytes =
956  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
957  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
958 
959  // Add the call target.
960  if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
961  uint64_t CalleeConstAddr =
962  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
963  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
964  } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
965  if (C->getOpcode() == Instruction::IntToPtr) {
966  uint64_t CalleeConstAddr =
967  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
968  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
969  } else
970  llvm_unreachable("Unsupported ConstantExpr.");
971  } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
973  } else if (isa<ConstantPointerNull>(Callee))
975  else
976  llvm_unreachable("Unsupported callee address.");
977 
978  // Adjust <numArgs> to account for any arguments that have been passed on
979  // the stack instead.
980  unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
981  Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
982 
983  // Add the calling convention
984  Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
985 
986  // Add the arguments we omitted previously. The register allocator should
987  // place these in any free register.
988  if (IsAnyRegCC) {
989  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
990  unsigned Reg = getRegForValue(I->getArgOperand(i));
991  if (!Reg)
992  return false;
993  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
994  }
995  }
996 
997  // Push the arguments from the call instruction.
998  for (auto Reg : CLI.OutRegs)
999  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
1000 
1001  // Push live variables for the stack map.
1002  if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
1003  return false;
1004 
1005  // Push the register mask info.
1007  TRI.getCallPreservedMask(*FuncInfo.MF, CC)));
1008 
1009  // Add scratch registers as implicit def and early clobber.
1010  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
1011  for (unsigned i = 0; ScratchRegs[i]; ++i)
1013  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
1014  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
1015 
1016  // Add implicit defs (return values).
1017  for (auto Reg : CLI.InRegs)
1018  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/true,
1019  /*IsImpl=*/true));
1020 
1021  // Insert the patchpoint instruction before the call generated by the target.
1022  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc,
1023  TII.get(TargetOpcode::PATCHPOINT));
1024 
1025  for (auto &MO : Ops)
1026  MIB.add(MO);
1027 
1028  MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1029 
1030  // Delete the original call instruction.
1031  CLI.Call->eraseFromParent();
1032 
1033  // Inform the Frame Information that we have a patchpoint in this function.
1034  FuncInfo.MF->getFrameInfo().setHasPatchPoint();
1035 
1036  if (CLI.NumResultRegs)
1037  updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
1038  return true;
1039 }
1040 
1042  const auto &Triple = TM.getTargetTriple();
1043  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1044  return true; // don't do anything to this instruction.
1046  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1047  /*IsDef=*/false));
1048  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1049  /*IsDef=*/false));
1050  MachineInstrBuilder MIB =
1051  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1052  TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
1053  for (auto &MO : Ops)
1054  MIB.add(MO);
1055 
1056  // Insert the Patchable Event Call instruction, that gets lowered properly.
1057  return true;
1058 }
1059 
1061  const auto &Triple = TM.getTargetTriple();
1062  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1063  return true; // don't do anything to this instruction.
1065  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1066  /*IsDef=*/false));
1067  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1068  /*IsDef=*/false));
1069  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(2)),
1070  /*IsDef=*/false));
1071  MachineInstrBuilder MIB =
1072  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1073  TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
1074  for (auto &MO : Ops)
1075  MIB.add(MO);
1076 
1077  // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
1078  return true;
1079 }
1080 
1081 /// Returns an AttributeList representing the attributes applied to the return
1082 /// value of the given call.
1085  if (CLI.RetSExt)
1086  Attrs.push_back(Attribute::SExt);
1087  if (CLI.RetZExt)
1088  Attrs.push_back(Attribute::ZExt);
1089  if (CLI.IsInReg)
1090  Attrs.push_back(Attribute::InReg);
1091 
1093  Attrs);
1094 }
1095 
1096 bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
1097  unsigned NumArgs) {
1098  MCContext &Ctx = MF->getContext();
1099  SmallString<32> MangledName;
1100  Mangler::getNameWithPrefix(MangledName, SymName, DL);
1101  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
1102  return lowerCallTo(CI, Sym, NumArgs);
1103 }
1104 
1106  unsigned NumArgs) {
1107  ImmutableCallSite CS(CI);
1108 
1109  FunctionType *FTy = CS.getFunctionType();
1110  Type *RetTy = CS.getType();
1111 
1112  ArgListTy Args;
1113  Args.reserve(NumArgs);
1114 
1115  // Populate the argument list.
1116  // Attributes for args start at offset 1, after the return attribute.
1117  for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
1118  Value *V = CI->getOperand(ArgI);
1119 
1120  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
1121 
1123  Entry.Val = V;
1124  Entry.Ty = V->getType();
1125  Entry.setAttributes(&CS, ArgI);
1126  Args.push_back(Entry);
1127  }
1128  TLI.markLibCallAttributes(MF, CS.getCallingConv(), Args);
1129 
1130  CallLoweringInfo CLI;
1131  CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
1132 
1133  return lowerCallTo(CLI);
1134 }
1135 
1137  // Handle the incoming return values from the call.
1138  CLI.clearIns();
1139  SmallVector<EVT, 4> RetTys;
1140  ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
1141 
1143  GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
1144 
1145  bool CanLowerReturn = TLI.CanLowerReturn(
1146  CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1147 
1148  // FIXME: sret demotion isn't supported yet - bail out.
1149  if (!CanLowerReturn)
1150  return false;
1151 
1152  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
1153  EVT VT = RetTys[I];
1154  MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1155  unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1156  for (unsigned i = 0; i != NumRegs; ++i) {
1157  ISD::InputArg MyFlags;
1158  MyFlags.VT = RegisterVT;
1159  MyFlags.ArgVT = VT;
1160  MyFlags.Used = CLI.IsReturnValueUsed;
1161  if (CLI.RetSExt)
1162  MyFlags.Flags.setSExt();
1163  if (CLI.RetZExt)
1164  MyFlags.Flags.setZExt();
1165  if (CLI.IsInReg)
1166  MyFlags.Flags.setInReg();
1167  CLI.Ins.push_back(MyFlags);
1168  }
1169  }
1170 
1171  // Handle all of the outgoing arguments.
1172  CLI.clearOuts();
1173  for (auto &Arg : CLI.getArgs()) {
1174  Type *FinalType = Arg.Ty;
1175  if (Arg.IsByVal)
1176  FinalType = cast<PointerType>(Arg.Ty)->getElementType();
1177  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1178  FinalType, CLI.CallConv, CLI.IsVarArg);
1179 
1180  ISD::ArgFlagsTy Flags;
1181  if (Arg.IsZExt)
1182  Flags.setZExt();
1183  if (Arg.IsSExt)
1184  Flags.setSExt();
1185  if (Arg.IsInReg)
1186  Flags.setInReg();
1187  if (Arg.IsSRet)
1188  Flags.setSRet();
1189  if (Arg.IsSwiftSelf)
1190  Flags.setSwiftSelf();
1191  if (Arg.IsSwiftError)
1192  Flags.setSwiftError();
1193  if (Arg.IsByVal)
1194  Flags.setByVal();
1195  if (Arg.IsInAlloca) {
1196  Flags.setInAlloca();
1197  // Set the byval flag for CCAssignFn callbacks that don't know about
1198  // inalloca. This way we can know how many bytes we should've allocated
1199  // and how many bytes a callee cleanup function will pop. If we port
1200  // inalloca to more targets, we'll have to add custom inalloca handling in
1201  // the various CC lowering callbacks.
1202  Flags.setByVal();
1203  }
1204  if (Arg.IsByVal || Arg.IsInAlloca) {
1205  PointerType *Ty = cast<PointerType>(Arg.Ty);
1206  Type *ElementTy = Ty->getElementType();
1207  unsigned FrameSize = DL.getTypeAllocSize(ElementTy);
1208  // For ByVal, alignment should come from FE. BE will guess if this info is
1209  // not there, but there are cases it cannot get right.
1210  unsigned FrameAlign = Arg.Alignment;
1211  if (!FrameAlign)
1212  FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
1213  Flags.setByValSize(FrameSize);
1214  Flags.setByValAlign(FrameAlign);
1215  }
1216  if (Arg.IsNest)
1217  Flags.setNest();
1218  if (NeedsRegBlock)
1219  Flags.setInConsecutiveRegs();
1220  unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
1221  Flags.setOrigAlign(OriginalAlignment);
1222 
1223  CLI.OutVals.push_back(Arg.Val);
1224  CLI.OutFlags.push_back(Flags);
1225  }
1226 
1227  if (!fastLowerCall(CLI))
1228  return false;
1229 
1230  // Set all unused physreg defs as dead.
1231  assert(CLI.Call && "No call instruction specified.");
1232  CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1233 
1234  if (CLI.NumResultRegs && CLI.CS)
1235  updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs);
1236 
1237  // Set labels for heapallocsite call.
1238  if (CLI.CS && CLI.CS->getInstruction()->getMetadata("heapallocsite")) {
1239  MDNode *MD = CLI.CS->getInstruction()->getMetadata("heapallocsite");
1240  MF->addCodeViewHeapAllocSite(CLI.Call, MD);
1241  }
1242 
1243  return true;
1244 }
1245 
1247  ImmutableCallSite CS(CI);
1248 
1249  FunctionType *FuncTy = CS.getFunctionType();
1250  Type *RetTy = CS.getType();
1251 
1252  ArgListTy Args;
1254  Args.reserve(CS.arg_size());
1255 
1256  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
1257  i != e; ++i) {
1258  Value *V = *i;
1259 
1260  // Skip empty types
1261  if (V->getType()->isEmptyTy())
1262  continue;
1263 
1264  Entry.Val = V;
1265  Entry.Ty = V->getType();
1266 
1267  // Skip the first return-type Attribute to get to params.
1268  Entry.setAttributes(&CS, i - CS.arg_begin());
1269  Args.push_back(Entry);
1270  }
1271 
1272  // Check if target-independent constraints permit a tail call here.
1273  // Target-dependent constraints are checked within fastLowerCall.
1274  bool IsTailCall = CI->isTailCall();
1275  if (IsTailCall && !isInTailCallPosition(CS, TM))
1276  IsTailCall = false;
1277 
1278  CallLoweringInfo CLI;
1279  CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
1280  .setTailCall(IsTailCall);
1281 
1282  return lowerCallTo(CLI);
1283 }
1284 
1286  const CallInst *Call = cast<CallInst>(I);
1287 
1288  // Handle simple inline asms.
1289  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
1290  // If the inline asm has side effects, then make sure that no local value
1291  // lives across by flushing the local value map.
1292  if (IA->hasSideEffects())
1293  flushLocalValueMap();
1294 
1295  // Don't attempt to handle constraints.
1296  if (!IA->getConstraintString().empty())
1297  return false;
1298 
1299  unsigned ExtraInfo = 0;
1300  if (IA->hasSideEffects())
1301  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1302  if (IA->isAlignStack())
1303  ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1304 
1305  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1307  .addExternalSymbol(IA->getAsmString().c_str())
1308  .addImm(ExtraInfo);
1309  return true;
1310  }
1311 
1312  // Handle intrinsic function calls.
1313  if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1314  return selectIntrinsicCall(II);
1315 
1316  // Usually, it does not make sense to initialize a value,
1317  // make an unrelated function call and use the value, because
1318  // it tends to be spilled on the stack. So, we move the pointer
1319  // to the last local value to the beginning of the block, so that
1320  // all the values which have already been materialized,
1321  // appear after the call. It also makes sense to skip intrinsics
1322  // since they tend to be inlined.
1323  flushLocalValueMap();
1324 
1325  return lowerCall(Call);
1326 }
1327 
1329  switch (II->getIntrinsicID()) {
1330  default:
1331  break;
1332  // At -O0 we don't care about the lifetime intrinsics.
1333  case Intrinsic::lifetime_start:
1334  case Intrinsic::lifetime_end:
1335  // The donothing intrinsic does, well, nothing.
1336  case Intrinsic::donothing:
1337  // Neither does the sideeffect intrinsic.
1338  case Intrinsic::sideeffect:
1339  // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1340  case Intrinsic::assume:
1341  return true;
1342  case Intrinsic::dbg_declare: {
1343  const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1344  assert(DI->getVariable() && "Missing variable");
1345  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1346  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1347  return true;
1348  }
1349 
1350  const Value *Address = DI->getAddress();
1351  if (!Address || isa<UndefValue>(Address)) {
1352  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1353  return true;
1354  }
1355 
1356  // Byval arguments with frame indices were already handled after argument
1357  // lowering and before isel.
1358  const auto *Arg =
1360  if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
1361  return true;
1362 
1364  if (unsigned Reg = lookUpRegForValue(Address))
1365  Op = MachineOperand::CreateReg(Reg, false);
1366 
1367  // If we have a VLA that has a "use" in a metadata node that's then used
1368  // here but it has no other uses, then we have a problem. E.g.,
1369  //
1370  // int foo (const int *x) {
1371  // char a[*x];
1372  // return 0;
1373  // }
1374  //
1375  // If we assign 'a' a vreg and fast isel later on has to use the selection
1376  // DAG isel, it will want to copy the value to the vreg. However, there are
1377  // no uses, which goes counter to what selection DAG isel expects.
1378  if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
1379  (!isa<AllocaInst>(Address) ||
1380  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
1381  Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address),
1382  false);
1383 
1384  if (Op) {
1386  "Expected inlined-at fields to agree");
1387  // A dbg.declare describes the address of a source variable, so lower it
1388  // into an indirect DBG_VALUE.
1389  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1390  TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true,
1391  *Op, DI->getVariable(), DI->getExpression());
1392  } else {
1393  // We can't yet handle anything else here because it would require
1394  // generating code, thus altering codegen because of debug info.
1395  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1396  }
1397  return true;
1398  }
1399  case Intrinsic::dbg_value: {
1400  // This form of DBG_VALUE is target-independent.
1401  const DbgValueInst *DI = cast<DbgValueInst>(II);
1402  const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1403  const Value *V = DI->getValue();
1405  "Expected inlined-at fields to agree");
1406  if (!V) {
1407  // Currently the optimizer can produce this; insert an undef to
1408  // help debugging. Probably the optimizer should not do this.
1409  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
1410  DI->getVariable(), DI->getExpression());
1411  } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1412  if (CI->getBitWidth() > 64)
1413  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1414  .addCImm(CI)
1415  .addImm(0U)
1416  .addMetadata(DI->getVariable())
1417  .addMetadata(DI->getExpression());
1418  else
1419  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1420  .addImm(CI->getZExtValue())
1421  .addImm(0U)
1422  .addMetadata(DI->getVariable())
1423  .addMetadata(DI->getExpression());
1424  } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1425  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1426  .addFPImm(CF)
1427  .addImm(0U)
1428  .addMetadata(DI->getVariable())
1429  .addMetadata(DI->getExpression());
1430  } else if (unsigned Reg = lookUpRegForValue(V)) {
1431  // FIXME: This does not handle register-indirect values at offset 0.
1432  bool IsIndirect = false;
1433  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
1434  DI->getVariable(), DI->getExpression());
1435  } else {
1436  // We can't yet handle anything else here because it would require
1437  // generating code, thus altering codegen because of debug info.
1438  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1439  }
1440  return true;
1441  }
1442  case Intrinsic::dbg_label: {
1443  const DbgLabelInst *DI = cast<DbgLabelInst>(II);
1444  assert(DI->getLabel() && "Missing label");
1445  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1446  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1447  return true;
1448  }
1449 
1450  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1451  TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
1452  return true;
1453  }
1454  case Intrinsic::objectsize: {
1455  ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
1456  unsigned long long Res = CI->isZero() ? -1ULL : 0;
1457  Constant *ResCI = ConstantInt::get(II->getType(), Res);
1458  unsigned ResultReg = getRegForValue(ResCI);
1459  if (!ResultReg)
1460  return false;
1461  updateValueMap(II, ResultReg);
1462  return true;
1463  }
1464  case Intrinsic::is_constant: {
1465  Constant *ResCI = ConstantInt::get(II->getType(), 0);
1466  unsigned ResultReg = getRegForValue(ResCI);
1467  if (!ResultReg)
1468  return false;
1469  updateValueMap(II, ResultReg);
1470  return true;
1471  }
1472  case Intrinsic::launder_invariant_group:
1473  case Intrinsic::strip_invariant_group:
1474  case Intrinsic::expect: {
1475  unsigned ResultReg = getRegForValue(II->getArgOperand(0));
1476  if (!ResultReg)
1477  return false;
1478  updateValueMap(II, ResultReg);
1479  return true;
1480  }
1481  case Intrinsic::experimental_stackmap:
1482  return selectStackmap(II);
1483  case Intrinsic::experimental_patchpoint_void:
1484  case Intrinsic::experimental_patchpoint_i64:
1485  return selectPatchpoint(II);
1486 
1487  case Intrinsic::xray_customevent:
1488  return selectXRayCustomEvent(II);
1489  case Intrinsic::xray_typedevent:
1490  return selectXRayTypedEvent(II);
1491  }
1492 
1493  return fastLowerIntrinsicCall(II);
1494 }
1495 
1496 bool FastISel::selectCast(const User *I, unsigned Opcode) {
1497  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1498  EVT DstVT = TLI.getValueType(DL, I->getType());
1499 
1500  if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1501  !DstVT.isSimple())
1502  // Unhandled type. Halt "fast" selection and bail.
1503  return false;
1504 
1505  // Check if the destination type is legal.
1506  if (!TLI.isTypeLegal(DstVT))
1507  return false;
1508 
1509  // Check if the source operand is legal.
1510  if (!TLI.isTypeLegal(SrcVT))
1511  return false;
1512 
1513  unsigned InputReg = getRegForValue(I->getOperand(0));
1514  if (!InputReg)
1515  // Unhandled operand. Halt "fast" selection and bail.
1516  return false;
1517 
1518  bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
1519 
1520  unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1521  Opcode, InputReg, InputRegIsKill);
1522  if (!ResultReg)
1523  return false;
1524 
1525  updateValueMap(I, ResultReg);
1526  return true;
1527 }
1528 
1530  // If the bitcast doesn't change the type, just use the operand value.
1531  if (I->getType() == I->getOperand(0)->getType()) {
1532  unsigned Reg = getRegForValue(I->getOperand(0));
1533  if (!Reg)
1534  return false;
1535  updateValueMap(I, Reg);
1536  return true;
1537  }
1538 
1539  // Bitcasts of other values become reg-reg copies or BITCAST operators.
1540  EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1541  EVT DstEVT = TLI.getValueType(DL, I->getType());
1542  if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1543  !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
1544  // Unhandled type. Halt "fast" selection and bail.
1545  return false;
1546 
1547  MVT SrcVT = SrcEVT.getSimpleVT();
1548  MVT DstVT = DstEVT.getSimpleVT();
1549  unsigned Op0 = getRegForValue(I->getOperand(0));
1550  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1551  return false;
1552  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
1553 
1554  // First, try to perform the bitcast by inserting a reg-reg copy.
1555  unsigned ResultReg = 0;
1556  if (SrcVT == DstVT) {
1557  const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
1558  const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
1559  // Don't attempt a cross-class copy. It will likely fail.
1560  if (SrcClass == DstClass) {
1561  ResultReg = createResultReg(DstClass);
1562  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1563  TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
1564  }
1565  }
1566 
1567  // If the reg-reg copy failed, select a BITCAST opcode.
1568  if (!ResultReg)
1569  ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
1570 
1571  if (!ResultReg)
1572  return false;
1573 
1574  updateValueMap(I, ResultReg);
1575  return true;
1576 }
1577 
1578 // Remove local value instructions starting from the instruction after
1579 // SavedLastLocalValue to the current function insert point.
1580 void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1581 {
1582  MachineInstr *CurLastLocalValue = getLastLocalValue();
1583  if (CurLastLocalValue != SavedLastLocalValue) {
1584  // Find the first local value instruction to be deleted.
1585  // This is the instruction after SavedLastLocalValue if it is non-NULL.
1586  // Otherwise it's the first instruction in the block.
1587  MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1588  if (SavedLastLocalValue)
1589  ++FirstDeadInst;
1590  else
1591  FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1592  setLastLocalValue(SavedLastLocalValue);
1593  removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1594  }
1595 }
1596 
1598  MachineInstr *SavedLastLocalValue = getLastLocalValue();
1599  // Just before the terminator instruction, insert instructions to
1600  // feed PHI nodes in successor blocks.
1601  if (I->isTerminator()) {
1602  if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1603  // PHI node handling may have generated local value instructions,
1604  // even though it failed to handle all PHI nodes.
1605  // We remove these instructions because SelectionDAGISel will generate
1606  // them again.
1607  removeDeadLocalValueCode(SavedLastLocalValue);
1608  return false;
1609  }
1610  }
1611 
1612  // FastISel does not handle any operand bundles except OB_funclet.
1614  for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
1615  if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1616  return false;
1617 
1618  DbgLoc = I->getDebugLoc();
1619 
1620  SavedInsertPt = FuncInfo.InsertPt;
1621 
1622  if (const auto *Call = dyn_cast<CallInst>(I)) {
1623  const Function *F = Call->getCalledFunction();
1624  LibFunc Func;
1625 
1626  // As a special case, don't handle calls to builtin library functions that
1627  // may be translated directly to target instructions.
1628  if (F && !F->hasLocalLinkage() && F->hasName() &&
1629  LibInfo->getLibFunc(F->getName(), Func) &&
1630  LibInfo->hasOptimizedCodeGen(Func))
1631  return false;
1632 
1633  // Don't handle Intrinsic::trap if a trap function is specified.
1634  if (F && F->getIntrinsicID() == Intrinsic::trap &&
1635  Call->hasFnAttr("trap-func-name"))
1636  return false;
1637  }
1638 
1639  // First, try doing target-independent selection.
1640  if (!SkipTargetIndependentISel) {
1641  if (selectOperator(I, I->getOpcode())) {
1642  ++NumFastIselSuccessIndependent;
1643  DbgLoc = DebugLoc();
1644  return true;
1645  }
1646  // Remove dead code.
1647  recomputeInsertPt();
1648  if (SavedInsertPt != FuncInfo.InsertPt)
1649  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1650  SavedInsertPt = FuncInfo.InsertPt;
1651  }
1652  // Next, try calling the target to attempt to handle the instruction.
1653  if (fastSelectInstruction(I)) {
1654  ++NumFastIselSuccessTarget;
1655  DbgLoc = DebugLoc();
1656  return true;
1657  }
1658  // Remove dead code.
1659  recomputeInsertPt();
1660  if (SavedInsertPt != FuncInfo.InsertPt)
1661  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1662 
1663  DbgLoc = DebugLoc();
1664  // Undo phi node updates, because they will be added again by SelectionDAG.
1665  if (I->isTerminator()) {
1666  // PHI node handling may have generated local value instructions.
1667  // We remove them because SelectionDAGISel will generate them again.
1668  removeDeadLocalValueCode(SavedLastLocalValue);
1669  FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
1670  }
1671  return false;
1672 }
1673 
1674 /// Emit an unconditional branch to the given block, unless it is the immediate
1675 /// (fall-through) successor, and update the CFG.
1677  const DebugLoc &DbgLoc) {
1678  if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
1679  FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
1680  // For more accurate line information if this is the only instruction
1681  // in the block then emit it, otherwise we have the unconditional
1682  // fall-through case, which needs no instructions.
1683  } else {
1684  // The unconditional branch case.
1685  TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1686  SmallVector<MachineOperand, 0>(), DbgLoc);
1687  }
1688  if (FuncInfo.BPI) {
1689  auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1690  FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
1691  FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
1692  } else
1693  FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
1694 }
1695 
1697  MachineBasicBlock *TrueMBB,
1698  MachineBasicBlock *FalseMBB) {
1699  // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1700  // happen in degenerate IR and MachineIR forbids to have a block twice in the
1701  // successor/predecessor lists.
1702  if (TrueMBB != FalseMBB) {
1703  if (FuncInfo.BPI) {
1704  auto BranchProbability =
1705  FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1706  FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
1707  } else
1708  FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
1709  }
1710 
1711  fastEmitBranch(FalseMBB, DbgLoc);
1712 }
1713 
1714 /// Emit an FNeg operation.
1716  Value *X;
1717  if (!match(I, m_FNeg(m_Value(X))))
1718  return false;
1719  unsigned OpReg = getRegForValue(X);
1720  if (!OpReg)
1721  return false;
1722  bool OpRegIsKill = hasTrivialKill(I);
1723 
1724  // If the target has ISD::FNEG, use it.
1725  EVT VT = TLI.getValueType(DL, I->getType());
1726  unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1727  OpReg, OpRegIsKill);
1728  if (ResultReg) {
1729  updateValueMap(I, ResultReg);
1730  return true;
1731  }
1732 
1733  // Bitcast the value to integer, twiddle the sign bit with xor,
1734  // and then bitcast it back to floating-point.
1735  if (VT.getSizeInBits() > 64)
1736  return false;
1737  EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1738  if (!TLI.isTypeLegal(IntVT))
1739  return false;
1740 
1741  unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1742  ISD::BITCAST, OpReg, OpRegIsKill);
1743  if (!IntReg)
1744  return false;
1745 
1746  unsigned IntResultReg = fastEmit_ri_(
1747  IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
1748  UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1749  if (!IntResultReg)
1750  return false;
1751 
1752  ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1753  IntResultReg, /*IsKill=*/true);
1754  if (!ResultReg)
1755  return false;
1756 
1757  updateValueMap(I, ResultReg);
1758  return true;
1759 }
1760 
1762  const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1763  if (!EVI)
1764  return false;
1765 
1766  // Make sure we only try to handle extracts with a legal result. But also
1767  // allow i1 because it's easy.
1768  EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1769  if (!RealVT.isSimple())
1770  return false;
1771  MVT VT = RealVT.getSimpleVT();
1772  if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1773  return false;
1774 
1775  const Value *Op0 = EVI->getOperand(0);
1776  Type *AggTy = Op0->getType();
1777 
1778  // Get the base result register.
1779  unsigned ResultReg;
1780  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(Op0);
1781  if (I != FuncInfo.ValueMap.end())
1782  ResultReg = I->second;
1783  else if (isa<Instruction>(Op0))
1784  ResultReg = FuncInfo.InitializeRegForValue(Op0);
1785  else
1786  return false; // fast-isel can't handle aggregate constants at the moment
1787 
1788  // Get the actual result register, which is an offset from the base register.
1789  unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1790 
1791  SmallVector<EVT, 4> AggValueVTs;
1792  ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1793 
1794  for (unsigned i = 0; i < VTIndex; i++)
1795  ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1796 
1797  updateValueMap(EVI, ResultReg);
1798  return true;
1799 }
1800 
1801 bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1802  switch (Opcode) {
1803  case Instruction::Add:
1804  return selectBinaryOp(I, ISD::ADD);
1805  case Instruction::FAdd:
1806  return selectBinaryOp(I, ISD::FADD);
1807  case Instruction::Sub:
1808  return selectBinaryOp(I, ISD::SUB);
1809  case Instruction::FSub:
1810  // FNeg is currently represented in LLVM IR as a special case of FSub.
1811  return selectFNeg(I) || selectBinaryOp(I, ISD::FSUB);
1812  case Instruction::Mul:
1813  return selectBinaryOp(I, ISD::MUL);
1814  case Instruction::FMul:
1815  return selectBinaryOp(I, ISD::FMUL);
1816  case Instruction::SDiv:
1817  return selectBinaryOp(I, ISD::SDIV);
1818  case Instruction::UDiv:
1819  return selectBinaryOp(I, ISD::UDIV);
1820  case Instruction::FDiv:
1821  return selectBinaryOp(I, ISD::FDIV);
1822  case Instruction::SRem:
1823  return selectBinaryOp(I, ISD::SREM);
1824  case Instruction::URem:
1825  return selectBinaryOp(I, ISD::UREM);
1826  case Instruction::FRem:
1827  return selectBinaryOp(I, ISD::FREM);
1828  case Instruction::Shl:
1829  return selectBinaryOp(I, ISD::SHL);
1830  case Instruction::LShr:
1831  return selectBinaryOp(I, ISD::SRL);
1832  case Instruction::AShr:
1833  return selectBinaryOp(I, ISD::SRA);
1834  case Instruction::And:
1835  return selectBinaryOp(I, ISD::AND);
1836  case Instruction::Or:
1837  return selectBinaryOp(I, ISD::OR);
1838  case Instruction::Xor:
1839  return selectBinaryOp(I, ISD::XOR);
1840 
1841  case Instruction::GetElementPtr:
1842  return selectGetElementPtr(I);
1843 
1844  case Instruction::Br: {
1845  const BranchInst *BI = cast<BranchInst>(I);
1846 
1847  if (BI->isUnconditional()) {
1848  const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1849  MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1850  fastEmitBranch(MSucc, BI->getDebugLoc());
1851  return true;
1852  }
1853 
1854  // Conditional branches are not handed yet.
1855  // Halt "fast" selection and bail.
1856  return false;
1857  }
1858 
1859  case Instruction::Unreachable:
1860  if (TM.Options.TrapUnreachable)
1861  return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
1862  else
1863  return true;
1864 
1865  case Instruction::Alloca:
1866  // FunctionLowering has the static-sized case covered.
1867  if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1868  return true;
1869 
1870  // Dynamic-sized alloca is not handled yet.
1871  return false;
1872 
1873  case Instruction::Call:
1874  return selectCall(I);
1875 
1876  case Instruction::BitCast:
1877  return selectBitCast(I);
1878 
1879  case Instruction::FPToSI:
1880  return selectCast(I, ISD::FP_TO_SINT);
1881  case Instruction::ZExt:
1882  return selectCast(I, ISD::ZERO_EXTEND);
1883  case Instruction::SExt:
1884  return selectCast(I, ISD::SIGN_EXTEND);
1885  case Instruction::Trunc:
1886  return selectCast(I, ISD::TRUNCATE);
1887  case Instruction::SIToFP:
1888  return selectCast(I, ISD::SINT_TO_FP);
1889 
1890  case Instruction::IntToPtr: // Deliberate fall-through.
1891  case Instruction::PtrToInt: {
1892  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1893  EVT DstVT = TLI.getValueType(DL, I->getType());
1894  if (DstVT.bitsGT(SrcVT))
1895  return selectCast(I, ISD::ZERO_EXTEND);
1896  if (DstVT.bitsLT(SrcVT))
1897  return selectCast(I, ISD::TRUNCATE);
1898  unsigned Reg = getRegForValue(I->getOperand(0));
1899  if (!Reg)
1900  return false;
1901  updateValueMap(I, Reg);
1902  return true;
1903  }
1904 
1905  case Instruction::ExtractValue:
1906  return selectExtractValue(I);
1907 
1908  case Instruction::PHI:
1909  llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1910 
1911  default:
1912  // Unhandled instruction. Halt "fast" selection and bail.
1913  return false;
1914  }
1915 }
1916 
1918  const TargetLibraryInfo *LibInfo,
1919  bool SkipTargetIndependentISel)
1920  : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1921  MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1922  TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1923  TII(*MF->getSubtarget().getInstrInfo()),
1924  TLI(*MF->getSubtarget().getTargetLowering()),
1925  TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1926  SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1927 
1928 FastISel::~FastISel() = default;
1929 
1930 bool FastISel::fastLowerArguments() { return false; }
1931 
1932 bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1933 
1935  return false;
1936 }
1937 
1938 unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1939 
1940 unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
1941  bool /*Op0IsKill*/) {
1942  return 0;
1943 }
1944 
1945 unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1946  bool /*Op0IsKill*/, unsigned /*Op1*/,
1947  bool /*Op1IsKill*/) {
1948  return 0;
1949 }
1950 
1951 unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1952  return 0;
1953 }
1954 
1955 unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1956  const ConstantFP * /*FPImm*/) {
1957  return 0;
1958 }
1959 
1960 unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1961  bool /*Op0IsKill*/, uint64_t /*Imm*/) {
1962  return 0;
1963 }
1964 
1965 /// This method is a wrapper of fastEmit_ri. It first tries to emit an
1966 /// instruction with an immediate operand using fastEmit_ri.
1967 /// If that fails, it materializes the immediate into a register and try
1968 /// fastEmit_rr instead.
1969 unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1970  bool Op0IsKill, uint64_t Imm, MVT ImmType) {
1971  // If this is a multiply by a power of two, emit this as a shift left.
1972  if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
1973  Opcode = ISD::SHL;
1974  Imm = Log2_64(Imm);
1975  } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
1976  // div x, 8 -> srl x, 3
1977  Opcode = ISD::SRL;
1978  Imm = Log2_64(Imm);
1979  }
1980 
1981  // Horrible hack (to be removed), check to make sure shift amounts are
1982  // in-range.
1983  if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1984  Imm >= VT.getSizeInBits())
1985  return 0;
1986 
1987  // First check if immediate type is legal. If not, we can't use the ri form.
1988  unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
1989  if (ResultReg)
1990  return ResultReg;
1991  unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1992  bool IsImmKill = true;
1993  if (!MaterialReg) {
1994  // This is a bit ugly/slow, but failing here means falling out of
1995  // fast-isel, which would be very slow.
1996  IntegerType *ITy =
1998  MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
1999  if (!MaterialReg)
2000  return 0;
2001  // FIXME: If the materialized register here has no uses yet then this
2002  // will be the first use and we should be able to mark it as killed.
2003  // However, the local value area for materialising constant expressions
2004  // grows down, not up, which means that any constant expressions we generate
2005  // later which also use 'Imm' could be after this instruction and therefore
2006  // after this kill.
2007  IsImmKill = false;
2008  }
2009  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
2010 }
2011 
2013  return MRI.createVirtualRegister(RC);
2014 }
2015 
2017  unsigned OpNum) {
2019  const TargetRegisterClass *RegClass =
2020  TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
2021  if (!MRI.constrainRegClass(Op, RegClass)) {
2022  // If it's not legal to COPY between the register classes, something
2023  // has gone very wrong before we got here.
2024  unsigned NewOp = createResultReg(RegClass);
2026  TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
2027  return NewOp;
2028  }
2029  }
2030  return Op;
2031 }
2032 
2033 unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2034  const TargetRegisterClass *RC) {
2035  unsigned ResultReg = createResultReg(RC);
2036  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2037 
2038  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
2039  return ResultReg;
2040 }
2041 
2042 unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2043  const TargetRegisterClass *RC, unsigned Op0,
2044  bool Op0IsKill) {
2045  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2046 
2047  unsigned ResultReg = createResultReg(RC);
2048  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2049 
2050  if (II.getNumDefs() >= 1)
2051  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2052  .addReg(Op0, getKillRegState(Op0IsKill));
2053  else {
2055  .addReg(Op0, getKillRegState(Op0IsKill));
2057  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2058  }
2059 
2060  return ResultReg;
2061 }
2062 
2063 unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2064  const TargetRegisterClass *RC, unsigned Op0,
2065  bool Op0IsKill, unsigned Op1,
2066  bool Op1IsKill) {
2067  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2068 
2069  unsigned ResultReg = createResultReg(RC);
2070  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2071  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2072 
2073  if (II.getNumDefs() >= 1)
2074  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2075  .addReg(Op0, getKillRegState(Op0IsKill))
2076  .addReg(Op1, getKillRegState(Op1IsKill));
2077  else {
2079  .addReg(Op0, getKillRegState(Op0IsKill))
2080  .addReg(Op1, getKillRegState(Op1IsKill));
2082  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2083  }
2084  return ResultReg;
2085 }
2086 
2087 unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2088  const TargetRegisterClass *RC, unsigned Op0,
2089  bool Op0IsKill, unsigned Op1,
2090  bool Op1IsKill, unsigned Op2,
2091  bool Op2IsKill) {
2092  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2093 
2094  unsigned ResultReg = createResultReg(RC);
2095  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2096  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2097  Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
2098 
2099  if (II.getNumDefs() >= 1)
2100  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2101  .addReg(Op0, getKillRegState(Op0IsKill))
2102  .addReg(Op1, getKillRegState(Op1IsKill))
2103  .addReg(Op2, getKillRegState(Op2IsKill));
2104  else {
2106  .addReg(Op0, getKillRegState(Op0IsKill))
2107  .addReg(Op1, getKillRegState(Op1IsKill))
2108  .addReg(Op2, getKillRegState(Op2IsKill));
2110  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2111  }
2112  return ResultReg;
2113 }
2114 
2115 unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2116  const TargetRegisterClass *RC, unsigned Op0,
2117  bool Op0IsKill, uint64_t Imm) {
2118  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2119 
2120  unsigned ResultReg = createResultReg(RC);
2121  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2122 
2123  if (II.getNumDefs() >= 1)
2124  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2125  .addReg(Op0, getKillRegState(Op0IsKill))
2126  .addImm(Imm);
2127  else {
2129  .addReg(Op0, getKillRegState(Op0IsKill))
2130  .addImm(Imm);
2132  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2133  }
2134  return ResultReg;
2135 }
2136 
2137 unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2138  const TargetRegisterClass *RC, unsigned Op0,
2139  bool Op0IsKill, uint64_t Imm1,
2140  uint64_t Imm2) {
2141  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2142 
2143  unsigned ResultReg = createResultReg(RC);
2144  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2145 
2146  if (II.getNumDefs() >= 1)
2147  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2148  .addReg(Op0, getKillRegState(Op0IsKill))
2149  .addImm(Imm1)
2150  .addImm(Imm2);
2151  else {
2153  .addReg(Op0, getKillRegState(Op0IsKill))
2154  .addImm(Imm1)
2155  .addImm(Imm2);
2157  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2158  }
2159  return ResultReg;
2160 }
2161 
2162 unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2163  const TargetRegisterClass *RC,
2164  const ConstantFP *FPImm) {
2165  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2166 
2167  unsigned ResultReg = createResultReg(RC);
2168 
2169  if (II.getNumDefs() >= 1)
2170  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2171  .addFPImm(FPImm);
2172  else {
2174  .addFPImm(FPImm);
2176  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2177  }
2178  return ResultReg;
2179 }
2180 
2181 unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2182  const TargetRegisterClass *RC, unsigned Op0,
2183  bool Op0IsKill, unsigned Op1,
2184  bool Op1IsKill, uint64_t Imm) {
2185  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2186 
2187  unsigned ResultReg = createResultReg(RC);
2188  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2189  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2190 
2191  if (II.getNumDefs() >= 1)
2192  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2193  .addReg(Op0, getKillRegState(Op0IsKill))
2194  .addReg(Op1, getKillRegState(Op1IsKill))
2195  .addImm(Imm);
2196  else {
2198  .addReg(Op0, getKillRegState(Op0IsKill))
2199  .addReg(Op1, getKillRegState(Op1IsKill))
2200  .addImm(Imm);
2202  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2203  }
2204  return ResultReg;
2205 }
2206 
2207 unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2208  const TargetRegisterClass *RC, uint64_t Imm) {
2209  unsigned ResultReg = createResultReg(RC);
2210  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2211 
2212  if (II.getNumDefs() >= 1)
2213  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2214  .addImm(Imm);
2215  else {
2218  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2219  }
2220  return ResultReg;
2221 }
2222 
2223 unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2224  bool Op0IsKill, uint32_t Idx) {
2225  unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2227  "Cannot yet extract from physregs");
2228  const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2230  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
2231  ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
2232  return ResultReg;
2233 }
2234 
2235 /// Emit MachineInstrs to compute the value of Op with all but the least
2236 /// significant bit set to zero.
2237 unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
2238  return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
2239 }
2240 
2241 /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2242 /// Emit code to ensure constants are copied into registers when needed.
2243 /// Remember the virtual registers that need to be added to the Machine PHI
2244 /// nodes as input. We cannot just directly add them, because expansion
2245 /// might result in multiple MBB's for one BB. As such, the start of the
2246 /// BB might correspond to a different MBB than the end.
2247 bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2248  const Instruction *TI = LLVMBB->getTerminator();
2249 
2252 
2253  // Check successor nodes' PHI nodes that expect a constant to be available
2254  // from this block.
2255  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
2256  const BasicBlock *SuccBB = TI->getSuccessor(succ);
2257  if (!isa<PHINode>(SuccBB->begin()))
2258  continue;
2259  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2260 
2261  // If this terminator has multiple identical successors (common for
2262  // switches), only handle each succ once.
2263  if (!SuccsHandled.insert(SuccMBB).second)
2264  continue;
2265 
2266  MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2267 
2268  // At this point we know that there is a 1-1 correspondence between LLVM PHI
2269  // nodes and Machine PHI nodes, but the incoming operands have not been
2270  // emitted yet.
2271  for (const PHINode &PN : SuccBB->phis()) {
2272  // Ignore dead phi's.
2273  if (PN.use_empty())
2274  continue;
2275 
2276  // Only handle legal types. Two interesting things to note here. First,
2277  // by bailing out early, we may leave behind some dead instructions,
2278  // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2279  // own moves. Second, this check is necessary because FastISel doesn't
2280  // use CreateRegs to create registers, so it always creates
2281  // exactly one register for each non-void instruction.
2282  EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2283  if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2284  // Handle integer promotions, though, because they're common and easy.
2285  if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2287  return false;
2288  }
2289  }
2290 
2291  const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2292 
2293  // Set the DebugLoc for the copy. Prefer the location of the operand
2294  // if there is one; use the location of the PHI otherwise.
2295  DbgLoc = PN.getDebugLoc();
2296  if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2297  DbgLoc = Inst->getDebugLoc();
2298 
2299  unsigned Reg = getRegForValue(PHIOp);
2300  if (!Reg) {
2302  return false;
2303  }
2304  FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2305  DbgLoc = DebugLoc();
2306  }
2307  }
2308 
2309  return true;
2310 }
2311 
2312 bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2313  assert(LI->hasOneUse() &&
2314  "tryToFoldLoad expected a LoadInst with a single use");
2315  // We know that the load has a single use, but don't know what it is. If it
2316  // isn't one of the folded instructions, then we can't succeed here. Handle
2317  // this by scanning the single-use users of the load until we get to FoldInst.
2318  unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2319 
2320  const Instruction *TheUser = LI->user_back();
2321  while (TheUser != FoldInst && // Scan up until we find FoldInst.
2322  // Stay in the right block.
2323  TheUser->getParent() == FoldInst->getParent() &&
2324  --MaxUsers) { // Don't scan too far.
2325  // If there are multiple or no uses of this instruction, then bail out.
2326  if (!TheUser->hasOneUse())
2327  return false;
2328 
2329  TheUser = TheUser->user_back();
2330  }
2331 
2332  // If we didn't find the fold instruction, then we failed to collapse the
2333  // sequence.
2334  if (TheUser != FoldInst)
2335  return false;
2336 
2337  // Don't try to fold volatile loads. Target has to deal with alignment
2338  // constraints.
2339  if (LI->isVolatile())
2340  return false;
2341 
2342  // Figure out which vreg this is going into. If there is no assigned vreg yet
2343  // then there actually was no reference to it. Perhaps the load is referenced
2344  // by a dead instruction.
2345  unsigned LoadReg = getRegForValue(LI);
2346  if (!LoadReg)
2347  return false;
2348 
2349  // We can't fold if this vreg has no uses or more than one use. Multiple uses
2350  // may mean that the instruction got lowered to multiple MIs, or the use of
2351  // the loaded value ended up being multiple operands of the result.
2352  if (!MRI.hasOneUse(LoadReg))
2353  return false;
2354 
2356  MachineInstr *User = RI->getParent();
2357 
2358  // Set the insertion point properly. Folding the load can cause generation of
2359  // other random instructions (like sign extends) for addressing modes; make
2360  // sure they get inserted in a logical place before the new instruction.
2361  FuncInfo.InsertPt = User;
2362  FuncInfo.MBB = User->getParent();
2363 
2364  // Ask the target to try folding the load.
2365  return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2366 }
2367 
2369  // Must be an add.
2370  if (!isa<AddOperator>(Add))
2371  return false;
2372  // Type size needs to match.
2373  if (DL.getTypeSizeInBits(GEP->getType()) !=
2374  DL.getTypeSizeInBits(Add->getType()))
2375  return false;
2376  // Must be in the same basic block.
2377  if (isa<Instruction>(Add) &&
2378  FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2379  return false;
2380  // Must have a constant operand.
2381  return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2382 }
2383 
2386  const Value *Ptr;
2387  Type *ValTy;
2388  unsigned Alignment;
2390  bool IsVolatile;
2391 
2392  if (const auto *LI = dyn_cast<LoadInst>(I)) {
2393  Alignment = LI->getAlignment();
2394  IsVolatile = LI->isVolatile();
2395  Flags = MachineMemOperand::MOLoad;
2396  Ptr = LI->getPointerOperand();
2397  ValTy = LI->getType();
2398  } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2399  Alignment = SI->getAlignment();
2400  IsVolatile = SI->isVolatile();
2402  Ptr = SI->getPointerOperand();
2403  ValTy = SI->getValueOperand()->getType();
2404  } else
2405  return nullptr;
2406 
2407  bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
2408  bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
2409  bool IsDereferenceable =
2411  const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2412 
2413  AAMDNodes AAInfo;
2414  I->getAAMetadata(AAInfo);
2415 
2416  if (Alignment == 0) // Ensure that codegen never sees alignment 0.
2417  Alignment = DL.getABITypeAlignment(ValTy);
2418 
2419  unsigned Size = DL.getTypeStoreSize(ValTy);
2420 
2421  if (IsVolatile)
2423  if (IsNonTemporal)
2425  if (IsDereferenceable)
2427  if (IsInvariant)
2429 
2430  return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
2431  Alignment, AAInfo, Ranges);
2432 }
2433 
2435  // If both operands are the same, then try to optimize or fold the cmp.
2437  if (CI->getOperand(0) != CI->getOperand(1))
2438  return Predicate;
2439 
2440  switch (Predicate) {
2441  default: llvm_unreachable("Invalid predicate!");
2442  case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2443  case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2444  case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2445  case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2446  case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2447  case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2448  case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2449  case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2450  case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2451  case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2452  case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2453  case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2454  case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2455  case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2456  case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2457  case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2458 
2459  case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2460  case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2461  case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2462  case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2463  case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2464  case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2465  case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2466  case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2467  case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2468  case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2469  }
2470 
2471  return Predicate;
2472 }
void setHasStackMap(bool s=true)
IterTy arg_end() const
Definition: CallSite.h:583
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:570
unsigned fastEmitInst_rrr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, unsigned Op2, bool Op2IsKill)
Emit a MachineInstr with three register operands and a result register in the given register class...
Definition: FastISel.cpp:2087
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
void setByValAlign(unsigned A)
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:70
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:636
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
unsigned fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill)
Emit MachineInstrs to compute the value of Op with all but the least significant bit set to zero...
Definition: FastISel.cpp:2237
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
bool hasLocalLinkage() const
Definition: GlobalValue.h:435
This instruction extracts a struct member or array element value from an aggregate value...
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
bool lowerCall(const CallInst *I)
Definition: FastISel.cpp:1246
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
This represents the llvm.dbg.label instruction.
IterTy arg_begin() const
Definition: CallSite.h:579
This class represents lattice values for constants.
Definition: AllocatorList.h:23
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
BasicBlock * getSuccessor(unsigned Idx) const
Return the specified successor. This instruction must be a terminator.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
bool selectXRayTypedEvent(const CallInst *II)
Definition: FastISel.cpp:1060
ImmutableCallSite * CS
Definition: FastISel.h:89
bool selectGetElementPtr(const User *I)
Definition: FastISel.cpp:675
void leaveLocalValueArea(SavePoint Old)
Reset InsertPt to the given old insert position.
Definition: FastISel.cpp:575
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
bool selectStackmap(const CallInst *I)
Definition: FastISel.cpp:790
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:304
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:672
unsigned less than
Definition: InstrTypes.h:671
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:652
MachineMemOperand * createMachineMemOperandFor(const Instruction *I) const
Create a machine mem operand from the given instruction.
Definition: FastISel.cpp:2385
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:709
bool isTerminator() const
Definition: Instruction.h:128
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:662
virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode)
This method is called by target-independent code to request that an instruction with the given type a...
Definition: FastISel.cpp:1938
BasicBlock * getSuccessor(unsigned i) const
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
Metadata node.
Definition: Metadata.h:863
F(f)
SmallVector< unsigned, 4 > InRegs
Definition: FastISel.h:98
An instruction for reading from memory.
Definition: Instructions.h:167
Hexagon Common GEP
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:137
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1951
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:458
void setPhysRegsDeadExcept(ArrayRef< unsigned > UsedRegs, const TargetRegisterInfo &TRI)
Mark every physreg used by this instruction as dead except those in the UsedRegs list.
FunctionType * getFunctionType() const
Definition: CallSite.h:328
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1932
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:274
CallingConv::ID getCallingConv() const
Get the calling convention of the call.
Definition: CallSite.h:320
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:268
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1597
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1155
unsigned fastEmitInst_rii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm1, uint64_t Imm2)
Emit a MachineInstr with one register operand and two immediate operands.
Definition: FastISel.cpp:2137
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1068
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:657
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:47
unsigned fastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm)
Emit a MachineInstr with a register operand, an immediate, and a result register in the given registe...
Definition: FastISel.cpp:2115
ArrayRef< unsigned > getIndices() const
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:656
unsigned fastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm)
Emit a MachineInstr with two register operands, an immediate, and a result register in the given regi...
Definition: FastISel.cpp:2181
CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const
Definition: FastISel.cpp:2434
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:231
A description of a memory reference used in the backend.
const HexagonInstrInfo * TII
TargetLoweringBase::ArgListTy ArgListTy
Definition: FastISel.h:69
Shift and rotation operations.
Definition: ISDOpcodes.h:409
const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
Class to represent struct types.
Definition: DerivedTypes.h:232
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
DILabel * getLabel() const
bool canFoldAddIntoGEP(const User *GEP, const Value *Add)
Check if Add is an add that can be safely folded into GEP.
Definition: FastISel.cpp:2368
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned fastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill)
Emit a MachineInstr with one register operand and a result register in the given register class...
Definition: FastISel.cpp:2042
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:653
This file contains the simple types necessary to represent the attributes associated with functions a...
The memory access is dereferenceable (i.e., doesn&#39;t trap).
static MachineOperand CreateRegMask(const uint32_t *Mask)
CreateRegMask - Creates a register mask operand referencing Mask.
virtual const TargetRegisterClass * getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const
Returns the largest legal sub-class of RC that supports the sub-register index Idx.
void setByValSize(unsigned S)
Type * getType() const
Return the type of the instruction that generated this call site.
Definition: CallSite.h:272
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:668
bool selectIntrinsicCall(const IntrinsicInst *II)
Definition: FastISel.cpp:1328
bool selectCast(const User *I, unsigned Opcode)
Definition: FastISel.cpp:1496
unsigned getSizeInBits() const
Context object for machine code objects.
Definition: MCContext.h:62
InstrTy * getInstruction() const
Definition: CallSite.h:96
Class to represent function types.
Definition: DerivedTypes.h:102
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
SmallVector< ISD::InputArg, 4 > Ins
Definition: FastISel.h:97
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
unsigned constrainOperandRegClass(const MCInstrDesc &II, unsigned Op, unsigned OpNum)
Try to constrain Op so that it is usable by argument OpNum of the provided MCInstrDesc.
Definition: FastISel.cpp:2016
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:477
bool selectOperator(const User *I, unsigned Opcode)
Do "fast" instruction selection for the given LLVM IR operator (Instruction or ConstantExpr), and append generated machine instructions to the current block.
Definition: FastISel.cpp:1801
ArchType getArch() const
getArch - Get the parsed architecture type of this triple.
Definition: Triple.h:291
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
bool selectExtractValue(const User *U)
Definition: FastISel.cpp:1761
unsigned getRegForValue(const Value *V)
Create a virtual register and arrange for it to be assigned the value for the given LLVM value...
Definition: FastISel.cpp:362
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
unsigned fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC)
Emit a MachineInstr with no operands and a result register in the given register class.
Definition: FastISel.cpp:2033
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:234
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:125
bool hasTrivialKill(const Value *V)
Test whether the given value has exactly one use.
Definition: FastISel.cpp:330
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
void setOrigAlign(unsigned A)
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:83
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
unsigned getNumSuccessors() const
Return the number of successors that this instruction has.
Value * getOperand(unsigned i) const
Definition: User.h:169
Class to represent pointers.
Definition: DerivedTypes.h:498
unsigned getKillRegState(bool B)
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:523
unsigned lookUpRegForValue(const Value *V)
Look up the value to see if its value is already cached in a register.
Definition: FastISel.cpp:475
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
void finishBasicBlock()
Flush the local value map and sink local values if possible.
Definition: FastISel.cpp:144
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:228
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
The memory access is volatile.
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc (such as G_OR or G_SDIV)...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:427
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
virtual ~FastISel()
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
unsigned const MachineRegisterInfo * MRI
CallLoweringInfo & setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, ImmutableCallSite &Call)
Definition: FastISel.h:104
Machine Value Type.
Value * getCalledValue() const
Definition: InstrTypes.h:1194
bool hasName() const
Definition: Value.h:250
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
const MachineInstrBuilder & addCImm(const ConstantInt *Val) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Simple binary floating point operators.
Definition: ISDOpcodes.h:282
Conditional or Unconditional Branch instruction.
Value * getAddress() const
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:544
Value * getValue() const
SmallVector< ISD::ArgFlagsTy, 16 > OutFlags
Definition: FastISel.h:95
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
const MCPhysReg * ImplicitDefs
Definition: MCInstrDesc.h:173
DIExpression * getExpression() const
virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1940
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:433
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:160
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We&#39;re checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2312
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:146
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:646
std::pair< unsigned, bool > getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
Definition: FastISel.cpp:507
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned char TargetFlags=0)
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:771
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:655
self_iterator getIterator()
Definition: ilist_node.h:81
The memory access is non-temporal.
Class to represent integer types.
Definition: DerivedTypes.h:39
bool selectXRayCustomEvent(const CallInst *II)
Definition: FastISel.cpp:1041
const TargetRegisterInfo & TRI
Definition: FastISel.h:213
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:663
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:192
static unsigned findSinkableLocalRegDef(MachineInstr &MI)
Return the defined register if this instruction defines exactly one virtual register and uses no othe...
Definition: FastISel.cpp:168
Extended Value Type.
Definition: ValueTypes.h:33
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:529
bool selectFNeg(const User *I)
Emit an FNeg operation.
Definition: FastISel.cpp:1715
size_t size() const
Definition: SmallVector.h:52
This class contains a discriminated union of information about pointers in memory operands...
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:661
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
SmallVector< Value *, 16 > OutVals
Definition: FastISel.h:94
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call...
Definition: FastISel.cpp:1083
const TargetInstrInfo & TII
Definition: FastISel.h:211
MachineBasicBlock * MBB
MBB - The current block.
bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:500
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:43
signed greater than
Definition: InstrTypes.h:673
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:676
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:530
The memory access writes data.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:50
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:650
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:239
virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1945
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1960
MachineOperand class - Representation of each machine instruction operand.
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:660
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
bool selectCall(const User *I)
Definition: FastISel.cpp:1285
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:63
Provides information about what library functions are available for the current target.
unsigned arg_size() const
Definition: CallSite.h:226
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB)
Emit an unconditional branch to FalseMBB, obtains the branch weight and adds TrueMBB and FalseMBB to ...
Definition: FastISel.cpp:1696
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:749
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:577
signed less than
Definition: InstrTypes.h:675
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:643
reg_iterator reg_begin(unsigned RegNo) const
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
unsigned fastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
Emit a MachineInstr with two register operands and a result register in the given register class...
Definition: FastISel.cpp:2063
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:631
void updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs=1)
Update the value map to include the new mapping for this instruction, or insert an extra copy to get ...
Definition: FastISel.cpp:486
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:225
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
Definition: FastISel.cpp:130
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
signed less or equal
Definition: InstrTypes.h:676
bool selectBitCast(const User *I)
Definition: FastISel.cpp:1529
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Target - Wrapper for Target specific information.
virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1955
SmallVector< unsigned, 16 > OutRegs
Definition: FastISel.h:96
const DataLayout & DL
Definition: FastISel.h:210
bool selectBinaryOp(const User *I, unsigned ISDOpcode)
Select and emit code for a binary operator instruction, which has an opcode which directly correspond...
Definition: FastISel.cpp:584
void setDebugLoc(DebugLoc dl)
Replace current source information with new such.
amdgpu Simplify well known AMD library false FunctionCallee Callee
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringRef > StandardNames)
Initialize the set of available library functions based on the specified target triple.
This file defines the FastISel class.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:240
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:467
bool isTailCall() const
DebugLoc DbgLoc
Definition: FastISel.h:208
Flags
Flags values. These may be or&#39;d together.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
The memory access reads data.
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:593
SavePoint enterLocalValueArea()
Prepare InsertPt to begin inserting instructions into the local value area and return the old insert ...
Definition: FastISel.cpp:566
Representation of each machine instruction.
Definition: MachineInstr.h:63
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:721
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering...
Definition: FastISel.cpp:1934
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match &#39;fneg X&#39; as &#39;fsub -0.0, X&#39;.
Definition: PatternMatch.h:688
bool selectPatchpoint(const CallInst *I)
Definition: FastISel.cpp:903
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1153
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:324
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:386
MachineRegisterInfo & MRI
Definition: FastISel.h:205
bool hasOneUse(unsigned RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:123
unsigned greater or equal
Definition: InstrTypes.h:670
This represents the llvm.dbg.value instruction.
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1258
bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs)
Definition: FastISel.cpp:1105
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:44
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
Establish a view to a call site for examination.
Definition: CallSite.h:892
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1201
unsigned fastEmitInst_i(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm)
Emit a MachineInstr with a single immediate operand, and a result register in the given register clas...
Definition: FastISel.cpp:2207
static MachineOperand CreateImm(int64_t Val)
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call&#39;s live variable operands to a stackmap or patchpoint target node&#39;s ope...
The memory access always returns the same value (or traps).
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:192
static bool isRegUsedByPhiNodes(unsigned DefReg, FunctionLoweringInfo &FuncInfo)
Definition: FastISel.cpp:220
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:654
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:324
DILocalVariable * getVariable() const
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool isUnconditional() const
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:658
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:316
const TargetLowering & TLI
Definition: FastISel.h:212
unsigned createResultReg(const TargetRegisterClass *RC)
Definition: FastISel.cpp:2012
CallLoweringInfo & setIsPatchPoint(bool Value=true)
Definition: FastISel.h:182
unsigned fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm, MVT ImmType)
This method is a wrapper of fastEmit_ri.
Definition: FastISel.cpp:1969
unsigned fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, bool Op0IsKill, uint32_t Idx)
Emit a MachineInstr for an extract_subreg from a specified index of a superregister to a specified ty...
Definition: FastISel.cpp:2223
MachineBasicBlock::iterator InsertPt
Definition: FastISel.h:318
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
Definition: Value.h:375
FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, bool SkipTargetIndependentISel=false)
Definition: FastISel.cpp:1917
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1930
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:649
LLVM Value Representation.
Definition: Value.h:72
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:659
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:444
static const Function * getParent(const Value *V)
static bool isCommutative(Instruction *I)
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable&#39;s name.
Definition: Mangler.cpp:111
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
IRTranslator LLVM IR MI
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:412
void fastEmitBranch(MachineBasicBlock *MSucc, const DebugLoc &DbgLoc)
Emit an unconditional branch to the given block, unless it is the immediate (fall-through) successor...
Definition: FastISel.cpp:1676
unsigned greater than
Definition: InstrTypes.h:669
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
static cl::opt< bool > SinkLocalValues("fast-isel-sink-local-values", cl::init(true), cl::Hidden, cl::desc("Sink local values in FastISel"))
unsigned fastEmitInst_f(unsigned MachineInstOpcode, const TargetRegisterClass *RC, const ConstantFP *FPImm)
Emit a MachineInstr with a floating point immediate, and a result register in the given register clas...
Definition: FastISel.cpp:2162
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty...
Definition: Type.cpp:97
Conversion operators.
Definition: ISDOpcodes.h:464
FunctionLoweringInfo & FuncInfo
Definition: FastISel.h:203
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
Definition: Value.cpp:537
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:473
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:125
#define LLVM_DEBUG(X)
Definition: Debug.h:122
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:651
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:35
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
This represents the llvm.dbg.declare instruction.
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:585
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:63
static MachineOperand CreateFI(int Idx)
bool use_empty() const
Definition: Value.h:322
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:544
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Type * getElementType() const
Definition: DerivedTypes.h:517
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:874
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:648
signed greater or equal
Definition: InstrTypes.h:674
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:43
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
gep_type_iterator gep_type_begin(const User *GEP)