LLVM  9.0.0svn
FastISel.cpp
Go to the documentation of this file.
1 //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the implementation of the FastISel class.
10 //
11 // "Fast" instruction selection is designed to emit very poor code quickly.
12 // Also, it is not designed to be able to do much lowering, so most illegal
13 // types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14 // also not intended to be able to do much optimization, except in a few cases
15 // where doing optimizations reduces overall compile time. For example, folding
16 // constants into immediate fields is often done, because it's cheap and it
17 // reduces the number of instructions later phases have to examine.
18 //
19 // "Fast" instruction selection is able to fail gracefully and transfer
20 // control to the SelectionDAG selector for operations that it doesn't
21 // support. In many cases, this allows us to avoid duplicating a lot of
22 // the complicated lowering logic that SelectionDAG currently has.
23 //
24 // The intended use for "fast" instruction selection is "-O0" mode
25 // compilation, where the quality of the generated code is irrelevant when
26 // weighed against the speed at which the code can be generated. Also,
27 // at -O0, the LLVM optimizers are not running, and this makes the
28 // compile time of codegen a much higher portion of the overall compile
29 // time. Despite its limitations, "fast" instruction selection is able to
30 // handle enough code on its own to provide noticeable overall speedups
31 // in -O0 compiles.
32 //
33 // Basic operations are supported in a target-independent way, by reading
34 // the same instruction descriptions that the SelectionDAG selector reads,
35 // and identifying simple arithmetic operations that can be directly selected
36 // from simple operators. More complicated operations currently require
37 // target-specific code.
38 //
39 //===----------------------------------------------------------------------===//
40 
41 #include "llvm/CodeGen/FastISel.h"
42 #include "llvm/ADT/APFloat.h"
43 #include "llvm/ADT/APSInt.h"
44 #include "llvm/ADT/DenseMap.h"
45 #include "llvm/ADT/Optional.h"
46 #include "llvm/ADT/SmallPtrSet.h"
47 #include "llvm/ADT/SmallString.h"
48 #include "llvm/ADT/SmallVector.h"
49 #include "llvm/ADT/Statistic.h"
52 #include "llvm/CodeGen/Analysis.h"
63 #include "llvm/CodeGen/StackMaps.h"
68 #include "llvm/IR/Argument.h"
69 #include "llvm/IR/Attributes.h"
70 #include "llvm/IR/BasicBlock.h"
71 #include "llvm/IR/CallSite.h"
72 #include "llvm/IR/CallingConv.h"
73 #include "llvm/IR/Constant.h"
74 #include "llvm/IR/Constants.h"
75 #include "llvm/IR/DataLayout.h"
76 #include "llvm/IR/DebugInfo.h"
77 #include "llvm/IR/DebugLoc.h"
78 #include "llvm/IR/DerivedTypes.h"
79 #include "llvm/IR/Function.h"
81 #include "llvm/IR/GlobalValue.h"
82 #include "llvm/IR/InlineAsm.h"
83 #include "llvm/IR/InstrTypes.h"
84 #include "llvm/IR/Instruction.h"
85 #include "llvm/IR/Instructions.h"
86 #include "llvm/IR/IntrinsicInst.h"
87 #include "llvm/IR/LLVMContext.h"
88 #include "llvm/IR/Mangler.h"
89 #include "llvm/IR/Metadata.h"
90 #include "llvm/IR/Operator.h"
91 #include "llvm/IR/PatternMatch.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCInstrDesc.h"
97 #include "llvm/MC/MCRegisterInfo.h"
98 #include "llvm/Support/Casting.h"
99 #include "llvm/Support/Debug.h"
102 #include "llvm/Support/MathExtras.h"
106 #include <algorithm>
107 #include <cassert>
108 #include <cstdint>
109 #include <iterator>
110 #include <utility>
111 
112 using namespace llvm;
113 using namespace PatternMatch;
114 
115 #define DEBUG_TYPE "isel"
116 
117 // FIXME: Remove this after the feature has proven reliable.
118 static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values",
119  cl::init(true), cl::Hidden,
120  cl::desc("Sink local values in FastISel"));
121 
122 STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
123  "target-independent selector");
124 STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
125  "target-specific selector");
126 STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
127 
128 /// Set the current block to which generated machine instructions will be
129 /// appended.
131  assert(LocalValueMap.empty() &&
132  "local values should be cleared after finishing a BB");
133 
134  // Instructions are appended to FuncInfo.MBB. If the basic block already
135  // contains labels or copies, use the last instruction as the last local
136  // value.
137  EmitStartPt = nullptr;
138  if (!FuncInfo.MBB->empty())
139  EmitStartPt = &FuncInfo.MBB->back();
140  LastLocalValue = EmitStartPt;
141 }
142 
143 /// Flush the local CSE map and sink anything we can.
144 void FastISel::finishBasicBlock() { flushLocalValueMap(); }
145 
147  if (!FuncInfo.CanLowerReturn)
148  // Fallback to SDISel argument lowering code to deal with sret pointer
149  // parameter.
150  return false;
151 
152  if (!fastLowerArguments())
153  return false;
154 
155  // Enter arguments into ValueMap for uses in non-entry BBs.
156  for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
157  E = FuncInfo.Fn->arg_end();
158  I != E; ++I) {
159  DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(&*I);
160  assert(VI != LocalValueMap.end() && "Missed an argument?");
161  FuncInfo.ValueMap[&*I] = VI->second;
162  }
163  return true;
164 }
165 
166 /// Return the defined register if this instruction defines exactly one
167 /// virtual register and uses no other virtual registers. Otherwise return 0.
169  unsigned RegDef = 0;
170  for (const MachineOperand &MO : MI.operands()) {
171  if (!MO.isReg())
172  continue;
173  if (MO.isDef()) {
174  if (RegDef)
175  return 0;
176  RegDef = MO.getReg();
177  } else if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
178  // This is another use of a vreg. Don't try to sink it.
179  return 0;
180  }
181  }
182  return RegDef;
183 }
184 
185 void FastISel::flushLocalValueMap() {
186  // Try to sink local values down to their first use so that we can give them a
187  // better debug location. This has the side effect of shrinking local value
188  // live ranges, which helps out fast regalloc.
189  if (SinkLocalValues && LastLocalValue != EmitStartPt) {
190  // Sink local value materialization instructions between EmitStartPt and
191  // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
192  // avoid inserting into the range that we're iterating over.
194  EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
195  : FuncInfo.MBB->rend();
196  MachineBasicBlock::reverse_iterator RI(LastLocalValue);
197 
198  InstOrderMap OrderMap;
199  for (; RI != RE;) {
200  MachineInstr &LocalMI = *RI;
201  ++RI;
202  bool Store = true;
203  if (!LocalMI.isSafeToMove(nullptr, Store))
204  continue;
205  unsigned DefReg = findSinkableLocalRegDef(LocalMI);
206  if (DefReg == 0)
207  continue;
208 
209  sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
210  }
211  }
212 
213  LocalValueMap.clear();
214  LastLocalValue = EmitStartPt;
215  recomputeInsertPt();
216  SavedInsertPt = FuncInfo.InsertPt;
217  LastFlushPoint = FuncInfo.InsertPt;
218 }
219 
220 static bool isRegUsedByPhiNodes(unsigned DefReg,
221  FunctionLoweringInfo &FuncInfo) {
222  for (auto &P : FuncInfo.PHINodesToUpdate)
223  if (P.second == DefReg)
224  return true;
225  return false;
226 }
227 
228 /// Build a map of instruction orders. Return the first terminator and its
229 /// order. Consider EH_LABEL instructions to be terminators as well, since local
230 /// values for phis after invokes must be materialized before the call.
232  MachineBasicBlock *MBB, MachineBasicBlock::iterator LastFlushPoint) {
233  unsigned Order = 0;
234  for (MachineInstr &I : *MBB) {
235  if (!FirstTerminator &&
236  (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) {
237  FirstTerminator = &I;
238  FirstTerminatorOrder = Order;
239  }
240  Orders[&I] = Order++;
241 
242  // We don't need to order instructions past the last flush point.
243  if (I.getIterator() == LastFlushPoint)
244  break;
245  }
246 }
247 
248 void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
249  unsigned DefReg,
250  InstOrderMap &OrderMap) {
251  // If this register is used by a register fixup, MRI will not contain all
252  // the uses until after register fixups, so don't attempt to sink or DCE
253  // this instruction. Register fixups typically come from no-op cast
254  // instructions, which replace the cast instruction vreg with the local
255  // value vreg.
256  if (FuncInfo.RegsWithFixups.count(DefReg))
257  return;
258 
259  // We can DCE this instruction if there are no uses and it wasn't a
260  // materialized for a successor PHI node.
261  bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
262  if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
263  if (EmitStartPt == &LocalMI)
264  EmitStartPt = EmitStartPt->getPrevNode();
265  LLVM_DEBUG(dbgs() << "removing dead local value materialization "
266  << LocalMI);
267  OrderMap.Orders.erase(&LocalMI);
268  LocalMI.eraseFromParent();
269  return;
270  }
271 
272  // Number the instructions if we haven't yet so we can efficiently find the
273  // earliest use.
274  if (OrderMap.Orders.empty())
275  OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
276 
277  // Find the first user in the BB.
278  MachineInstr *FirstUser = nullptr;
279  unsigned FirstOrder = std::numeric_limits<unsigned>::max();
280  for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
281  auto I = OrderMap.Orders.find(&UseInst);
282  assert(I != OrderMap.Orders.end() &&
283  "local value used by instruction outside local region");
284  unsigned UseOrder = I->second;
285  if (UseOrder < FirstOrder) {
286  FirstOrder = UseOrder;
287  FirstUser = &UseInst;
288  }
289  }
290 
291  // The insertion point will be the first terminator or the first user,
292  // whichever came first. If there was no terminator, this must be a
293  // fallthrough block and the insertion point is the end of the block.
295  if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder) {
296  FirstOrder = OrderMap.FirstTerminatorOrder;
297  SinkPos = OrderMap.FirstTerminator->getIterator();
298  } else if (FirstUser) {
299  SinkPos = FirstUser->getIterator();
300  } else {
301  assert(UsedByPHI && "must be users if not used by a phi");
302  SinkPos = FuncInfo.MBB->instr_end();
303  }
304 
305  // Collect all DBG_VALUEs before the new insertion position so that we can
306  // sink them.
308  for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
309  if (!DbgVal.isDebugValue())
310  continue;
311  unsigned UseOrder = OrderMap.Orders[&DbgVal];
312  if (UseOrder < FirstOrder)
313  DbgValues.push_back(&DbgVal);
314  }
315 
316  // Sink LocalMI before SinkPos and assign it the same DebugLoc.
317  LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
318  FuncInfo.MBB->remove(&LocalMI);
319  FuncInfo.MBB->insert(SinkPos, &LocalMI);
320  if (SinkPos != FuncInfo.MBB->end())
321  LocalMI.setDebugLoc(SinkPos->getDebugLoc());
322 
323  // Sink any debug values that we've collected.
324  for (MachineInstr *DI : DbgValues) {
325  FuncInfo.MBB->remove(DI);
326  FuncInfo.MBB->insert(SinkPos, DI);
327  }
328 }
329 
331  // Don't consider constants or arguments to have trivial kills.
332  const Instruction *I = dyn_cast<Instruction>(V);
333  if (!I)
334  return false;
335 
336  // No-op casts are trivially coalesced by fast-isel.
337  if (const auto *Cast = dyn_cast<CastInst>(I))
338  if (Cast->isNoopCast(DL) && !hasTrivialKill(Cast->getOperand(0)))
339  return false;
340 
341  // Even the value might have only one use in the LLVM IR, it is possible that
342  // FastISel might fold the use into another instruction and now there is more
343  // than one use at the Machine Instruction level.
344  unsigned Reg = lookUpRegForValue(V);
345  if (Reg && !MRI.use_empty(Reg))
346  return false;
347 
348  // GEPs with all zero indices are trivially coalesced by fast-isel.
349  if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
350  if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0)))
351  return false;
352 
353  // Only instructions with a single use in the same basic block are considered
354  // to have trivial kills.
355  return I->hasOneUse() &&
356  !(I->getOpcode() == Instruction::BitCast ||
357  I->getOpcode() == Instruction::PtrToInt ||
358  I->getOpcode() == Instruction::IntToPtr) &&
359  cast<Instruction>(*I->user_begin())->getParent() == I->getParent();
360 }
361 
362 unsigned FastISel::getRegForValue(const Value *V) {
363  EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
364  // Don't handle non-simple values in FastISel.
365  if (!RealVT.isSimple())
366  return 0;
367 
368  // Ignore illegal types. We must do this before looking up the value
369  // in ValueMap because Arguments are given virtual registers regardless
370  // of whether FastISel can handle them.
371  MVT VT = RealVT.getSimpleVT();
372  if (!TLI.isTypeLegal(VT)) {
373  // Handle integer promotions, though, because they're common and easy.
374  if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
375  VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
376  else
377  return 0;
378  }
379 
380  // Look up the value to see if we already have a register for it.
381  unsigned Reg = lookUpRegForValue(V);
382  if (Reg)
383  return Reg;
384 
385  // In bottom-up mode, just create the virtual register which will be used
386  // to hold the value. It will be materialized later.
387  if (isa<Instruction>(V) &&
388  (!isa<AllocaInst>(V) ||
389  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
390  return FuncInfo.InitializeRegForValue(V);
391 
392  SavePoint SaveInsertPt = enterLocalValueArea();
393 
394  // Materialize the value in a register. Emit any instructions in the
395  // local value area.
396  Reg = materializeRegForValue(V, VT);
397 
398  leaveLocalValueArea(SaveInsertPt);
399 
400  return Reg;
401 }
402 
403 unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
404  unsigned Reg = 0;
405  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
406  if (CI->getValue().getActiveBits() <= 64)
407  Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
408  } else if (isa<AllocaInst>(V))
409  Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
410  else if (isa<ConstantPointerNull>(V))
411  // Translate this as an integer zero so that it can be
412  // local-CSE'd with actual integer zeros.
413  Reg = getRegForValue(
414  Constant::getNullValue(DL.getIntPtrType(V->getContext())));
415  else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
416  if (CF->isNullValue())
417  Reg = fastMaterializeFloatZero(CF);
418  else
419  // Try to emit the constant directly.
420  Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
421 
422  if (!Reg) {
423  // Try to emit the constant by using an integer constant with a cast.
424  const APFloat &Flt = CF->getValueAPF();
425  EVT IntVT = TLI.getPointerTy(DL);
426  uint32_t IntBitWidth = IntVT.getSizeInBits();
427  APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
428  bool isExact;
429  (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
430  if (isExact) {
431  unsigned IntegerReg =
432  getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
433  if (IntegerReg != 0)
434  Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
435  /*Kill=*/false);
436  }
437  }
438  } else if (const auto *Op = dyn_cast<Operator>(V)) {
439  if (!selectOperator(Op, Op->getOpcode()))
440  if (!isa<Instruction>(Op) ||
441  !fastSelectInstruction(cast<Instruction>(Op)))
442  return 0;
443  Reg = lookUpRegForValue(Op);
444  } else if (isa<UndefValue>(V)) {
445  Reg = createResultReg(TLI.getRegClassFor(VT));
446  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
447  TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
448  }
449  return Reg;
450 }
451 
452 /// Helper for getRegForValue. This function is called when the value isn't
453 /// already available in a register and must be materialized with new
454 /// instructions.
455 unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
456  unsigned Reg = 0;
457  // Give the target-specific code a try first.
458  if (isa<Constant>(V))
459  Reg = fastMaterializeConstant(cast<Constant>(V));
460 
461  // If target-specific code couldn't or didn't want to handle the value, then
462  // give target-independent code a try.
463  if (!Reg)
464  Reg = materializeConstant(V, VT);
465 
466  // Don't cache constant materializations in the general ValueMap.
467  // To do so would require tracking what uses they dominate.
468  if (Reg) {
469  LocalValueMap[V] = Reg;
470  LastLocalValue = MRI.getVRegDef(Reg);
471  }
472  return Reg;
473 }
474 
475 unsigned FastISel::lookUpRegForValue(const Value *V) {
476  // Look up the value to see if we already have a register for it. We
477  // cache values defined by Instructions across blocks, and other values
478  // only locally. This is because Instructions already have the SSA
479  // def-dominates-use requirement enforced.
480  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(V);
481  if (I != FuncInfo.ValueMap.end())
482  return I->second;
483  return LocalValueMap[V];
484 }
485 
486 void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
487  if (!isa<Instruction>(I)) {
488  LocalValueMap[I] = Reg;
489  return;
490  }
491 
492  unsigned &AssignedReg = FuncInfo.ValueMap[I];
493  if (AssignedReg == 0)
494  // Use the new register.
495  AssignedReg = Reg;
496  else if (Reg != AssignedReg) {
497  // Arrange for uses of AssignedReg to be replaced by uses of Reg.
498  for (unsigned i = 0; i < NumRegs; i++) {
499  FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
500  FuncInfo.RegsWithFixups.insert(Reg + i);
501  }
502 
503  AssignedReg = Reg;
504  }
505 }
506 
507 std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
508  unsigned IdxN = getRegForValue(Idx);
509  if (IdxN == 0)
510  // Unhandled operand. Halt "fast" selection and bail.
511  return std::pair<unsigned, bool>(0, false);
512 
513  bool IdxNIsKill = hasTrivialKill(Idx);
514 
515  // If the index is smaller or larger than intptr_t, truncate or extend it.
516  MVT PtrVT = TLI.getPointerTy(DL);
517  EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
518  if (IdxVT.bitsLT(PtrVT)) {
519  IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
520  IdxNIsKill);
521  IdxNIsKill = true;
522  } else if (IdxVT.bitsGT(PtrVT)) {
523  IdxN =
524  fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
525  IdxNIsKill = true;
526  }
527  return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
528 }
529 
531  if (getLastLocalValue()) {
532  FuncInfo.InsertPt = getLastLocalValue();
533  FuncInfo.MBB = FuncInfo.InsertPt->getParent();
534  ++FuncInfo.InsertPt;
535  } else
536  FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
537 
538  // Now skip past any EH_LABELs, which must remain at the beginning.
539  while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
540  FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL)
541  ++FuncInfo.InsertPt;
542 }
543 
546  assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
547  "Invalid iterator!");
548  while (I != E) {
549  if (LastFlushPoint == I)
550  LastFlushPoint = E;
551  if (SavedInsertPt == I)
552  SavedInsertPt = E;
553  if (EmitStartPt == I)
554  EmitStartPt = E.isValid() ? &*E : nullptr;
555  if (LastLocalValue == I)
556  LastLocalValue = E.isValid() ? &*E : nullptr;
557 
558  MachineInstr *Dead = &*I;
559  ++I;
560  Dead->eraseFromParent();
561  ++NumFastIselDead;
562  }
563  recomputeInsertPt();
564 }
565 
567  MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt;
568  DebugLoc OldDL = DbgLoc;
569  recomputeInsertPt();
570  DbgLoc = DebugLoc();
571  SavePoint SP = {OldInsertPt, OldDL};
572  return SP;
573 }
574 
576  if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
577  LastLocalValue = &*std::prev(FuncInfo.InsertPt);
578 
579  // Restore the previous insert position.
580  FuncInfo.InsertPt = OldInsertPt.InsertPt;
581  DbgLoc = OldInsertPt.DL;
582 }
583 
584 bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
585  EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
586  if (VT == MVT::Other || !VT.isSimple())
587  // Unhandled type. Halt "fast" selection and bail.
588  return false;
589 
590  // We only handle legal types. For example, on x86-32 the instruction
591  // selector contains all of the 64-bit instructions from x86-64,
592  // under the assumption that i64 won't be used if the target doesn't
593  // support it.
594  if (!TLI.isTypeLegal(VT)) {
595  // MVT::i1 is special. Allow AND, OR, or XOR because they
596  // don't require additional zeroing, which makes them easy.
597  if (VT == MVT::i1 && (ISDOpcode == ISD::AND || ISDOpcode == ISD::OR ||
598  ISDOpcode == ISD::XOR))
599  VT = TLI.getTypeToTransformTo(I->getContext(), VT);
600  else
601  return false;
602  }
603 
604  // Check if the first operand is a constant, and handle it as "ri". At -O0,
605  // we don't have anything that canonicalizes operand order.
606  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
607  if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
608  unsigned Op1 = getRegForValue(I->getOperand(1));
609  if (!Op1)
610  return false;
611  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
612 
613  unsigned ResultReg =
614  fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
615  CI->getZExtValue(), VT.getSimpleVT());
616  if (!ResultReg)
617  return false;
618 
619  // We successfully emitted code for the given LLVM Instruction.
620  updateValueMap(I, ResultReg);
621  return true;
622  }
623 
624  unsigned Op0 = getRegForValue(I->getOperand(0));
625  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
626  return false;
627  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
628 
629  // Check if the second operand is a constant and handle it appropriately.
630  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
631  uint64_t Imm = CI->getSExtValue();
632 
633  // Transform "sdiv exact X, 8" -> "sra X, 3".
634  if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
635  cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
636  Imm = Log2_64(Imm);
637  ISDOpcode = ISD::SRA;
638  }
639 
640  // Transform "urem x, pow2" -> "and x, pow2-1".
641  if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
642  isPowerOf2_64(Imm)) {
643  --Imm;
644  ISDOpcode = ISD::AND;
645  }
646 
647  unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
648  Op0IsKill, Imm, VT.getSimpleVT());
649  if (!ResultReg)
650  return false;
651 
652  // We successfully emitted code for the given LLVM Instruction.
653  updateValueMap(I, ResultReg);
654  return true;
655  }
656 
657  unsigned Op1 = getRegForValue(I->getOperand(1));
658  if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
659  return false;
660  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
661 
662  // Now we have both operands in registers. Emit the instruction.
663  unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
664  ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
665  if (!ResultReg)
666  // Target-specific code wasn't able to find a machine opcode for
667  // the given ISD opcode and type. Halt "fast" selection and bail.
668  return false;
669 
670  // We successfully emitted code for the given LLVM Instruction.
671  updateValueMap(I, ResultReg);
672  return true;
673 }
674 
676  unsigned N = getRegForValue(I->getOperand(0));
677  if (!N) // Unhandled operand. Halt "fast" selection and bail.
678  return false;
679  bool NIsKill = hasTrivialKill(I->getOperand(0));
680 
681  // Keep a running tab of the total offset to coalesce multiple N = N + Offset
682  // into a single N = N + TotalOffset.
683  uint64_t TotalOffs = 0;
684  // FIXME: What's a good SWAG number for MaxOffs?
685  uint64_t MaxOffs = 2048;
686  MVT VT = TLI.getPointerTy(DL);
687  for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
688  GTI != E; ++GTI) {
689  const Value *Idx = GTI.getOperand();
690  if (StructType *StTy = GTI.getStructTypeOrNull()) {
691  uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
692  if (Field) {
693  // N = N + Offset
694  TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
695  if (TotalOffs >= MaxOffs) {
696  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
697  if (!N) // Unhandled operand. Halt "fast" selection and bail.
698  return false;
699  NIsKill = true;
700  TotalOffs = 0;
701  }
702  }
703  } else {
704  Type *Ty = GTI.getIndexedType();
705 
706  // If this is a constant subscript, handle it quickly.
707  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
708  if (CI->isZero())
709  continue;
710  // N = N + Offset
711  uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
712  TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
713  if (TotalOffs >= MaxOffs) {
714  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
715  if (!N) // Unhandled operand. Halt "fast" selection and bail.
716  return false;
717  NIsKill = true;
718  TotalOffs = 0;
719  }
720  continue;
721  }
722  if (TotalOffs) {
723  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
724  if (!N) // Unhandled operand. Halt "fast" selection and bail.
725  return false;
726  NIsKill = true;
727  TotalOffs = 0;
728  }
729 
730  // N = N + Idx * ElementSize;
731  uint64_t ElementSize = DL.getTypeAllocSize(Ty);
732  std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
733  unsigned IdxN = Pair.first;
734  bool IdxNIsKill = Pair.second;
735  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
736  return false;
737 
738  if (ElementSize != 1) {
739  IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
740  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
741  return false;
742  IdxNIsKill = true;
743  }
744  N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
745  if (!N) // Unhandled operand. Halt "fast" selection and bail.
746  return false;
747  }
748  }
749  if (TotalOffs) {
750  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
751  if (!N) // Unhandled operand. Halt "fast" selection and bail.
752  return false;
753  }
754 
755  // We successfully emitted code for the given LLVM Instruction.
756  updateValueMap(I, N);
757  return true;
758 }
759 
760 bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
761  const CallInst *CI, unsigned StartIdx) {
762  for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) {
763  Value *Val = CI->getArgOperand(i);
764  // Check for constants and encode them with a StackMaps::ConstantOp prefix.
765  if (const auto *C = dyn_cast<ConstantInt>(Val)) {
766  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
767  Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
768  } else if (isa<ConstantPointerNull>(Val)) {
769  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
771  } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
772  // Values coming from a stack location also require a special encoding,
773  // but that is added later on by the target specific frame index
774  // elimination implementation.
775  auto SI = FuncInfo.StaticAllocaMap.find(AI);
776  if (SI != FuncInfo.StaticAllocaMap.end())
777  Ops.push_back(MachineOperand::CreateFI(SI->second));
778  else
779  return false;
780  } else {
781  unsigned Reg = getRegForValue(Val);
782  if (!Reg)
783  return false;
784  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
785  }
786  }
787  return true;
788 }
789 
791  // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
792  // [live variables...])
794  "Stackmap cannot return a value.");
795 
796  // The stackmap intrinsic only records the live variables (the arguments
797  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
798  // intrinsic, this won't be lowered to a function call. This means we don't
799  // have to worry about calling conventions and target-specific lowering code.
800  // Instead we perform the call lowering right here.
801  //
802  // CALLSEQ_START(0, 0...)
803  // STACKMAP(id, nbytes, ...)
804  // CALLSEQ_END(0, 0)
805  //
807 
808  // Add the <id> and <numBytes> constants.
809  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
810  "Expected a constant integer.");
811  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
812  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
813 
814  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
815  "Expected a constant integer.");
816  const auto *NumBytes =
817  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
818  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
819 
820  // Push live variables for the stack map (skipping the first two arguments
821  // <id> and <numBytes>).
822  if (!addStackMapLiveVars(Ops, I, 2))
823  return false;
824 
825  // We are not adding any register mask info here, because the stackmap doesn't
826  // clobber anything.
827 
828  // Add scratch registers as implicit def and early clobber.
830  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
831  for (unsigned i = 0; ScratchRegs[i]; ++i)
832  Ops.push_back(MachineOperand::CreateReg(
833  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
834  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
835 
836  // Issue CALLSEQ_START
837  unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
838  auto Builder =
839  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
840  const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
841  for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
842  Builder.addImm(0);
843 
844  // Issue STACKMAP.
845  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
846  TII.get(TargetOpcode::STACKMAP));
847  for (auto const &MO : Ops)
848  MIB.add(MO);
849 
850  // Issue CALLSEQ_END
851  unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
852  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
853  .addImm(0)
854  .addImm(0);
855 
856  // Inform the Frame Information that we have a stackmap in this function.
857  FuncInfo.MF->getFrameInfo().setHasStackMap();
858 
859  return true;
860 }
861 
862 /// Lower an argument list according to the target calling convention.
863 ///
864 /// This is a helper for lowering intrinsics that follow a target calling
865 /// convention or require stack pointer adjustment. Only a subset of the
866 /// intrinsic's operands need to participate in the calling convention.
867 bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
868  unsigned NumArgs, const Value *Callee,
869  bool ForceRetVoidTy, CallLoweringInfo &CLI) {
870  ArgListTy Args;
871  Args.reserve(NumArgs);
872 
873  // Populate the argument list.
874  ImmutableCallSite CS(CI);
875  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
876  Value *V = CI->getOperand(ArgI);
877 
878  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
879 
880  ArgListEntry Entry;
881  Entry.Val = V;
882  Entry.Ty = V->getType();
883  Entry.setAttributes(&CS, ArgI);
884  Args.push_back(Entry);
885  }
886 
887  Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
888  : CI->getType();
889  CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
890 
891  return lowerCallTo(CLI);
892 }
893 
895  const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
896  StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
897  SmallString<32> MangledName;
898  Mangler::getNameWithPrefix(MangledName, Target, DL);
899  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
900  return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
901 }
902 
904  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
905  // i32 <numBytes>,
906  // i8* <target>,
907  // i32 <numArgs>,
908  // [Args...],
909  // [live variables...])
911  bool IsAnyRegCC = CC == CallingConv::AnyReg;
912  bool HasDef = !I->getType()->isVoidTy();
914 
915  // Get the real number of arguments participating in the call <numArgs>
916  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
917  "Expected a constant integer.");
918  const auto *NumArgsVal =
919  cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
920  unsigned NumArgs = NumArgsVal->getZExtValue();
921 
922  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
923  // This includes all meta-operands up to but not including CC.
924  unsigned NumMetaOpers = PatchPointOpers::CCPos;
925  assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
926  "Not enough arguments provided to the patchpoint intrinsic");
927 
928  // For AnyRegCC the arguments are lowered later on manually.
929  unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
930  CallLoweringInfo CLI;
931  CLI.setIsPatchPoint();
932  if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
933  return false;
934 
935  assert(CLI.Call && "No call instruction specified.");
936 
938 
939  // Add an explicit result reg if we use the anyreg calling convention.
940  if (IsAnyRegCC && HasDef) {
941  assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
942  CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64));
943  CLI.NumResultRegs = 1;
944  Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*IsDef=*/true));
945  }
946 
947  // Add the <id> and <numBytes> constants.
948  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
949  "Expected a constant integer.");
950  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
951  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
952 
953  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
954  "Expected a constant integer.");
955  const auto *NumBytes =
956  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
957  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
958 
959  // Add the call target.
960  if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
961  uint64_t CalleeConstAddr =
962  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
963  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
964  } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
965  if (C->getOpcode() == Instruction::IntToPtr) {
966  uint64_t CalleeConstAddr =
967  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
968  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
969  } else
970  llvm_unreachable("Unsupported ConstantExpr.");
971  } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
973  } else if (isa<ConstantPointerNull>(Callee))
975  else
976  llvm_unreachable("Unsupported callee address.");
977 
978  // Adjust <numArgs> to account for any arguments that have been passed on
979  // the stack instead.
980  unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
981  Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
982 
983  // Add the calling convention
984  Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
985 
986  // Add the arguments we omitted previously. The register allocator should
987  // place these in any free register.
988  if (IsAnyRegCC) {
989  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
990  unsigned Reg = getRegForValue(I->getArgOperand(i));
991  if (!Reg)
992  return false;
993  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
994  }
995  }
996 
997  // Push the arguments from the call instruction.
998  for (auto Reg : CLI.OutRegs)
999  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false));
1000 
1001  // Push live variables for the stack map.
1002  if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
1003  return false;
1004 
1005  // Push the register mask info.
1007  TRI.getCallPreservedMask(*FuncInfo.MF, CC)));
1008 
1009  // Add scratch registers as implicit def and early clobber.
1010  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
1011  for (unsigned i = 0; ScratchRegs[i]; ++i)
1013  ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false,
1014  /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true));
1015 
1016  // Add implicit defs (return values).
1017  for (auto Reg : CLI.InRegs)
1018  Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/true,
1019  /*IsImpl=*/true));
1020 
1021  // Insert the patchpoint instruction before the call generated by the target.
1022  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc,
1023  TII.get(TargetOpcode::PATCHPOINT));
1024 
1025  for (auto &MO : Ops)
1026  MIB.add(MO);
1027 
1028  MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1029 
1030  // Delete the original call instruction.
1031  CLI.Call->eraseFromParent();
1032 
1033  // Inform the Frame Information that we have a patchpoint in this function.
1034  FuncInfo.MF->getFrameInfo().setHasPatchPoint();
1035 
1036  if (CLI.NumResultRegs)
1037  updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
1038  return true;
1039 }
1040 
1042  const auto &Triple = TM.getTargetTriple();
1043  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1044  return true; // don't do anything to this instruction.
1046  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1047  /*IsDef=*/false));
1048  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1049  /*IsDef=*/false));
1050  MachineInstrBuilder MIB =
1051  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1052  TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
1053  for (auto &MO : Ops)
1054  MIB.add(MO);
1055 
1056  // Insert the Patchable Event Call instruction, that gets lowered properly.
1057  return true;
1058 }
1059 
1061  const auto &Triple = TM.getTargetTriple();
1062  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1063  return true; // don't do anything to this instruction.
1065  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1066  /*IsDef=*/false));
1067  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1068  /*IsDef=*/false));
1069  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(2)),
1070  /*IsDef=*/false));
1071  MachineInstrBuilder MIB =
1072  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1073  TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
1074  for (auto &MO : Ops)
1075  MIB.add(MO);
1076 
1077  // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
1078  return true;
1079 }
1080 
1081 /// Returns an AttributeList representing the attributes applied to the return
1082 /// value of the given call.
1085  if (CLI.RetSExt)
1086  Attrs.push_back(Attribute::SExt);
1087  if (CLI.RetZExt)
1088  Attrs.push_back(Attribute::ZExt);
1089  if (CLI.IsInReg)
1090  Attrs.push_back(Attribute::InReg);
1091 
1093  Attrs);
1094 }
1095 
1096 bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
1097  unsigned NumArgs) {
1098  MCContext &Ctx = MF->getContext();
1099  SmallString<32> MangledName;
1100  Mangler::getNameWithPrefix(MangledName, SymName, DL);
1101  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
1102  return lowerCallTo(CI, Sym, NumArgs);
1103 }
1104 
1106  unsigned NumArgs) {
1107  ImmutableCallSite CS(CI);
1108 
1109  FunctionType *FTy = CS.getFunctionType();
1110  Type *RetTy = CS.getType();
1111 
1112  ArgListTy Args;
1113  Args.reserve(NumArgs);
1114 
1115  // Populate the argument list.
1116  // Attributes for args start at offset 1, after the return attribute.
1117  for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
1118  Value *V = CI->getOperand(ArgI);
1119 
1120  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
1121 
1122  ArgListEntry Entry;
1123  Entry.Val = V;
1124  Entry.Ty = V->getType();
1125  Entry.setAttributes(&CS, ArgI);
1126  Args.push_back(Entry);
1127  }
1128  TLI.markLibCallAttributes(MF, CS.getCallingConv(), Args);
1129 
1130  CallLoweringInfo CLI;
1131  CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
1132 
1133  return lowerCallTo(CLI);
1134 }
1135 
1137  // Handle the incoming return values from the call.
1138  CLI.clearIns();
1139  SmallVector<EVT, 4> RetTys;
1140  ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
1141 
1143  GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
1144 
1145  bool CanLowerReturn = TLI.CanLowerReturn(
1146  CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1147 
1148  // FIXME: sret demotion isn't supported yet - bail out.
1149  if (!CanLowerReturn)
1150  return false;
1151 
1152  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
1153  EVT VT = RetTys[I];
1154  MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1155  unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1156  for (unsigned i = 0; i != NumRegs; ++i) {
1157  ISD::InputArg MyFlags;
1158  MyFlags.VT = RegisterVT;
1159  MyFlags.ArgVT = VT;
1160  MyFlags.Used = CLI.IsReturnValueUsed;
1161  if (CLI.RetSExt)
1162  MyFlags.Flags.setSExt();
1163  if (CLI.RetZExt)
1164  MyFlags.Flags.setZExt();
1165  if (CLI.IsInReg)
1166  MyFlags.Flags.setInReg();
1167  CLI.Ins.push_back(MyFlags);
1168  }
1169  }
1170 
1171  // Handle all of the outgoing arguments.
1172  CLI.clearOuts();
1173  for (auto &Arg : CLI.getArgs()) {
1174  Type *FinalType = Arg.Ty;
1175  if (Arg.IsByVal)
1176  FinalType = cast<PointerType>(Arg.Ty)->getElementType();
1177  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1178  FinalType, CLI.CallConv, CLI.IsVarArg);
1179 
1180  ISD::ArgFlagsTy Flags;
1181  if (Arg.IsZExt)
1182  Flags.setZExt();
1183  if (Arg.IsSExt)
1184  Flags.setSExt();
1185  if (Arg.IsInReg)
1186  Flags.setInReg();
1187  if (Arg.IsSRet)
1188  Flags.setSRet();
1189  if (Arg.IsSwiftSelf)
1190  Flags.setSwiftSelf();
1191  if (Arg.IsSwiftError)
1192  Flags.setSwiftError();
1193  if (Arg.IsByVal)
1194  Flags.setByVal();
1195  if (Arg.IsInAlloca) {
1196  Flags.setInAlloca();
1197  // Set the byval flag for CCAssignFn callbacks that don't know about
1198  // inalloca. This way we can know how many bytes we should've allocated
1199  // and how many bytes a callee cleanup function will pop. If we port
1200  // inalloca to more targets, we'll have to add custom inalloca handling in
1201  // the various CC lowering callbacks.
1202  Flags.setByVal();
1203  }
1204  if (Arg.IsByVal || Arg.IsInAlloca) {
1205  PointerType *Ty = cast<PointerType>(Arg.Ty);
1206  Type *ElementTy = Ty->getElementType();
1207  unsigned FrameSize = DL.getTypeAllocSize(ElementTy);
1208  // For ByVal, alignment should come from FE. BE will guess if this info is
1209  // not there, but there are cases it cannot get right.
1210  unsigned FrameAlign = Arg.Alignment;
1211  if (!FrameAlign)
1212  FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
1213  Flags.setByValSize(FrameSize);
1214  Flags.setByValAlign(FrameAlign);
1215  }
1216  if (Arg.IsNest)
1217  Flags.setNest();
1218  if (NeedsRegBlock)
1219  Flags.setInConsecutiveRegs();
1220  unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
1221  Flags.setOrigAlign(OriginalAlignment);
1222 
1223  CLI.OutVals.push_back(Arg.Val);
1224  CLI.OutFlags.push_back(Flags);
1225  }
1226 
1227  if (!fastLowerCall(CLI))
1228  return false;
1229 
1230  // Set all unused physreg defs as dead.
1231  assert(CLI.Call && "No call instruction specified.");
1232  CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1233 
1234  if (CLI.NumResultRegs && CLI.CS)
1235  updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs);
1236 
1237  return true;
1238 }
1239 
1241  ImmutableCallSite CS(CI);
1242 
1243  FunctionType *FuncTy = CS.getFunctionType();
1244  Type *RetTy = CS.getType();
1245 
1246  ArgListTy Args;
1247  ArgListEntry Entry;
1248  Args.reserve(CS.arg_size());
1249 
1250  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
1251  i != e; ++i) {
1252  Value *V = *i;
1253 
1254  // Skip empty types
1255  if (V->getType()->isEmptyTy())
1256  continue;
1257 
1258  Entry.Val = V;
1259  Entry.Ty = V->getType();
1260 
1261  // Skip the first return-type Attribute to get to params.
1262  Entry.setAttributes(&CS, i - CS.arg_begin());
1263  Args.push_back(Entry);
1264  }
1265 
1266  // Check if target-independent constraints permit a tail call here.
1267  // Target-dependent constraints are checked within fastLowerCall.
1268  bool IsTailCall = CI->isTailCall();
1269  if (IsTailCall && !isInTailCallPosition(CS, TM))
1270  IsTailCall = false;
1271 
1272  CallLoweringInfo CLI;
1273  CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
1274  .setTailCall(IsTailCall);
1275 
1276  return lowerCallTo(CLI);
1277 }
1278 
1280  const CallInst *Call = cast<CallInst>(I);
1281 
1282  // Handle simple inline asms.
1283  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
1284  // If the inline asm has side effects, then make sure that no local value
1285  // lives across by flushing the local value map.
1286  if (IA->hasSideEffects())
1287  flushLocalValueMap();
1288 
1289  // Don't attempt to handle constraints.
1290  if (!IA->getConstraintString().empty())
1291  return false;
1292 
1293  unsigned ExtraInfo = 0;
1294  if (IA->hasSideEffects())
1295  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1296  if (IA->isAlignStack())
1297  ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1298 
1299  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1301  .addExternalSymbol(IA->getAsmString().c_str())
1302  .addImm(ExtraInfo);
1303  return true;
1304  }
1305 
1306  // Handle intrinsic function calls.
1307  if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1308  return selectIntrinsicCall(II);
1309 
1310  // Usually, it does not make sense to initialize a value,
1311  // make an unrelated function call and use the value, because
1312  // it tends to be spilled on the stack. So, we move the pointer
1313  // to the last local value to the beginning of the block, so that
1314  // all the values which have already been materialized,
1315  // appear after the call. It also makes sense to skip intrinsics
1316  // since they tend to be inlined.
1317  flushLocalValueMap();
1318 
1319  return lowerCall(Call);
1320 }
1321 
1323  switch (II->getIntrinsicID()) {
1324  default:
1325  break;
1326  // At -O0 we don't care about the lifetime intrinsics.
1327  case Intrinsic::lifetime_start:
1328  case Intrinsic::lifetime_end:
1329  // The donothing intrinsic does, well, nothing.
1330  case Intrinsic::donothing:
1331  // Neither does the sideeffect intrinsic.
1332  case Intrinsic::sideeffect:
1333  // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1334  case Intrinsic::assume:
1335  return true;
1336  case Intrinsic::dbg_declare: {
1337  const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1338  assert(DI->getVariable() && "Missing variable");
1339  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1340  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1341  return true;
1342  }
1343 
1344  const Value *Address = DI->getAddress();
1345  if (!Address || isa<UndefValue>(Address)) {
1346  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1347  return true;
1348  }
1349 
1350  // Byval arguments with frame indices were already handled after argument
1351  // lowering and before isel.
1352  const auto *Arg =
1354  if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
1355  return true;
1356 
1358  if (unsigned Reg = lookUpRegForValue(Address))
1359  Op = MachineOperand::CreateReg(Reg, false);
1360 
1361  // If we have a VLA that has a "use" in a metadata node that's then used
1362  // here but it has no other uses, then we have a problem. E.g.,
1363  //
1364  // int foo (const int *x) {
1365  // char a[*x];
1366  // return 0;
1367  // }
1368  //
1369  // If we assign 'a' a vreg and fast isel later on has to use the selection
1370  // DAG isel, it will want to copy the value to the vreg. However, there are
1371  // no uses, which goes counter to what selection DAG isel expects.
1372  if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
1373  (!isa<AllocaInst>(Address) ||
1374  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
1375  Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address),
1376  false);
1377 
1378  if (Op) {
1380  "Expected inlined-at fields to agree");
1381  // A dbg.declare describes the address of a source variable, so lower it
1382  // into an indirect DBG_VALUE.
1383  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1384  TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true,
1385  *Op, DI->getVariable(), DI->getExpression());
1386  } else {
1387  // We can't yet handle anything else here because it would require
1388  // generating code, thus altering codegen because of debug info.
1389  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1390  }
1391  return true;
1392  }
1393  case Intrinsic::dbg_value: {
1394  // This form of DBG_VALUE is target-independent.
1395  const DbgValueInst *DI = cast<DbgValueInst>(II);
1396  const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1397  const Value *V = DI->getValue();
1399  "Expected inlined-at fields to agree");
1400  if (!V) {
1401  // Currently the optimizer can produce this; insert an undef to
1402  // help debugging. Probably the optimizer should not do this.
1403  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
1404  DI->getVariable(), DI->getExpression());
1405  } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1406  if (CI->getBitWidth() > 64)
1407  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1408  .addCImm(CI)
1409  .addImm(0U)
1410  .addMetadata(DI->getVariable())
1411  .addMetadata(DI->getExpression());
1412  else
1413  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1414  .addImm(CI->getZExtValue())
1415  .addImm(0U)
1416  .addMetadata(DI->getVariable())
1417  .addMetadata(DI->getExpression());
1418  } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1419  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1420  .addFPImm(CF)
1421  .addImm(0U)
1422  .addMetadata(DI->getVariable())
1423  .addMetadata(DI->getExpression());
1424  } else if (unsigned Reg = lookUpRegForValue(V)) {
1425  // FIXME: This does not handle register-indirect values at offset 0.
1426  bool IsIndirect = false;
1427  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
1428  DI->getVariable(), DI->getExpression());
1429  } else {
1430  // We can't yet handle anything else here because it would require
1431  // generating code, thus altering codegen because of debug info.
1432  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1433  }
1434  return true;
1435  }
1436  case Intrinsic::dbg_label: {
1437  const DbgLabelInst *DI = cast<DbgLabelInst>(II);
1438  assert(DI->getLabel() && "Missing label");
1439  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1440  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1441  return true;
1442  }
1443 
1444  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1445  TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
1446  return true;
1447  }
1448  case Intrinsic::objectsize: {
1449  ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
1450  unsigned long long Res = CI->isZero() ? -1ULL : 0;
1451  Constant *ResCI = ConstantInt::get(II->getType(), Res);
1452  unsigned ResultReg = getRegForValue(ResCI);
1453  if (!ResultReg)
1454  return false;
1455  updateValueMap(II, ResultReg);
1456  return true;
1457  }
1458  case Intrinsic::is_constant: {
1459  Constant *ResCI = ConstantInt::get(II->getType(), 0);
1460  unsigned ResultReg = getRegForValue(ResCI);
1461  if (!ResultReg)
1462  return false;
1463  updateValueMap(II, ResultReg);
1464  return true;
1465  }
1466  case Intrinsic::launder_invariant_group:
1467  case Intrinsic::strip_invariant_group:
1468  case Intrinsic::expect: {
1469  unsigned ResultReg = getRegForValue(II->getArgOperand(0));
1470  if (!ResultReg)
1471  return false;
1472  updateValueMap(II, ResultReg);
1473  return true;
1474  }
1475  case Intrinsic::experimental_stackmap:
1476  return selectStackmap(II);
1477  case Intrinsic::experimental_patchpoint_void:
1478  case Intrinsic::experimental_patchpoint_i64:
1479  return selectPatchpoint(II);
1480 
1481  case Intrinsic::xray_customevent:
1482  return selectXRayCustomEvent(II);
1483  case Intrinsic::xray_typedevent:
1484  return selectXRayTypedEvent(II);
1485  }
1486 
1487  return fastLowerIntrinsicCall(II);
1488 }
1489 
1490 bool FastISel::selectCast(const User *I, unsigned Opcode) {
1491  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1492  EVT DstVT = TLI.getValueType(DL, I->getType());
1493 
1494  if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1495  !DstVT.isSimple())
1496  // Unhandled type. Halt "fast" selection and bail.
1497  return false;
1498 
1499  // Check if the destination type is legal.
1500  if (!TLI.isTypeLegal(DstVT))
1501  return false;
1502 
1503  // Check if the source operand is legal.
1504  if (!TLI.isTypeLegal(SrcVT))
1505  return false;
1506 
1507  unsigned InputReg = getRegForValue(I->getOperand(0));
1508  if (!InputReg)
1509  // Unhandled operand. Halt "fast" selection and bail.
1510  return false;
1511 
1512  bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
1513 
1514  unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1515  Opcode, InputReg, InputRegIsKill);
1516  if (!ResultReg)
1517  return false;
1518 
1519  updateValueMap(I, ResultReg);
1520  return true;
1521 }
1522 
1524  // If the bitcast doesn't change the type, just use the operand value.
1525  if (I->getType() == I->getOperand(0)->getType()) {
1526  unsigned Reg = getRegForValue(I->getOperand(0));
1527  if (!Reg)
1528  return false;
1529  updateValueMap(I, Reg);
1530  return true;
1531  }
1532 
1533  // Bitcasts of other values become reg-reg copies or BITCAST operators.
1534  EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1535  EVT DstEVT = TLI.getValueType(DL, I->getType());
1536  if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1537  !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
1538  // Unhandled type. Halt "fast" selection and bail.
1539  return false;
1540 
1541  MVT SrcVT = SrcEVT.getSimpleVT();
1542  MVT DstVT = DstEVT.getSimpleVT();
1543  unsigned Op0 = getRegForValue(I->getOperand(0));
1544  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1545  return false;
1546  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
1547 
1548  // First, try to perform the bitcast by inserting a reg-reg copy.
1549  unsigned ResultReg = 0;
1550  if (SrcVT == DstVT) {
1551  const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
1552  const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
1553  // Don't attempt a cross-class copy. It will likely fail.
1554  if (SrcClass == DstClass) {
1555  ResultReg = createResultReg(DstClass);
1556  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1557  TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
1558  }
1559  }
1560 
1561  // If the reg-reg copy failed, select a BITCAST opcode.
1562  if (!ResultReg)
1563  ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
1564 
1565  if (!ResultReg)
1566  return false;
1567 
1568  updateValueMap(I, ResultReg);
1569  return true;
1570 }
1571 
1572 // Remove local value instructions starting from the instruction after
1573 // SavedLastLocalValue to the current function insert point.
1574 void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1575 {
1576  MachineInstr *CurLastLocalValue = getLastLocalValue();
1577  if (CurLastLocalValue != SavedLastLocalValue) {
1578  // Find the first local value instruction to be deleted.
1579  // This is the instruction after SavedLastLocalValue if it is non-NULL.
1580  // Otherwise it's the first instruction in the block.
1581  MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1582  if (SavedLastLocalValue)
1583  ++FirstDeadInst;
1584  else
1585  FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1586  setLastLocalValue(SavedLastLocalValue);
1587  removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1588  }
1589 }
1590 
1592  MachineInstr *SavedLastLocalValue = getLastLocalValue();
1593  // Just before the terminator instruction, insert instructions to
1594  // feed PHI nodes in successor blocks.
1595  if (I->isTerminator()) {
1596  if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1597  // PHI node handling may have generated local value instructions,
1598  // even though it failed to handle all PHI nodes.
1599  // We remove these instructions because SelectionDAGISel will generate
1600  // them again.
1601  removeDeadLocalValueCode(SavedLastLocalValue);
1602  return false;
1603  }
1604  }
1605 
1606  // FastISel does not handle any operand bundles except OB_funclet.
1608  for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
1609  if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1610  return false;
1611 
1612  DbgLoc = I->getDebugLoc();
1613 
1614  SavedInsertPt = FuncInfo.InsertPt;
1615 
1616  if (const auto *Call = dyn_cast<CallInst>(I)) {
1617  const Function *F = Call->getCalledFunction();
1618  LibFunc Func;
1619 
1620  // As a special case, don't handle calls to builtin library functions that
1621  // may be translated directly to target instructions.
1622  if (F && !F->hasLocalLinkage() && F->hasName() &&
1623  LibInfo->getLibFunc(F->getName(), Func) &&
1624  LibInfo->hasOptimizedCodeGen(Func))
1625  return false;
1626 
1627  // Don't handle Intrinsic::trap if a trap function is specified.
1628  if (F && F->getIntrinsicID() == Intrinsic::trap &&
1629  Call->hasFnAttr("trap-func-name"))
1630  return false;
1631  }
1632 
1633  // First, try doing target-independent selection.
1634  if (!SkipTargetIndependentISel) {
1635  if (selectOperator(I, I->getOpcode())) {
1636  ++NumFastIselSuccessIndependent;
1637  DbgLoc = DebugLoc();
1638  return true;
1639  }
1640  // Remove dead code.
1641  recomputeInsertPt();
1642  if (SavedInsertPt != FuncInfo.InsertPt)
1643  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1644  SavedInsertPt = FuncInfo.InsertPt;
1645  }
1646  // Next, try calling the target to attempt to handle the instruction.
1647  if (fastSelectInstruction(I)) {
1648  ++NumFastIselSuccessTarget;
1649  DbgLoc = DebugLoc();
1650  return true;
1651  }
1652  // Remove dead code.
1653  recomputeInsertPt();
1654  if (SavedInsertPt != FuncInfo.InsertPt)
1655  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1656 
1657  DbgLoc = DebugLoc();
1658  // Undo phi node updates, because they will be added again by SelectionDAG.
1659  if (I->isTerminator()) {
1660  // PHI node handling may have generated local value instructions.
1661  // We remove them because SelectionDAGISel will generate them again.
1662  removeDeadLocalValueCode(SavedLastLocalValue);
1663  FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
1664  }
1665  return false;
1666 }
1667 
1668 /// Emit an unconditional branch to the given block, unless it is the immediate
1669 /// (fall-through) successor, and update the CFG.
1671  const DebugLoc &DbgLoc) {
1672  if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
1673  FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
1674  // For more accurate line information if this is the only instruction
1675  // in the block then emit it, otherwise we have the unconditional
1676  // fall-through case, which needs no instructions.
1677  } else {
1678  // The unconditional branch case.
1679  TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1680  SmallVector<MachineOperand, 0>(), DbgLoc);
1681  }
1682  if (FuncInfo.BPI) {
1683  auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1684  FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
1685  FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
1686  } else
1687  FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
1688 }
1689 
1691  MachineBasicBlock *TrueMBB,
1692  MachineBasicBlock *FalseMBB) {
1693  // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1694  // happen in degenerate IR and MachineIR forbids to have a block twice in the
1695  // successor/predecessor lists.
1696  if (TrueMBB != FalseMBB) {
1697  if (FuncInfo.BPI) {
1698  auto BranchProbability =
1699  FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1700  FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
1701  } else
1702  FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
1703  }
1704 
1705  fastEmitBranch(FalseMBB, DbgLoc);
1706 }
1707 
1708 /// Emit an FNeg operation.
1710  Value *X;
1711  if (!match(I, m_FNeg(m_Value(X))))
1712  return false;
1713  unsigned OpReg = getRegForValue(X);
1714  if (!OpReg)
1715  return false;
1716  bool OpRegIsKill = hasTrivialKill(I);
1717 
1718  // If the target has ISD::FNEG, use it.
1719  EVT VT = TLI.getValueType(DL, I->getType());
1720  unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1721  OpReg, OpRegIsKill);
1722  if (ResultReg) {
1723  updateValueMap(I, ResultReg);
1724  return true;
1725  }
1726 
1727  // Bitcast the value to integer, twiddle the sign bit with xor,
1728  // and then bitcast it back to floating-point.
1729  if (VT.getSizeInBits() > 64)
1730  return false;
1731  EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1732  if (!TLI.isTypeLegal(IntVT))
1733  return false;
1734 
1735  unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1736  ISD::BITCAST, OpReg, OpRegIsKill);
1737  if (!IntReg)
1738  return false;
1739 
1740  unsigned IntResultReg = fastEmit_ri_(
1741  IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
1742  UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1743  if (!IntResultReg)
1744  return false;
1745 
1746  ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1747  IntResultReg, /*IsKill=*/true);
1748  if (!ResultReg)
1749  return false;
1750 
1751  updateValueMap(I, ResultReg);
1752  return true;
1753 }
1754 
1756  const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1757  if (!EVI)
1758  return false;
1759 
1760  // Make sure we only try to handle extracts with a legal result. But also
1761  // allow i1 because it's easy.
1762  EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1763  if (!RealVT.isSimple())
1764  return false;
1765  MVT VT = RealVT.getSimpleVT();
1766  if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1767  return false;
1768 
1769  const Value *Op0 = EVI->getOperand(0);
1770  Type *AggTy = Op0->getType();
1771 
1772  // Get the base result register.
1773  unsigned ResultReg;
1774  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(Op0);
1775  if (I != FuncInfo.ValueMap.end())
1776  ResultReg = I->second;
1777  else if (isa<Instruction>(Op0))
1778  ResultReg = FuncInfo.InitializeRegForValue(Op0);
1779  else
1780  return false; // fast-isel can't handle aggregate constants at the moment
1781 
1782  // Get the actual result register, which is an offset from the base register.
1783  unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1784 
1785  SmallVector<EVT, 4> AggValueVTs;
1786  ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1787 
1788  for (unsigned i = 0; i < VTIndex; i++)
1789  ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1790 
1791  updateValueMap(EVI, ResultReg);
1792  return true;
1793 }
1794 
1795 bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1796  switch (Opcode) {
1797  case Instruction::Add:
1798  return selectBinaryOp(I, ISD::ADD);
1799  case Instruction::FAdd:
1800  return selectBinaryOp(I, ISD::FADD);
1801  case Instruction::Sub:
1802  return selectBinaryOp(I, ISD::SUB);
1803  case Instruction::FSub:
1804  // FNeg is currently represented in LLVM IR as a special case of FSub.
1805  return selectFNeg(I) || selectBinaryOp(I, ISD::FSUB);
1806  case Instruction::Mul:
1807  return selectBinaryOp(I, ISD::MUL);
1808  case Instruction::FMul:
1809  return selectBinaryOp(I, ISD::FMUL);
1810  case Instruction::SDiv:
1811  return selectBinaryOp(I, ISD::SDIV);
1812  case Instruction::UDiv:
1813  return selectBinaryOp(I, ISD::UDIV);
1814  case Instruction::FDiv:
1815  return selectBinaryOp(I, ISD::FDIV);
1816  case Instruction::SRem:
1817  return selectBinaryOp(I, ISD::SREM);
1818  case Instruction::URem:
1819  return selectBinaryOp(I, ISD::UREM);
1820  case Instruction::FRem:
1821  return selectBinaryOp(I, ISD::FREM);
1822  case Instruction::Shl:
1823  return selectBinaryOp(I, ISD::SHL);
1824  case Instruction::LShr:
1825  return selectBinaryOp(I, ISD::SRL);
1826  case Instruction::AShr:
1827  return selectBinaryOp(I, ISD::SRA);
1828  case Instruction::And:
1829  return selectBinaryOp(I, ISD::AND);
1830  case Instruction::Or:
1831  return selectBinaryOp(I, ISD::OR);
1832  case Instruction::Xor:
1833  return selectBinaryOp(I, ISD::XOR);
1834 
1835  case Instruction::GetElementPtr:
1836  return selectGetElementPtr(I);
1837 
1838  case Instruction::Br: {
1839  const BranchInst *BI = cast<BranchInst>(I);
1840 
1841  if (BI->isUnconditional()) {
1842  const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1843  MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1844  fastEmitBranch(MSucc, BI->getDebugLoc());
1845  return true;
1846  }
1847 
1848  // Conditional branches are not handed yet.
1849  // Halt "fast" selection and bail.
1850  return false;
1851  }
1852 
1853  case Instruction::Unreachable:
1854  if (TM.Options.TrapUnreachable)
1855  return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
1856  else
1857  return true;
1858 
1859  case Instruction::Alloca:
1860  // FunctionLowering has the static-sized case covered.
1861  if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1862  return true;
1863 
1864  // Dynamic-sized alloca is not handled yet.
1865  return false;
1866 
1867  case Instruction::Call:
1868  return selectCall(I);
1869 
1870  case Instruction::BitCast:
1871  return selectBitCast(I);
1872 
1873  case Instruction::FPToSI:
1874  return selectCast(I, ISD::FP_TO_SINT);
1875  case Instruction::ZExt:
1876  return selectCast(I, ISD::ZERO_EXTEND);
1877  case Instruction::SExt:
1878  return selectCast(I, ISD::SIGN_EXTEND);
1879  case Instruction::Trunc:
1880  return selectCast(I, ISD::TRUNCATE);
1881  case Instruction::SIToFP:
1882  return selectCast(I, ISD::SINT_TO_FP);
1883 
1884  case Instruction::IntToPtr: // Deliberate fall-through.
1885  case Instruction::PtrToInt: {
1886  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1887  EVT DstVT = TLI.getValueType(DL, I->getType());
1888  if (DstVT.bitsGT(SrcVT))
1889  return selectCast(I, ISD::ZERO_EXTEND);
1890  if (DstVT.bitsLT(SrcVT))
1891  return selectCast(I, ISD::TRUNCATE);
1892  unsigned Reg = getRegForValue(I->getOperand(0));
1893  if (!Reg)
1894  return false;
1895  updateValueMap(I, Reg);
1896  return true;
1897  }
1898 
1899  case Instruction::ExtractValue:
1900  return selectExtractValue(I);
1901 
1902  case Instruction::PHI:
1903  llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1904 
1905  default:
1906  // Unhandled instruction. Halt "fast" selection and bail.
1907  return false;
1908  }
1909 }
1910 
1912  const TargetLibraryInfo *LibInfo,
1913  bool SkipTargetIndependentISel)
1914  : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1915  MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1916  TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1917  TII(*MF->getSubtarget().getInstrInfo()),
1918  TLI(*MF->getSubtarget().getTargetLowering()),
1919  TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1920  SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1921 
1922 FastISel::~FastISel() = default;
1923 
1924 bool FastISel::fastLowerArguments() { return false; }
1925 
1926 bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1927 
1929  return false;
1930 }
1931 
1932 unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1933 
1934 unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
1935  bool /*Op0IsKill*/) {
1936  return 0;
1937 }
1938 
1939 unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1940  bool /*Op0IsKill*/, unsigned /*Op1*/,
1941  bool /*Op1IsKill*/) {
1942  return 0;
1943 }
1944 
1945 unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1946  return 0;
1947 }
1948 
1949 unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1950  const ConstantFP * /*FPImm*/) {
1951  return 0;
1952 }
1953 
1954 unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1955  bool /*Op0IsKill*/, uint64_t /*Imm*/) {
1956  return 0;
1957 }
1958 
1959 /// This method is a wrapper of fastEmit_ri. It first tries to emit an
1960 /// instruction with an immediate operand using fastEmit_ri.
1961 /// If that fails, it materializes the immediate into a register and try
1962 /// fastEmit_rr instead.
1963 unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1964  bool Op0IsKill, uint64_t Imm, MVT ImmType) {
1965  // If this is a multiply by a power of two, emit this as a shift left.
1966  if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
1967  Opcode = ISD::SHL;
1968  Imm = Log2_64(Imm);
1969  } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
1970  // div x, 8 -> srl x, 3
1971  Opcode = ISD::SRL;
1972  Imm = Log2_64(Imm);
1973  }
1974 
1975  // Horrible hack (to be removed), check to make sure shift amounts are
1976  // in-range.
1977  if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1978  Imm >= VT.getSizeInBits())
1979  return 0;
1980 
1981  // First check if immediate type is legal. If not, we can't use the ri form.
1982  unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
1983  if (ResultReg)
1984  return ResultReg;
1985  unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1986  bool IsImmKill = true;
1987  if (!MaterialReg) {
1988  // This is a bit ugly/slow, but failing here means falling out of
1989  // fast-isel, which would be very slow.
1990  IntegerType *ITy =
1992  MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
1993  if (!MaterialReg)
1994  return 0;
1995  // FIXME: If the materialized register here has no uses yet then this
1996  // will be the first use and we should be able to mark it as killed.
1997  // However, the local value area for materialising constant expressions
1998  // grows down, not up, which means that any constant expressions we generate
1999  // later which also use 'Imm' could be after this instruction and therefore
2000  // after this kill.
2001  IsImmKill = false;
2002  }
2003  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
2004 }
2005 
2007  return MRI.createVirtualRegister(RC);
2008 }
2009 
2011  unsigned OpNum) {
2013  const TargetRegisterClass *RegClass =
2014  TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
2015  if (!MRI.constrainRegClass(Op, RegClass)) {
2016  // If it's not legal to COPY between the register classes, something
2017  // has gone very wrong before we got here.
2018  unsigned NewOp = createResultReg(RegClass);
2020  TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
2021  return NewOp;
2022  }
2023  }
2024  return Op;
2025 }
2026 
2027 unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2028  const TargetRegisterClass *RC) {
2029  unsigned ResultReg = createResultReg(RC);
2030  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2031 
2032  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
2033  return ResultReg;
2034 }
2035 
2036 unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2037  const TargetRegisterClass *RC, unsigned Op0,
2038  bool Op0IsKill) {
2039  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2040 
2041  unsigned ResultReg = createResultReg(RC);
2042  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2043 
2044  if (II.getNumDefs() >= 1)
2045  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2046  .addReg(Op0, getKillRegState(Op0IsKill));
2047  else {
2049  .addReg(Op0, getKillRegState(Op0IsKill));
2051  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2052  }
2053 
2054  return ResultReg;
2055 }
2056 
2057 unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2058  const TargetRegisterClass *RC, unsigned Op0,
2059  bool Op0IsKill, unsigned Op1,
2060  bool Op1IsKill) {
2061  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2062 
2063  unsigned ResultReg = createResultReg(RC);
2064  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2065  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2066 
2067  if (II.getNumDefs() >= 1)
2068  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2069  .addReg(Op0, getKillRegState(Op0IsKill))
2070  .addReg(Op1, getKillRegState(Op1IsKill));
2071  else {
2073  .addReg(Op0, getKillRegState(Op0IsKill))
2074  .addReg(Op1, getKillRegState(Op1IsKill));
2076  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2077  }
2078  return ResultReg;
2079 }
2080 
2081 unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2082  const TargetRegisterClass *RC, unsigned Op0,
2083  bool Op0IsKill, unsigned Op1,
2084  bool Op1IsKill, unsigned Op2,
2085  bool Op2IsKill) {
2086  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2087 
2088  unsigned ResultReg = createResultReg(RC);
2089  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2090  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2091  Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
2092 
2093  if (II.getNumDefs() >= 1)
2094  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2095  .addReg(Op0, getKillRegState(Op0IsKill))
2096  .addReg(Op1, getKillRegState(Op1IsKill))
2097  .addReg(Op2, getKillRegState(Op2IsKill));
2098  else {
2100  .addReg(Op0, getKillRegState(Op0IsKill))
2101  .addReg(Op1, getKillRegState(Op1IsKill))
2102  .addReg(Op2, getKillRegState(Op2IsKill));
2104  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2105  }
2106  return ResultReg;
2107 }
2108 
2109 unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2110  const TargetRegisterClass *RC, unsigned Op0,
2111  bool Op0IsKill, uint64_t Imm) {
2112  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2113 
2114  unsigned ResultReg = createResultReg(RC);
2115  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2116 
2117  if (II.getNumDefs() >= 1)
2118  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2119  .addReg(Op0, getKillRegState(Op0IsKill))
2120  .addImm(Imm);
2121  else {
2123  .addReg(Op0, getKillRegState(Op0IsKill))
2124  .addImm(Imm);
2126  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2127  }
2128  return ResultReg;
2129 }
2130 
2131 unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2132  const TargetRegisterClass *RC, unsigned Op0,
2133  bool Op0IsKill, uint64_t Imm1,
2134  uint64_t Imm2) {
2135  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2136 
2137  unsigned ResultReg = createResultReg(RC);
2138  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2139 
2140  if (II.getNumDefs() >= 1)
2141  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2142  .addReg(Op0, getKillRegState(Op0IsKill))
2143  .addImm(Imm1)
2144  .addImm(Imm2);
2145  else {
2147  .addReg(Op0, getKillRegState(Op0IsKill))
2148  .addImm(Imm1)
2149  .addImm(Imm2);
2151  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2152  }
2153  return ResultReg;
2154 }
2155 
2156 unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2157  const TargetRegisterClass *RC,
2158  const ConstantFP *FPImm) {
2159  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2160 
2161  unsigned ResultReg = createResultReg(RC);
2162 
2163  if (II.getNumDefs() >= 1)
2164  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2165  .addFPImm(FPImm);
2166  else {
2168  .addFPImm(FPImm);
2170  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2171  }
2172  return ResultReg;
2173 }
2174 
2175 unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2176  const TargetRegisterClass *RC, unsigned Op0,
2177  bool Op0IsKill, unsigned Op1,
2178  bool Op1IsKill, uint64_t Imm) {
2179  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2180 
2181  unsigned ResultReg = createResultReg(RC);
2182  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2183  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2184 
2185  if (II.getNumDefs() >= 1)
2186  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2187  .addReg(Op0, getKillRegState(Op0IsKill))
2188  .addReg(Op1, getKillRegState(Op1IsKill))
2189  .addImm(Imm);
2190  else {
2192  .addReg(Op0, getKillRegState(Op0IsKill))
2193  .addReg(Op1, getKillRegState(Op1IsKill))
2194  .addImm(Imm);
2196  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2197  }
2198  return ResultReg;
2199 }
2200 
2201 unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2202  const TargetRegisterClass *RC, uint64_t Imm) {
2203  unsigned ResultReg = createResultReg(RC);
2204  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2205 
2206  if (II.getNumDefs() >= 1)
2207  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2208  .addImm(Imm);
2209  else {
2212  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2213  }
2214  return ResultReg;
2215 }
2216 
2217 unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2218  bool Op0IsKill, uint32_t Idx) {
2219  unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2221  "Cannot yet extract from physregs");
2222  const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2224  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
2225  ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
2226  return ResultReg;
2227 }
2228 
2229 /// Emit MachineInstrs to compute the value of Op with all but the least
2230 /// significant bit set to zero.
2231 unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
2232  return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
2233 }
2234 
2235 /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2236 /// Emit code to ensure constants are copied into registers when needed.
2237 /// Remember the virtual registers that need to be added to the Machine PHI
2238 /// nodes as input. We cannot just directly add them, because expansion
2239 /// might result in multiple MBB's for one BB. As such, the start of the
2240 /// BB might correspond to a different MBB than the end.
2241 bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2242  const Instruction *TI = LLVMBB->getTerminator();
2243 
2246 
2247  // Check successor nodes' PHI nodes that expect a constant to be available
2248  // from this block.
2249  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
2250  const BasicBlock *SuccBB = TI->getSuccessor(succ);
2251  if (!isa<PHINode>(SuccBB->begin()))
2252  continue;
2253  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2254 
2255  // If this terminator has multiple identical successors (common for
2256  // switches), only handle each succ once.
2257  if (!SuccsHandled.insert(SuccMBB).second)
2258  continue;
2259 
2260  MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2261 
2262  // At this point we know that there is a 1-1 correspondence between LLVM PHI
2263  // nodes and Machine PHI nodes, but the incoming operands have not been
2264  // emitted yet.
2265  for (const PHINode &PN : SuccBB->phis()) {
2266  // Ignore dead phi's.
2267  if (PN.use_empty())
2268  continue;
2269 
2270  // Only handle legal types. Two interesting things to note here. First,
2271  // by bailing out early, we may leave behind some dead instructions,
2272  // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2273  // own moves. Second, this check is necessary because FastISel doesn't
2274  // use CreateRegs to create registers, so it always creates
2275  // exactly one register for each non-void instruction.
2276  EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2277  if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2278  // Handle integer promotions, though, because they're common and easy.
2279  if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2281  return false;
2282  }
2283  }
2284 
2285  const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2286 
2287  // Set the DebugLoc for the copy. Prefer the location of the operand
2288  // if there is one; use the location of the PHI otherwise.
2289  DbgLoc = PN.getDebugLoc();
2290  if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2291  DbgLoc = Inst->getDebugLoc();
2292 
2293  unsigned Reg = getRegForValue(PHIOp);
2294  if (!Reg) {
2296  return false;
2297  }
2298  FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2299  DbgLoc = DebugLoc();
2300  }
2301  }
2302 
2303  return true;
2304 }
2305 
2306 bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2307  assert(LI->hasOneUse() &&
2308  "tryToFoldLoad expected a LoadInst with a single use");
2309  // We know that the load has a single use, but don't know what it is. If it
2310  // isn't one of the folded instructions, then we can't succeed here. Handle
2311  // this by scanning the single-use users of the load until we get to FoldInst.
2312  unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2313 
2314  const Instruction *TheUser = LI->user_back();
2315  while (TheUser != FoldInst && // Scan up until we find FoldInst.
2316  // Stay in the right block.
2317  TheUser->getParent() == FoldInst->getParent() &&
2318  --MaxUsers) { // Don't scan too far.
2319  // If there are multiple or no uses of this instruction, then bail out.
2320  if (!TheUser->hasOneUse())
2321  return false;
2322 
2323  TheUser = TheUser->user_back();
2324  }
2325 
2326  // If we didn't find the fold instruction, then we failed to collapse the
2327  // sequence.
2328  if (TheUser != FoldInst)
2329  return false;
2330 
2331  // Don't try to fold volatile loads. Target has to deal with alignment
2332  // constraints.
2333  if (LI->isVolatile())
2334  return false;
2335 
2336  // Figure out which vreg this is going into. If there is no assigned vreg yet
2337  // then there actually was no reference to it. Perhaps the load is referenced
2338  // by a dead instruction.
2339  unsigned LoadReg = getRegForValue(LI);
2340  if (!LoadReg)
2341  return false;
2342 
2343  // We can't fold if this vreg has no uses or more than one use. Multiple uses
2344  // may mean that the instruction got lowered to multiple MIs, or the use of
2345  // the loaded value ended up being multiple operands of the result.
2346  if (!MRI.hasOneUse(LoadReg))
2347  return false;
2348 
2350  MachineInstr *User = RI->getParent();
2351 
2352  // Set the insertion point properly. Folding the load can cause generation of
2353  // other random instructions (like sign extends) for addressing modes; make
2354  // sure they get inserted in a logical place before the new instruction.
2355  FuncInfo.InsertPt = User;
2356  FuncInfo.MBB = User->getParent();
2357 
2358  // Ask the target to try folding the load.
2359  return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2360 }
2361 
2363  // Must be an add.
2364  if (!isa<AddOperator>(Add))
2365  return false;
2366  // Type size needs to match.
2367  if (DL.getTypeSizeInBits(GEP->getType()) !=
2368  DL.getTypeSizeInBits(Add->getType()))
2369  return false;
2370  // Must be in the same basic block.
2371  if (isa<Instruction>(Add) &&
2372  FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2373  return false;
2374  // Must have a constant operand.
2375  return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2376 }
2377 
2380  const Value *Ptr;
2381  Type *ValTy;
2382  unsigned Alignment;
2384  bool IsVolatile;
2385 
2386  if (const auto *LI = dyn_cast<LoadInst>(I)) {
2387  Alignment = LI->getAlignment();
2388  IsVolatile = LI->isVolatile();
2389  Flags = MachineMemOperand::MOLoad;
2390  Ptr = LI->getPointerOperand();
2391  ValTy = LI->getType();
2392  } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2393  Alignment = SI->getAlignment();
2394  IsVolatile = SI->isVolatile();
2396  Ptr = SI->getPointerOperand();
2397  ValTy = SI->getValueOperand()->getType();
2398  } else
2399  return nullptr;
2400 
2401  bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
2402  bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
2403  bool IsDereferenceable =
2405  const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2406 
2407  AAMDNodes AAInfo;
2408  I->getAAMetadata(AAInfo);
2409 
2410  if (Alignment == 0) // Ensure that codegen never sees alignment 0.
2411  Alignment = DL.getABITypeAlignment(ValTy);
2412 
2413  unsigned Size = DL.getTypeStoreSize(ValTy);
2414 
2415  if (IsVolatile)
2417  if (IsNonTemporal)
2419  if (IsDereferenceable)
2421  if (IsInvariant)
2423 
2424  return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
2425  Alignment, AAInfo, Ranges);
2426 }
2427 
2429  // If both operands are the same, then try to optimize or fold the cmp.
2431  if (CI->getOperand(0) != CI->getOperand(1))
2432  return Predicate;
2433 
2434  switch (Predicate) {
2435  default: llvm_unreachable("Invalid predicate!");
2436  case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2437  case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2438  case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2439  case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2440  case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2441  case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2442  case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2443  case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2444  case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2445  case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2446  case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2447  case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2448  case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2449  case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2450  case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2451  case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2452 
2453  case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2454  case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2455  case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2456  case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2457  case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2458  case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2459  case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2460  case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2461  case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2462  case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2463  }
2464 
2465  return Predicate;
2466 }
void setHasStackMap(bool s=true)
IterTy arg_end() const
Definition: CallSite.h:583
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:570
unsigned fastEmitInst_rrr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, unsigned Op2, bool Op2IsKill)
Emit a MachineInstr with three register operands and a result register in the given register class...
Definition: FastISel.cpp:2081
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
void setByValAlign(unsigned A)
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:70
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:636
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
unsigned fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill)
Emit MachineInstrs to compute the value of Op with all but the least significant bit set to zero...
Definition: FastISel.cpp:2231
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
bool hasLocalLinkage() const
Definition: GlobalValue.h:435
This instruction extracts a struct member or array element value from an aggregate value...
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
bool lowerCall(const CallInst *I)
Definition: FastISel.cpp:1240
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
This represents the llvm.dbg.label instruction.
IterTy arg_begin() const
Definition: CallSite.h:579
This class represents lattice values for constants.
Definition: AllocatorList.h:23
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
BasicBlock * getSuccessor(unsigned Idx) const
Return the specified successor. This instruction must be a terminator.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
bool selectXRayTypedEvent(const CallInst *II)
Definition: FastISel.cpp:1060
ImmutableCallSite * CS
Definition: FastISel.h:89
bool selectGetElementPtr(const User *I)
Definition: FastISel.cpp:675
void leaveLocalValueArea(SavePoint Old)
Reset InsertPt to the given old insert position.
Definition: FastISel.cpp:575
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
bool selectStackmap(const CallInst *I)
Definition: FastISel.cpp:790
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:304
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:672
unsigned less than
Definition: InstrTypes.h:671
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:652
MachineMemOperand * createMachineMemOperandFor(const Instruction *I) const
Create a machine mem operand from the given instruction.
Definition: FastISel.cpp:2379
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:705
bool isTerminator() const
Definition: Instruction.h:128
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:662
virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode)
This method is called by target-independent code to request that an instruction with the given type a...
Definition: FastISel.cpp:1932
BasicBlock * getSuccessor(unsigned i) const
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
Metadata node.
Definition: Metadata.h:863
F(f)
SmallVector< unsigned, 4 > InRegs
Definition: FastISel.h:98
An instruction for reading from memory.
Definition: Instructions.h:167
Hexagon Common GEP
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:137
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1945
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:458
void setPhysRegsDeadExcept(ArrayRef< unsigned > UsedRegs, const TargetRegisterInfo &TRI)
Mark every physreg used by this instruction as dead except those in the UsedRegs list.
FunctionType * getFunctionType() const
Definition: CallSite.h:328
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1926
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:264
CallingConv::ID getCallingConv() const
Get the calling convention of the call.
Definition: CallSite.h:320
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:268
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1591
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1155
unsigned fastEmitInst_rii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm1, uint64_t Imm2)
Emit a MachineInstr with one register operand and two immediate operands.
Definition: FastISel.cpp:2131
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1068
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:657
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:47
unsigned fastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm)
Emit a MachineInstr with a register operand, an immediate, and a result register in the given registe...
Definition: FastISel.cpp:2109
ArrayRef< unsigned > getIndices() const
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:656
unsigned fastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm)
Emit a MachineInstr with two register operands, an immediate, and a result register in the given regi...
Definition: FastISel.cpp:2175
CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const
Definition: FastISel.cpp:2428
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:231
A description of a memory reference used in the backend.
const HexagonInstrInfo * TII
TargetLoweringBase::ArgListTy ArgListTy
Definition: FastISel.h:69
Shift and rotation operations.
Definition: ISDOpcodes.h:409
const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
Class to represent struct types.
Definition: DerivedTypes.h:232
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
DILabel * getLabel() const
bool canFoldAddIntoGEP(const User *GEP, const Value *Add)
Check if Add is an add that can be safely folded into GEP.
Definition: FastISel.cpp:2362
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned fastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill)
Emit a MachineInstr with one register operand and a result register in the given register class...
Definition: FastISel.cpp:2036
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:653
This file contains the simple types necessary to represent the attributes associated with functions a...
The memory access is dereferenceable (i.e., doesn&#39;t trap).
static MachineOperand CreateRegMask(const uint32_t *Mask)
CreateRegMask - Creates a register mask operand referencing Mask.
virtual const TargetRegisterClass * getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const
Returns the largest legal sub-class of RC that supports the sub-register index Idx.
void setByValSize(unsigned S)
Type * getType() const
Return the type of the instruction that generated this call site.
Definition: CallSite.h:272
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:668
bool selectIntrinsicCall(const IntrinsicInst *II)
Definition: FastISel.cpp:1322
bool selectCast(const User *I, unsigned Opcode)
Definition: FastISel.cpp:1490
unsigned getSizeInBits() const
Context object for machine code objects.
Definition: MCContext.h:62
InstrTy * getInstruction() const
Definition: CallSite.h:96
Class to represent function types.
Definition: DerivedTypes.h:102
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
SmallVector< ISD::InputArg, 4 > Ins
Definition: FastISel.h:97
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
unsigned constrainOperandRegClass(const MCInstrDesc &II, unsigned Op, unsigned OpNum)
Try to constrain Op so that it is usable by argument OpNum of the provided MCInstrDesc.
Definition: FastISel.cpp:2010
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:477
bool selectOperator(const User *I, unsigned Opcode)
Do "fast" instruction selection for the given LLVM IR operator (Instruction or ConstantExpr), and append generated machine instructions to the current block.
Definition: FastISel.cpp:1795
ArchType getArch() const
getArch - Get the parsed architecture type of this triple.
Definition: Triple.h:289
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
bool selectExtractValue(const User *U)
Definition: FastISel.cpp:1755
unsigned getRegForValue(const Value *V)
Create a virtual register and arrange for it to be assigned the value for the given LLVM value...
Definition: FastISel.cpp:362
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
unsigned fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC)
Emit a MachineInstr with no operands and a result register in the given register class.
Definition: FastISel.cpp:2027
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:234
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:125
bool hasTrivialKill(const Value *V)
Test whether the given value has exactly one use.
Definition: FastISel.cpp:330
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
void setOrigAlign(unsigned A)
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:83
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
unsigned getNumSuccessors() const
Return the number of successors that this instruction has.
Value * getOperand(unsigned i) const
Definition: User.h:169
Class to represent pointers.
Definition: DerivedTypes.h:498
unsigned getKillRegState(bool B)
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:523
unsigned lookUpRegForValue(const Value *V)
Look up the value to see if its value is already cached in a register.
Definition: FastISel.cpp:475
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
void finishBasicBlock()
Flush the local value map and sink local values if possible.
Definition: FastISel.cpp:144
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:228
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
The memory access is volatile.
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc (such as G_OR or G_SDIV)...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:422
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
virtual ~FastISel()
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
unsigned const MachineRegisterInfo * MRI
CallLoweringInfo & setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, ImmutableCallSite &Call)
Definition: FastISel.h:104
Machine Value Type.
Value * getCalledValue() const
Definition: InstrTypes.h:1194
bool hasName() const
Definition: Value.h:250
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
const MachineInstrBuilder & addCImm(const ConstantInt *Val) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Simple binary floating point operators.
Definition: ISDOpcodes.h:282
Conditional or Unconditional Branch instruction.
Value * getAddress() const
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:544
Value * getValue() const
SmallVector< ISD::ArgFlagsTy, 16 > OutFlags
Definition: FastISel.h:95
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
const MCPhysReg * ImplicitDefs
Definition: MCInstrDesc.h:173
DIExpression * getExpression() const
virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1934
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:433
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:160
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We&#39;re checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2306
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:146
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:646
std::pair< unsigned, bool > getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
Definition: FastISel.cpp:507
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned char TargetFlags=0)
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:771
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:655
self_iterator getIterator()
Definition: ilist_node.h:81
The memory access is non-temporal.
Class to represent integer types.
Definition: DerivedTypes.h:39
bool selectXRayCustomEvent(const CallInst *II)
Definition: FastISel.cpp:1041
const TargetRegisterInfo & TRI
Definition: FastISel.h:213
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:663
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:192
static unsigned findSinkableLocalRegDef(MachineInstr &MI)
Return the defined register if this instruction defines exactly one virtual register and uses no othe...
Definition: FastISel.cpp:168
Extended Value Type.
Definition: ValueTypes.h:33
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:529
bool selectFNeg(const User *I)
Emit an FNeg operation.
Definition: FastISel.cpp:1709
size_t size() const
Definition: SmallVector.h:52
This class contains a discriminated union of information about pointers in memory operands...
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:661
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
SmallVector< Value *, 16 > OutVals
Definition: FastISel.h:94
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call...
Definition: FastISel.cpp:1083
const TargetInstrInfo & TII
Definition: FastISel.h:211
MachineBasicBlock * MBB
MBB - The current block.
bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:470
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:43
signed greater than
Definition: InstrTypes.h:673
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:676
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:530
The memory access writes data.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:50
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:650
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:239
virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1939
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1954
MachineOperand class - Representation of each machine instruction operand.
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:660
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:839
bool selectCall(const User *I)
Definition: FastISel.cpp:1279
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:63
Provides information about what library functions are available for the current target.
unsigned arg_size() const
Definition: CallSite.h:226
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB)
Emit an unconditional branch to FalseMBB, obtains the branch weight and adds TrueMBB and FalseMBB to ...
Definition: FastISel.cpp:1690
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:729
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:575
signed less than
Definition: InstrTypes.h:675
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:643
reg_iterator reg_begin(unsigned RegNo) const
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
unsigned fastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
Emit a MachineInstr with two register operands and a result register in the given register class...
Definition: FastISel.cpp:2057
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:621
void updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs=1)
Update the value map to include the new mapping for this instruction, or insert an extra copy to get ...
Definition: FastISel.cpp:486
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:225
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
Definition: FastISel.cpp:130
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
signed less or equal
Definition: InstrTypes.h:676
bool selectBitCast(const User *I)
Definition: FastISel.cpp:1523
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Target - Wrapper for Target specific information.
virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1949
SmallVector< unsigned, 16 > OutRegs
Definition: FastISel.h:96
const DataLayout & DL
Definition: FastISel.h:210
bool selectBinaryOp(const User *I, unsigned ISDOpcode)
Select and emit code for a binary operator instruction, which has an opcode which directly correspond...
Definition: FastISel.cpp:584
void setDebugLoc(DebugLoc dl)
Replace current source information with new such.
amdgpu Simplify well known AMD library false FunctionCallee Callee
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringRef > StandardNames)
Initialize the set of available library functions based on the specified target triple.
This file defines the FastISel class.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:240
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:467
bool isTailCall() const
DebugLoc DbgLoc
Definition: FastISel.h:208
Flags
Flags values. These may be or&#39;d together.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
The memory access reads data.
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:567
SavePoint enterLocalValueArea()
Prepare InsertPt to begin inserting instructions into the local value area and return the old insert ...
Definition: FastISel.cpp:566
Representation of each machine instruction.
Definition: MachineInstr.h:63
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:721
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering...
Definition: FastISel.cpp:1928
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match &#39;fneg X&#39; as &#39;fsub -0.0, X&#39;.
Definition: PatternMatch.h:688
bool selectPatchpoint(const CallInst *I)
Definition: FastISel.cpp:903
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1153
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:324
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:386
MachineRegisterInfo & MRI
Definition: FastISel.h:205
bool hasOneUse(unsigned RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:122
unsigned greater or equal
Definition: InstrTypes.h:670
This represents the llvm.dbg.value instruction.
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1258
bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs)
Definition: FastISel.cpp:1105
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:44
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
Establish a view to a call site for examination.
Definition: CallSite.h:892
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1201
unsigned fastEmitInst_i(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm)
Emit a MachineInstr with a single immediate operand, and a result register in the given register clas...
Definition: FastISel.cpp:2201
static MachineOperand CreateImm(int64_t Val)
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call&#39;s live variable operands to a stackmap or patchpoint target node&#39;s ope...
The memory access always returns the same value (or traps).
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:192
static bool isRegUsedByPhiNodes(unsigned DefReg, FunctionLoweringInfo &FuncInfo)
Definition: FastISel.cpp:220
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:654
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:322
uint32_t Size
Definition: Profile.cpp:46
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:324
DILocalVariable * getVariable() const
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool isUnconditional() const
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:658
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:308
const TargetLowering & TLI
Definition: FastISel.h:212
unsigned createResultReg(const TargetRegisterClass *RC)
Definition: FastISel.cpp:2006
CallLoweringInfo & setIsPatchPoint(bool Value=true)
Definition: FastISel.h:182
unsigned fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm, MVT ImmType)
This method is a wrapper of fastEmit_ri.
Definition: FastISel.cpp:1963
unsigned fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, bool Op0IsKill, uint32_t Idx)
Emit a MachineInstr for an extract_subreg from a specified index of a superregister to a specified ty...
Definition: FastISel.cpp:2217
MachineBasicBlock::iterator InsertPt
Definition: FastISel.h:318
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
Definition: Value.h:375
FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, bool SkipTargetIndependentISel=false)
Definition: FastISel.cpp:1911
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1924
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:649
LLVM Value Representation.
Definition: Value.h:72
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:659
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:418
static const Function * getParent(const Value *V)
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable&#39;s name.
Definition: Mangler.cpp:111
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
IRTranslator LLVM IR MI
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:412
void fastEmitBranch(MachineBasicBlock *MSucc, const DebugLoc &DbgLoc)
Emit an unconditional branch to the given block, unless it is the immediate (fall-through) successor...
Definition: FastISel.cpp:1670
unsigned greater than
Definition: InstrTypes.h:669
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
static cl::opt< bool > SinkLocalValues("fast-isel-sink-local-values", cl::init(true), cl::Hidden, cl::desc("Sink local values in FastISel"))
unsigned fastEmitInst_f(unsigned MachineInstOpcode, const TargetRegisterClass *RC, const ConstantFP *FPImm)
Emit a MachineInstr with a floating point immediate, and a result register in the given register clas...
Definition: FastISel.cpp:2156
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty...
Definition: Type.cpp:97
Conversion operators.
Definition: ISDOpcodes.h:464
FunctionLoweringInfo & FuncInfo
Definition: FastISel.h:203
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
Definition: Value.cpp:537
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:473
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:125
#define LLVM_DEBUG(X)
Definition: Debug.h:122
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:651
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:35
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
This represents the llvm.dbg.declare instruction.
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:585
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:63
static MachineOperand CreateFI(int Idx)
bool use_empty() const
Definition: Value.h:322
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:544
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Type * getElementType() const
Definition: DerivedTypes.h:517
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:872
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:648
signed greater or equal
Definition: InstrTypes.h:674
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:43
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
gep_type_iterator gep_type_begin(const User *GEP)