LLVM  10.0.0svn
FastISel.cpp
Go to the documentation of this file.
1 //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the implementation of the FastISel class.
10 //
11 // "Fast" instruction selection is designed to emit very poor code quickly.
12 // Also, it is not designed to be able to do much lowering, so most illegal
13 // types (e.g. i64 on 32-bit targets) and operations are not supported. It is
14 // also not intended to be able to do much optimization, except in a few cases
15 // where doing optimizations reduces overall compile time. For example, folding
16 // constants into immediate fields is often done, because it's cheap and it
17 // reduces the number of instructions later phases have to examine.
18 //
19 // "Fast" instruction selection is able to fail gracefully and transfer
20 // control to the SelectionDAG selector for operations that it doesn't
21 // support. In many cases, this allows us to avoid duplicating a lot of
22 // the complicated lowering logic that SelectionDAG currently has.
23 //
24 // The intended use for "fast" instruction selection is "-O0" mode
25 // compilation, where the quality of the generated code is irrelevant when
26 // weighed against the speed at which the code can be generated. Also,
27 // at -O0, the LLVM optimizers are not running, and this makes the
28 // compile time of codegen a much higher portion of the overall compile
29 // time. Despite its limitations, "fast" instruction selection is able to
30 // handle enough code on its own to provide noticeable overall speedups
31 // in -O0 compiles.
32 //
33 // Basic operations are supported in a target-independent way, by reading
34 // the same instruction descriptions that the SelectionDAG selector reads,
35 // and identifying simple arithmetic operations that can be directly selected
36 // from simple operators. More complicated operations currently require
37 // target-specific code.
38 //
39 //===----------------------------------------------------------------------===//
40 
41 #include "llvm/CodeGen/FastISel.h"
42 #include "llvm/ADT/APFloat.h"
43 #include "llvm/ADT/APSInt.h"
44 #include "llvm/ADT/DenseMap.h"
45 #include "llvm/ADT/Optional.h"
46 #include "llvm/ADT/SmallPtrSet.h"
47 #include "llvm/ADT/SmallString.h"
48 #include "llvm/ADT/SmallVector.h"
49 #include "llvm/ADT/Statistic.h"
52 #include "llvm/CodeGen/Analysis.h"
63 #include "llvm/CodeGen/StackMaps.h"
68 #include "llvm/IR/Argument.h"
69 #include "llvm/IR/Attributes.h"
70 #include "llvm/IR/BasicBlock.h"
71 #include "llvm/IR/CallSite.h"
72 #include "llvm/IR/CallingConv.h"
73 #include "llvm/IR/Constant.h"
74 #include "llvm/IR/Constants.h"
75 #include "llvm/IR/DataLayout.h"
76 #include "llvm/IR/DebugInfo.h"
77 #include "llvm/IR/DebugLoc.h"
78 #include "llvm/IR/DerivedTypes.h"
79 #include "llvm/IR/Function.h"
81 #include "llvm/IR/GlobalValue.h"
82 #include "llvm/IR/InlineAsm.h"
83 #include "llvm/IR/InstrTypes.h"
84 #include "llvm/IR/Instruction.h"
85 #include "llvm/IR/Instructions.h"
86 #include "llvm/IR/IntrinsicInst.h"
87 #include "llvm/IR/LLVMContext.h"
88 #include "llvm/IR/Mangler.h"
89 #include "llvm/IR/Metadata.h"
90 #include "llvm/IR/Operator.h"
91 #include "llvm/IR/PatternMatch.h"
92 #include "llvm/IR/Type.h"
93 #include "llvm/IR/User.h"
94 #include "llvm/IR/Value.h"
95 #include "llvm/MC/MCContext.h"
96 #include "llvm/MC/MCInstrDesc.h"
97 #include "llvm/MC/MCRegisterInfo.h"
98 #include "llvm/Support/Casting.h"
99 #include "llvm/Support/Debug.h"
102 #include "llvm/Support/MathExtras.h"
106 #include <algorithm>
107 #include <cassert>
108 #include <cstdint>
109 #include <iterator>
110 #include <utility>
111 
112 using namespace llvm;
113 using namespace PatternMatch;
114 
115 #define DEBUG_TYPE "isel"
116 
117 // FIXME: Remove this after the feature has proven reliable.
118 static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values",
119  cl::init(true), cl::Hidden,
120  cl::desc("Sink local values in FastISel"));
121 
122 STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
123  "target-independent selector");
124 STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
125  "target-specific selector");
126 STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
127 
128 /// Set the current block to which generated machine instructions will be
129 /// appended.
131  assert(LocalValueMap.empty() &&
132  "local values should be cleared after finishing a BB");
133 
134  // Instructions are appended to FuncInfo.MBB. If the basic block already
135  // contains labels or copies, use the last instruction as the last local
136  // value.
137  EmitStartPt = nullptr;
138  if (!FuncInfo.MBB->empty())
139  EmitStartPt = &FuncInfo.MBB->back();
140  LastLocalValue = EmitStartPt;
141 }
142 
143 /// Flush the local CSE map and sink anything we can.
144 void FastISel::finishBasicBlock() { flushLocalValueMap(); }
145 
147  if (!FuncInfo.CanLowerReturn)
148  // Fallback to SDISel argument lowering code to deal with sret pointer
149  // parameter.
150  return false;
151 
152  if (!fastLowerArguments())
153  return false;
154 
155  // Enter arguments into ValueMap for uses in non-entry BBs.
156  for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
157  E = FuncInfo.Fn->arg_end();
158  I != E; ++I) {
159  DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(&*I);
160  assert(VI != LocalValueMap.end() && "Missed an argument?");
161  FuncInfo.ValueMap[&*I] = VI->second;
162  }
163  return true;
164 }
165 
166 /// Return the defined register if this instruction defines exactly one
167 /// virtual register and uses no other virtual registers. Otherwise return 0.
169  unsigned RegDef = 0;
170  for (const MachineOperand &MO : MI.operands()) {
171  if (!MO.isReg())
172  continue;
173  if (MO.isDef()) {
174  if (RegDef)
175  return 0;
176  RegDef = MO.getReg();
177  } else if (Register::isVirtualRegister(MO.getReg())) {
178  // This is another use of a vreg. Don't try to sink it.
179  return 0;
180  }
181  }
182  return RegDef;
183 }
184 
185 void FastISel::flushLocalValueMap() {
186  // Try to sink local values down to their first use so that we can give them a
187  // better debug location. This has the side effect of shrinking local value
188  // live ranges, which helps out fast regalloc.
189  if (SinkLocalValues && LastLocalValue != EmitStartPt) {
190  // Sink local value materialization instructions between EmitStartPt and
191  // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
192  // avoid inserting into the range that we're iterating over.
194  EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
195  : FuncInfo.MBB->rend();
196  MachineBasicBlock::reverse_iterator RI(LastLocalValue);
197 
198  InstOrderMap OrderMap;
199  for (; RI != RE;) {
200  MachineInstr &LocalMI = *RI;
201  ++RI;
202  bool Store = true;
203  if (!LocalMI.isSafeToMove(nullptr, Store))
204  continue;
205  unsigned DefReg = findSinkableLocalRegDef(LocalMI);
206  if (DefReg == 0)
207  continue;
208 
209  sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
210  }
211  }
212 
213  LocalValueMap.clear();
214  LastLocalValue = EmitStartPt;
215  recomputeInsertPt();
216  SavedInsertPt = FuncInfo.InsertPt;
217  LastFlushPoint = FuncInfo.InsertPt;
218 }
219 
220 static bool isRegUsedByPhiNodes(unsigned DefReg,
221  FunctionLoweringInfo &FuncInfo) {
222  for (auto &P : FuncInfo.PHINodesToUpdate)
223  if (P.second == DefReg)
224  return true;
225  return false;
226 }
227 
228 /// Build a map of instruction orders. Return the first terminator and its
229 /// order. Consider EH_LABEL instructions to be terminators as well, since local
230 /// values for phis after invokes must be materialized before the call.
232  MachineBasicBlock *MBB, MachineBasicBlock::iterator LastFlushPoint) {
233  unsigned Order = 0;
234  for (MachineInstr &I : *MBB) {
235  if (!FirstTerminator &&
236  (I.isTerminator() || (I.isEHLabel() && &I != &MBB->front()))) {
237  FirstTerminator = &I;
238  FirstTerminatorOrder = Order;
239  }
240  Orders[&I] = Order++;
241 
242  // We don't need to order instructions past the last flush point.
243  if (I.getIterator() == LastFlushPoint)
244  break;
245  }
246 }
247 
248 void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
249  unsigned DefReg,
250  InstOrderMap &OrderMap) {
251  // If this register is used by a register fixup, MRI will not contain all
252  // the uses until after register fixups, so don't attempt to sink or DCE
253  // this instruction. Register fixups typically come from no-op cast
254  // instructions, which replace the cast instruction vreg with the local
255  // value vreg.
256  if (FuncInfo.RegsWithFixups.count(DefReg))
257  return;
258 
259  // We can DCE this instruction if there are no uses and it wasn't a
260  // materialized for a successor PHI node.
261  bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
262  if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
263  if (EmitStartPt == &LocalMI)
264  EmitStartPt = EmitStartPt->getPrevNode();
265  LLVM_DEBUG(dbgs() << "removing dead local value materialization "
266  << LocalMI);
267  OrderMap.Orders.erase(&LocalMI);
268  LocalMI.eraseFromParent();
269  return;
270  }
271 
272  // Number the instructions if we haven't yet so we can efficiently find the
273  // earliest use.
274  if (OrderMap.Orders.empty())
275  OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
276 
277  // Find the first user in the BB.
278  MachineInstr *FirstUser = nullptr;
279  unsigned FirstOrder = std::numeric_limits<unsigned>::max();
280  for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
281  auto I = OrderMap.Orders.find(&UseInst);
282  assert(I != OrderMap.Orders.end() &&
283  "local value used by instruction outside local region");
284  unsigned UseOrder = I->second;
285  if (UseOrder < FirstOrder) {
286  FirstOrder = UseOrder;
287  FirstUser = &UseInst;
288  }
289  }
290 
291  // The insertion point will be the first terminator or the first user,
292  // whichever came first. If there was no terminator, this must be a
293  // fallthrough block and the insertion point is the end of the block.
295  if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder) {
296  FirstOrder = OrderMap.FirstTerminatorOrder;
297  SinkPos = OrderMap.FirstTerminator->getIterator();
298  } else if (FirstUser) {
299  SinkPos = FirstUser->getIterator();
300  } else {
301  assert(UsedByPHI && "must be users if not used by a phi");
302  SinkPos = FuncInfo.MBB->instr_end();
303  }
304 
305  // Collect all DBG_VALUEs before the new insertion position so that we can
306  // sink them.
308  for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
309  if (!DbgVal.isDebugValue())
310  continue;
311  unsigned UseOrder = OrderMap.Orders[&DbgVal];
312  if (UseOrder < FirstOrder)
313  DbgValues.push_back(&DbgVal);
314  }
315 
316  // Sink LocalMI before SinkPos and assign it the same DebugLoc.
317  LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
318  FuncInfo.MBB->remove(&LocalMI);
319  FuncInfo.MBB->insert(SinkPos, &LocalMI);
320  if (SinkPos != FuncInfo.MBB->end())
321  LocalMI.setDebugLoc(SinkPos->getDebugLoc());
322 
323  // Sink any debug values that we've collected.
324  for (MachineInstr *DI : DbgValues) {
325  FuncInfo.MBB->remove(DI);
326  FuncInfo.MBB->insert(SinkPos, DI);
327  }
328 }
329 
331  // Don't consider constants or arguments to have trivial kills.
332  const Instruction *I = dyn_cast<Instruction>(V);
333  if (!I)
334  return false;
335 
336  // No-op casts are trivially coalesced by fast-isel.
337  if (const auto *Cast = dyn_cast<CastInst>(I))
338  if (Cast->isNoopCast(DL) && !hasTrivialKill(Cast->getOperand(0)))
339  return false;
340 
341  // Even the value might have only one use in the LLVM IR, it is possible that
342  // FastISel might fold the use into another instruction and now there is more
343  // than one use at the Machine Instruction level.
344  unsigned Reg = lookUpRegForValue(V);
345  if (Reg && !MRI.use_empty(Reg))
346  return false;
347 
348  // GEPs with all zero indices are trivially coalesced by fast-isel.
349  if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
350  if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0)))
351  return false;
352 
353  // Only instructions with a single use in the same basic block are considered
354  // to have trivial kills.
355  return I->hasOneUse() &&
356  !(I->getOpcode() == Instruction::BitCast ||
357  I->getOpcode() == Instruction::PtrToInt ||
358  I->getOpcode() == Instruction::IntToPtr) &&
359  cast<Instruction>(*I->user_begin())->getParent() == I->getParent();
360 }
361 
362 unsigned FastISel::getRegForValue(const Value *V) {
363  EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
364  // Don't handle non-simple values in FastISel.
365  if (!RealVT.isSimple())
366  return 0;
367 
368  // Ignore illegal types. We must do this before looking up the value
369  // in ValueMap because Arguments are given virtual registers regardless
370  // of whether FastISel can handle them.
371  MVT VT = RealVT.getSimpleVT();
372  if (!TLI.isTypeLegal(VT)) {
373  // Handle integer promotions, though, because they're common and easy.
374  if (VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)
375  VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
376  else
377  return 0;
378  }
379 
380  // Look up the value to see if we already have a register for it.
381  unsigned Reg = lookUpRegForValue(V);
382  if (Reg)
383  return Reg;
384 
385  // In bottom-up mode, just create the virtual register which will be used
386  // to hold the value. It will be materialized later.
387  if (isa<Instruction>(V) &&
388  (!isa<AllocaInst>(V) ||
389  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))))
390  return FuncInfo.InitializeRegForValue(V);
391 
392  SavePoint SaveInsertPt = enterLocalValueArea();
393 
394  // Materialize the value in a register. Emit any instructions in the
395  // local value area.
396  Reg = materializeRegForValue(V, VT);
397 
398  leaveLocalValueArea(SaveInsertPt);
399 
400  return Reg;
401 }
402 
403 unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
404  unsigned Reg = 0;
405  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
406  if (CI->getValue().getActiveBits() <= 64)
407  Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
408  } else if (isa<AllocaInst>(V))
409  Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
410  else if (isa<ConstantPointerNull>(V))
411  // Translate this as an integer zero so that it can be
412  // local-CSE'd with actual integer zeros.
413  Reg = getRegForValue(
414  Constant::getNullValue(DL.getIntPtrType(V->getContext())));
415  else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
416  if (CF->isNullValue())
417  Reg = fastMaterializeFloatZero(CF);
418  else
419  // Try to emit the constant directly.
420  Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
421 
422  if (!Reg) {
423  // Try to emit the constant by using an integer constant with a cast.
424  const APFloat &Flt = CF->getValueAPF();
425  EVT IntVT = TLI.getPointerTy(DL);
426  uint32_t IntBitWidth = IntVT.getSizeInBits();
427  APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
428  bool isExact;
429  (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
430  if (isExact) {
431  unsigned IntegerReg =
432  getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
433  if (IntegerReg != 0)
434  Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
435  /*Kill=*/false);
436  }
437  }
438  } else if (const auto *Op = dyn_cast<Operator>(V)) {
439  if (!selectOperator(Op, Op->getOpcode()))
440  if (!isa<Instruction>(Op) ||
441  !fastSelectInstruction(cast<Instruction>(Op)))
442  return 0;
443  Reg = lookUpRegForValue(Op);
444  } else if (isa<UndefValue>(V)) {
445  Reg = createResultReg(TLI.getRegClassFor(VT));
446  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
447  TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
448  }
449  return Reg;
450 }
451 
452 /// Helper for getRegForValue. This function is called when the value isn't
453 /// already available in a register and must be materialized with new
454 /// instructions.
455 unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
456  unsigned Reg = 0;
457  // Give the target-specific code a try first.
458  if (isa<Constant>(V))
459  Reg = fastMaterializeConstant(cast<Constant>(V));
460 
461  // If target-specific code couldn't or didn't want to handle the value, then
462  // give target-independent code a try.
463  if (!Reg)
464  Reg = materializeConstant(V, VT);
465 
466  // Don't cache constant materializations in the general ValueMap.
467  // To do so would require tracking what uses they dominate.
468  if (Reg) {
469  LocalValueMap[V] = Reg;
470  LastLocalValue = MRI.getVRegDef(Reg);
471  }
472  return Reg;
473 }
474 
475 unsigned FastISel::lookUpRegForValue(const Value *V) {
476  // Look up the value to see if we already have a register for it. We
477  // cache values defined by Instructions across blocks, and other values
478  // only locally. This is because Instructions already have the SSA
479  // def-dominates-use requirement enforced.
480  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(V);
481  if (I != FuncInfo.ValueMap.end())
482  return I->second;
483  return LocalValueMap[V];
484 }
485 
486 void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
487  if (!isa<Instruction>(I)) {
488  LocalValueMap[I] = Reg;
489  return;
490  }
491 
492  unsigned &AssignedReg = FuncInfo.ValueMap[I];
493  if (AssignedReg == 0)
494  // Use the new register.
495  AssignedReg = Reg;
496  else if (Reg != AssignedReg) {
497  // Arrange for uses of AssignedReg to be replaced by uses of Reg.
498  for (unsigned i = 0; i < NumRegs; i++) {
499  FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
500  FuncInfo.RegsWithFixups.insert(Reg + i);
501  }
502 
503  AssignedReg = Reg;
504  }
505 }
506 
507 std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
508  unsigned IdxN = getRegForValue(Idx);
509  if (IdxN == 0)
510  // Unhandled operand. Halt "fast" selection and bail.
511  return std::pair<unsigned, bool>(0, false);
512 
513  bool IdxNIsKill = hasTrivialKill(Idx);
514 
515  // If the index is smaller or larger than intptr_t, truncate or extend it.
516  MVT PtrVT = TLI.getPointerTy(DL);
517  EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
518  if (IdxVT.bitsLT(PtrVT)) {
519  IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
520  IdxNIsKill);
521  IdxNIsKill = true;
522  } else if (IdxVT.bitsGT(PtrVT)) {
523  IdxN =
524  fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
525  IdxNIsKill = true;
526  }
527  return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
528 }
529 
531  if (getLastLocalValue()) {
532  FuncInfo.InsertPt = getLastLocalValue();
533  FuncInfo.MBB = FuncInfo.InsertPt->getParent();
534  ++FuncInfo.InsertPt;
535  } else
536  FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
537 
538  // Now skip past any EH_LABELs, which must remain at the beginning.
539  while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
540  FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL)
541  ++FuncInfo.InsertPt;
542 }
543 
546  assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
547  "Invalid iterator!");
548  while (I != E) {
549  if (LastFlushPoint == I)
550  LastFlushPoint = E;
551  if (SavedInsertPt == I)
552  SavedInsertPt = E;
553  if (EmitStartPt == I)
554  EmitStartPt = E.isValid() ? &*E : nullptr;
555  if (LastLocalValue == I)
556  LastLocalValue = E.isValid() ? &*E : nullptr;
557 
558  MachineInstr *Dead = &*I;
559  ++I;
560  Dead->eraseFromParent();
561  ++NumFastIselDead;
562  }
563  recomputeInsertPt();
564 }
565 
567  MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt;
568  DebugLoc OldDL = DbgLoc;
569  recomputeInsertPt();
570  DbgLoc = DebugLoc();
571  SavePoint SP = {OldInsertPt, OldDL};
572  return SP;
573 }
574 
576  if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
577  LastLocalValue = &*std::prev(FuncInfo.InsertPt);
578 
579  // Restore the previous insert position.
580  FuncInfo.InsertPt = OldInsertPt.InsertPt;
581  DbgLoc = OldInsertPt.DL;
582 }
583 
584 bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
585  EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
586  if (VT == MVT::Other || !VT.isSimple())
587  // Unhandled type. Halt "fast" selection and bail.
588  return false;
589 
590  // We only handle legal types. For example, on x86-32 the instruction
591  // selector contains all of the 64-bit instructions from x86-64,
592  // under the assumption that i64 won't be used if the target doesn't
593  // support it.
594  if (!TLI.isTypeLegal(VT)) {
595  // MVT::i1 is special. Allow AND, OR, or XOR because they
596  // don't require additional zeroing, which makes them easy.
597  if (VT == MVT::i1 && (ISDOpcode == ISD::AND || ISDOpcode == ISD::OR ||
598  ISDOpcode == ISD::XOR))
599  VT = TLI.getTypeToTransformTo(I->getContext(), VT);
600  else
601  return false;
602  }
603 
604  // Check if the first operand is a constant, and handle it as "ri". At -O0,
605  // we don't have anything that canonicalizes operand order.
606  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
607  if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
608  unsigned Op1 = getRegForValue(I->getOperand(1));
609  if (!Op1)
610  return false;
611  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
612 
613  unsigned ResultReg =
614  fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
615  CI->getZExtValue(), VT.getSimpleVT());
616  if (!ResultReg)
617  return false;
618 
619  // We successfully emitted code for the given LLVM Instruction.
620  updateValueMap(I, ResultReg);
621  return true;
622  }
623 
624  unsigned Op0 = getRegForValue(I->getOperand(0));
625  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
626  return false;
627  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
628 
629  // Check if the second operand is a constant and handle it appropriately.
630  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
631  uint64_t Imm = CI->getSExtValue();
632 
633  // Transform "sdiv exact X, 8" -> "sra X, 3".
634  if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I) &&
635  cast<BinaryOperator>(I)->isExact() && isPowerOf2_64(Imm)) {
636  Imm = Log2_64(Imm);
637  ISDOpcode = ISD::SRA;
638  }
639 
640  // Transform "urem x, pow2" -> "and x, pow2-1".
641  if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I) &&
642  isPowerOf2_64(Imm)) {
643  --Imm;
644  ISDOpcode = ISD::AND;
645  }
646 
647  unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
648  Op0IsKill, Imm, VT.getSimpleVT());
649  if (!ResultReg)
650  return false;
651 
652  // We successfully emitted code for the given LLVM Instruction.
653  updateValueMap(I, ResultReg);
654  return true;
655  }
656 
657  unsigned Op1 = getRegForValue(I->getOperand(1));
658  if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
659  return false;
660  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
661 
662  // Now we have both operands in registers. Emit the instruction.
663  unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
664  ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
665  if (!ResultReg)
666  // Target-specific code wasn't able to find a machine opcode for
667  // the given ISD opcode and type. Halt "fast" selection and bail.
668  return false;
669 
670  // We successfully emitted code for the given LLVM Instruction.
671  updateValueMap(I, ResultReg);
672  return true;
673 }
674 
676  unsigned N = getRegForValue(I->getOperand(0));
677  if (!N) // Unhandled operand. Halt "fast" selection and bail.
678  return false;
679  bool NIsKill = hasTrivialKill(I->getOperand(0));
680 
681  // Keep a running tab of the total offset to coalesce multiple N = N + Offset
682  // into a single N = N + TotalOffset.
683  uint64_t TotalOffs = 0;
684  // FIXME: What's a good SWAG number for MaxOffs?
685  uint64_t MaxOffs = 2048;
686  MVT VT = TLI.getPointerTy(DL);
687  for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
688  GTI != E; ++GTI) {
689  const Value *Idx = GTI.getOperand();
690  if (StructType *StTy = GTI.getStructTypeOrNull()) {
691  uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
692  if (Field) {
693  // N = N + Offset
694  TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
695  if (TotalOffs >= MaxOffs) {
696  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
697  if (!N) // Unhandled operand. Halt "fast" selection and bail.
698  return false;
699  NIsKill = true;
700  TotalOffs = 0;
701  }
702  }
703  } else {
704  Type *Ty = GTI.getIndexedType();
705 
706  // If this is a constant subscript, handle it quickly.
707  if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
708  if (CI->isZero())
709  continue;
710  // N = N + Offset
711  uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
712  TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
713  if (TotalOffs >= MaxOffs) {
714  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
715  if (!N) // Unhandled operand. Halt "fast" selection and bail.
716  return false;
717  NIsKill = true;
718  TotalOffs = 0;
719  }
720  continue;
721  }
722  if (TotalOffs) {
723  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
724  if (!N) // Unhandled operand. Halt "fast" selection and bail.
725  return false;
726  NIsKill = true;
727  TotalOffs = 0;
728  }
729 
730  // N = N + Idx * ElementSize;
731  uint64_t ElementSize = DL.getTypeAllocSize(Ty);
732  std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
733  unsigned IdxN = Pair.first;
734  bool IdxNIsKill = Pair.second;
735  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
736  return false;
737 
738  if (ElementSize != 1) {
739  IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
740  if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
741  return false;
742  IdxNIsKill = true;
743  }
744  N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
745  if (!N) // Unhandled operand. Halt "fast" selection and bail.
746  return false;
747  }
748  }
749  if (TotalOffs) {
750  N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
751  if (!N) // Unhandled operand. Halt "fast" selection and bail.
752  return false;
753  }
754 
755  // We successfully emitted code for the given LLVM Instruction.
756  updateValueMap(I, N);
757  return true;
758 }
759 
760 bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
761  const CallInst *CI, unsigned StartIdx) {
762  for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) {
763  Value *Val = CI->getArgOperand(i);
764  // Check for constants and encode them with a StackMaps::ConstantOp prefix.
765  if (const auto *C = dyn_cast<ConstantInt>(Val)) {
766  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
767  Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
768  } else if (isa<ConstantPointerNull>(Val)) {
769  Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
771  } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
772  // Values coming from a stack location also require a special encoding,
773  // but that is added later on by the target specific frame index
774  // elimination implementation.
775  auto SI = FuncInfo.StaticAllocaMap.find(AI);
776  if (SI != FuncInfo.StaticAllocaMap.end())
777  Ops.push_back(MachineOperand::CreateFI(SI->second));
778  else
779  return false;
780  } else {
781  unsigned Reg = getRegForValue(Val);
782  if (!Reg)
783  return false;
784  Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
785  }
786  }
787  return true;
788 }
789 
791  // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
792  // [live variables...])
794  "Stackmap cannot return a value.");
795 
796  // The stackmap intrinsic only records the live variables (the arguments
797  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
798  // intrinsic, this won't be lowered to a function call. This means we don't
799  // have to worry about calling conventions and target-specific lowering code.
800  // Instead we perform the call lowering right here.
801  //
802  // CALLSEQ_START(0, 0...)
803  // STACKMAP(id, nbytes, ...)
804  // CALLSEQ_END(0, 0)
805  //
807 
808  // Add the <id> and <numBytes> constants.
809  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
810  "Expected a constant integer.");
811  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
812  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
813 
814  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
815  "Expected a constant integer.");
816  const auto *NumBytes =
817  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
818  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
819 
820  // Push live variables for the stack map (skipping the first two arguments
821  // <id> and <numBytes>).
822  if (!addStackMapLiveVars(Ops, I, 2))
823  return false;
824 
825  // We are not adding any register mask info here, because the stackmap doesn't
826  // clobber anything.
827 
828  // Add scratch registers as implicit def and early clobber.
830  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
831  for (unsigned i = 0; ScratchRegs[i]; ++i)
832  Ops.push_back(MachineOperand::CreateReg(
833  ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
834  /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
835 
836  // Issue CALLSEQ_START
837  unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
838  auto Builder =
839  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
840  const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
841  for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I)
842  Builder.addImm(0);
843 
844  // Issue STACKMAP.
845  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
846  TII.get(TargetOpcode::STACKMAP));
847  for (auto const &MO : Ops)
848  MIB.add(MO);
849 
850  // Issue CALLSEQ_END
851  unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
852  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
853  .addImm(0)
854  .addImm(0);
855 
856  // Inform the Frame Information that we have a stackmap in this function.
857  FuncInfo.MF->getFrameInfo().setHasStackMap();
858 
859  return true;
860 }
861 
862 /// Lower an argument list according to the target calling convention.
863 ///
864 /// This is a helper for lowering intrinsics that follow a target calling
865 /// convention or require stack pointer adjustment. Only a subset of the
866 /// intrinsic's operands need to participate in the calling convention.
867 bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
868  unsigned NumArgs, const Value *Callee,
869  bool ForceRetVoidTy, CallLoweringInfo &CLI) {
870  ArgListTy Args;
871  Args.reserve(NumArgs);
872 
873  // Populate the argument list.
874  ImmutableCallSite CS(CI);
875  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) {
876  Value *V = CI->getOperand(ArgI);
877 
878  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
879 
881  Entry.Val = V;
882  Entry.Ty = V->getType();
883  Entry.setAttributes(&CS, ArgI);
884  Args.push_back(Entry);
885  }
886 
887  Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())
888  : CI->getType();
889  CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
890 
891  return lowerCallTo(CLI);
892 }
893 
895  const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
896  StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
897  SmallString<32> MangledName;
898  Mangler::getNameWithPrefix(MangledName, Target, DL);
899  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
900  return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
901 }
902 
904  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
905  // i32 <numBytes>,
906  // i8* <target>,
907  // i32 <numArgs>,
908  // [Args...],
909  // [live variables...])
911  bool IsAnyRegCC = CC == CallingConv::AnyReg;
912  bool HasDef = !I->getType()->isVoidTy();
914 
915  // Get the real number of arguments participating in the call <numArgs>
916  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
917  "Expected a constant integer.");
918  const auto *NumArgsVal =
919  cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
920  unsigned NumArgs = NumArgsVal->getZExtValue();
921 
922  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
923  // This includes all meta-operands up to but not including CC.
924  unsigned NumMetaOpers = PatchPointOpers::CCPos;
925  assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
926  "Not enough arguments provided to the patchpoint intrinsic");
927 
928  // For AnyRegCC the arguments are lowered later on manually.
929  unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
930  CallLoweringInfo CLI;
931  CLI.setIsPatchPoint();
932  if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
933  return false;
934 
935  assert(CLI.Call && "No call instruction specified.");
936 
938 
939  // Add an explicit result reg if we use the anyreg calling convention.
940  if (IsAnyRegCC && HasDef) {
941  assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
942  CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64));
943  CLI.NumResultRegs = 1;
944  Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*isDef=*/true));
945  }
946 
947  // Add the <id> and <numBytes> constants.
948  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
949  "Expected a constant integer.");
950  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
951  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
952 
953  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
954  "Expected a constant integer.");
955  const auto *NumBytes =
956  cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
957  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
958 
959  // Add the call target.
960  if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
961  uint64_t CalleeConstAddr =
962  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
963  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
964  } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
965  if (C->getOpcode() == Instruction::IntToPtr) {
966  uint64_t CalleeConstAddr =
967  cast<ConstantInt>(C->getOperand(0))->getZExtValue();
968  Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
969  } else
970  llvm_unreachable("Unsupported ConstantExpr.");
971  } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
973  } else if (isa<ConstantPointerNull>(Callee))
975  else
976  llvm_unreachable("Unsupported callee address.");
977 
978  // Adjust <numArgs> to account for any arguments that have been passed on
979  // the stack instead.
980  unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs : CLI.OutRegs.size();
981  Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
982 
983  // Add the calling convention
984  Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
985 
986  // Add the arguments we omitted previously. The register allocator should
987  // place these in any free register.
988  if (IsAnyRegCC) {
989  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) {
990  unsigned Reg = getRegForValue(I->getArgOperand(i));
991  if (!Reg)
992  return false;
993  Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
994  }
995  }
996 
997  // Push the arguments from the call instruction.
998  for (auto Reg : CLI.OutRegs)
999  Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
1000 
1001  // Push live variables for the stack map.
1002  if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
1003  return false;
1004 
1005  // Push the register mask info.
1007  TRI.getCallPreservedMask(*FuncInfo.MF, CC)));
1008 
1009  // Add scratch registers as implicit def and early clobber.
1010  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
1011  for (unsigned i = 0; ScratchRegs[i]; ++i)
1013  ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
1014  /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
1015 
1016  // Add implicit defs (return values).
1017  for (auto Reg : CLI.InRegs)
1018  Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/true,
1019  /*isImp=*/true));
1020 
1021  // Insert the patchpoint instruction before the call generated by the target.
1022  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc,
1023  TII.get(TargetOpcode::PATCHPOINT));
1024 
1025  for (auto &MO : Ops)
1026  MIB.add(MO);
1027 
1028  MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1029 
1030  // Delete the original call instruction.
1031  CLI.Call->eraseFromParent();
1032 
1033  // Inform the Frame Information that we have a patchpoint in this function.
1034  FuncInfo.MF->getFrameInfo().setHasPatchPoint();
1035 
1036  if (CLI.NumResultRegs)
1037  updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
1038  return true;
1039 }
1040 
1042  const auto &Triple = TM.getTargetTriple();
1043  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1044  return true; // don't do anything to this instruction.
1046  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1047  /*isDef=*/false));
1048  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1049  /*isDef=*/false));
1050  MachineInstrBuilder MIB =
1051  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1052  TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
1053  for (auto &MO : Ops)
1054  MIB.add(MO);
1055 
1056  // Insert the Patchable Event Call instruction, that gets lowered properly.
1057  return true;
1058 }
1059 
1061  const auto &Triple = TM.getTargetTriple();
1062  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1063  return true; // don't do anything to this instruction.
1065  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1066  /*isDef=*/false));
1067  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1068  /*isDef=*/false));
1069  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(2)),
1070  /*isDef=*/false));
1071  MachineInstrBuilder MIB =
1072  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1073  TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
1074  for (auto &MO : Ops)
1075  MIB.add(MO);
1076 
1077  // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
1078  return true;
1079 }
1080 
1081 /// Returns an AttributeList representing the attributes applied to the return
1082 /// value of the given call.
1085  if (CLI.RetSExt)
1086  Attrs.push_back(Attribute::SExt);
1087  if (CLI.RetZExt)
1088  Attrs.push_back(Attribute::ZExt);
1089  if (CLI.IsInReg)
1090  Attrs.push_back(Attribute::InReg);
1091 
1093  Attrs);
1094 }
1095 
1096 bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
1097  unsigned NumArgs) {
1098  MCContext &Ctx = MF->getContext();
1099  SmallString<32> MangledName;
1100  Mangler::getNameWithPrefix(MangledName, SymName, DL);
1101  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
1102  return lowerCallTo(CI, Sym, NumArgs);
1103 }
1104 
1106  unsigned NumArgs) {
1107  ImmutableCallSite CS(CI);
1108 
1109  FunctionType *FTy = CS.getFunctionType();
1110  Type *RetTy = CS.getType();
1111 
1112  ArgListTy Args;
1113  Args.reserve(NumArgs);
1114 
1115  // Populate the argument list.
1116  // Attributes for args start at offset 1, after the return attribute.
1117  for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI) {
1118  Value *V = CI->getOperand(ArgI);
1119 
1120  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
1121 
1123  Entry.Val = V;
1124  Entry.Ty = V->getType();
1125  Entry.setAttributes(&CS, ArgI);
1126  Args.push_back(Entry);
1127  }
1128  TLI.markLibCallAttributes(MF, CS.getCallingConv(), Args);
1129 
1130  CallLoweringInfo CLI;
1131  CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
1132 
1133  return lowerCallTo(CLI);
1134 }
1135 
1137  // Handle the incoming return values from the call.
1138  CLI.clearIns();
1139  SmallVector<EVT, 4> RetTys;
1140  ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
1141 
1143  GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
1144 
1145  bool CanLowerReturn = TLI.CanLowerReturn(
1146  CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1147 
1148  // FIXME: sret demotion isn't supported yet - bail out.
1149  if (!CanLowerReturn)
1150  return false;
1151 
1152  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
1153  EVT VT = RetTys[I];
1154  MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1155  unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1156  for (unsigned i = 0; i != NumRegs; ++i) {
1157  ISD::InputArg MyFlags;
1158  MyFlags.VT = RegisterVT;
1159  MyFlags.ArgVT = VT;
1160  MyFlags.Used = CLI.IsReturnValueUsed;
1161  if (CLI.RetSExt)
1162  MyFlags.Flags.setSExt();
1163  if (CLI.RetZExt)
1164  MyFlags.Flags.setZExt();
1165  if (CLI.IsInReg)
1166  MyFlags.Flags.setInReg();
1167  CLI.Ins.push_back(MyFlags);
1168  }
1169  }
1170 
1171  // Handle all of the outgoing arguments.
1172  CLI.clearOuts();
1173  for (auto &Arg : CLI.getArgs()) {
1174  Type *FinalType = Arg.Ty;
1175  if (Arg.IsByVal)
1176  FinalType = cast<PointerType>(Arg.Ty)->getElementType();
1177  bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1178  FinalType, CLI.CallConv, CLI.IsVarArg);
1179 
1180  ISD::ArgFlagsTy Flags;
1181  if (Arg.IsZExt)
1182  Flags.setZExt();
1183  if (Arg.IsSExt)
1184  Flags.setSExt();
1185  if (Arg.IsInReg)
1186  Flags.setInReg();
1187  if (Arg.IsSRet)
1188  Flags.setSRet();
1189  if (Arg.IsSwiftSelf)
1190  Flags.setSwiftSelf();
1191  if (Arg.IsSwiftError)
1192  Flags.setSwiftError();
1193  if (Arg.IsByVal)
1194  Flags.setByVal();
1195  if (Arg.IsInAlloca) {
1196  Flags.setInAlloca();
1197  // Set the byval flag for CCAssignFn callbacks that don't know about
1198  // inalloca. This way we can know how many bytes we should've allocated
1199  // and how many bytes a callee cleanup function will pop. If we port
1200  // inalloca to more targets, we'll have to add custom inalloca handling in
1201  // the various CC lowering callbacks.
1202  Flags.setByVal();
1203  }
1204  if (Arg.IsByVal || Arg.IsInAlloca) {
1205  PointerType *Ty = cast<PointerType>(Arg.Ty);
1206  Type *ElementTy = Ty->getElementType();
1207  unsigned FrameSize =
1208  DL.getTypeAllocSize(Arg.ByValType ? Arg.ByValType : ElementTy);
1209 
1210  // For ByVal, alignment should come from FE. BE will guess if this info
1211  // is not there, but there are cases it cannot get right.
1212  unsigned FrameAlign = Arg.Alignment;
1213  if (!FrameAlign)
1214  FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
1215  Flags.setByValSize(FrameSize);
1216  Flags.setByValAlign(FrameAlign);
1217  }
1218  if (Arg.IsNest)
1219  Flags.setNest();
1220  if (NeedsRegBlock)
1221  Flags.setInConsecutiveRegs();
1222  unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
1223  Flags.setOrigAlign(OriginalAlignment);
1224 
1225  CLI.OutVals.push_back(Arg.Val);
1226  CLI.OutFlags.push_back(Flags);
1227  }
1228 
1229  if (!fastLowerCall(CLI))
1230  return false;
1231 
1232  // Set all unused physreg defs as dead.
1233  assert(CLI.Call && "No call instruction specified.");
1234  CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1235 
1236  if (CLI.NumResultRegs && CLI.CS)
1237  updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs);
1238 
1239  // Set labels for heapallocsite call.
1240  if (CLI.CS && CLI.CS->getInstruction()->hasMetadata("heapallocsite")) {
1241  const MDNode *MD = CLI.CS->getInstruction()->getMetadata("heapallocsite");
1242  MF->addCodeViewHeapAllocSite(CLI.Call, MD);
1243  }
1244 
1245  return true;
1246 }
1247 
1249  ImmutableCallSite CS(CI);
1250 
1251  FunctionType *FuncTy = CS.getFunctionType();
1252  Type *RetTy = CS.getType();
1253 
1254  ArgListTy Args;
1256  Args.reserve(CS.arg_size());
1257 
1258  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
1259  i != e; ++i) {
1260  Value *V = *i;
1261 
1262  // Skip empty types
1263  if (V->getType()->isEmptyTy())
1264  continue;
1265 
1266  Entry.Val = V;
1267  Entry.Ty = V->getType();
1268 
1269  // Skip the first return-type Attribute to get to params.
1270  Entry.setAttributes(&CS, i - CS.arg_begin());
1271  Args.push_back(Entry);
1272  }
1273 
1274  // Check if target-independent constraints permit a tail call here.
1275  // Target-dependent constraints are checked within fastLowerCall.
1276  bool IsTailCall = CI->isTailCall();
1277  if (IsTailCall && !isInTailCallPosition(CS, TM))
1278  IsTailCall = false;
1279 
1280  CallLoweringInfo CLI;
1281  CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
1282  .setTailCall(IsTailCall);
1283 
1284  return lowerCallTo(CLI);
1285 }
1286 
1288  const CallInst *Call = cast<CallInst>(I);
1289 
1290  // Handle simple inline asms.
1291  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
1292  // If the inline asm has side effects, then make sure that no local value
1293  // lives across by flushing the local value map.
1294  if (IA->hasSideEffects())
1295  flushLocalValueMap();
1296 
1297  // Don't attempt to handle constraints.
1298  if (!IA->getConstraintString().empty())
1299  return false;
1300 
1301  unsigned ExtraInfo = 0;
1302  if (IA->hasSideEffects())
1303  ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1304  if (IA->isAlignStack())
1305  ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1306  ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
1307 
1308  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1310  .addExternalSymbol(IA->getAsmString().c_str())
1311  .addImm(ExtraInfo);
1312  return true;
1313  }
1314 
1315  // Handle intrinsic function calls.
1316  if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1317  return selectIntrinsicCall(II);
1318 
1319  // Usually, it does not make sense to initialize a value,
1320  // make an unrelated function call and use the value, because
1321  // it tends to be spilled on the stack. So, we move the pointer
1322  // to the last local value to the beginning of the block, so that
1323  // all the values which have already been materialized,
1324  // appear after the call. It also makes sense to skip intrinsics
1325  // since they tend to be inlined.
1326  flushLocalValueMap();
1327 
1328  return lowerCall(Call);
1329 }
1330 
1332  switch (II->getIntrinsicID()) {
1333  default:
1334  break;
1335  // At -O0 we don't care about the lifetime intrinsics.
1336  case Intrinsic::lifetime_start:
1337  case Intrinsic::lifetime_end:
1338  // The donothing intrinsic does, well, nothing.
1339  case Intrinsic::donothing:
1340  // Neither does the sideeffect intrinsic.
1341  case Intrinsic::sideeffect:
1342  // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1343  case Intrinsic::assume:
1344  return true;
1345  case Intrinsic::dbg_declare: {
1346  const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1347  assert(DI->getVariable() && "Missing variable");
1348  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1349  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1350  return true;
1351  }
1352 
1353  const Value *Address = DI->getAddress();
1354  if (!Address || isa<UndefValue>(Address)) {
1355  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1356  return true;
1357  }
1358 
1359  // Byval arguments with frame indices were already handled after argument
1360  // lowering and before isel.
1361  const auto *Arg =
1363  if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX)
1364  return true;
1365 
1367  if (unsigned Reg = lookUpRegForValue(Address))
1368  Op = MachineOperand::CreateReg(Reg, false);
1369 
1370  // If we have a VLA that has a "use" in a metadata node that's then used
1371  // here but it has no other uses, then we have a problem. E.g.,
1372  //
1373  // int foo (const int *x) {
1374  // char a[*x];
1375  // return 0;
1376  // }
1377  //
1378  // If we assign 'a' a vreg and fast isel later on has to use the selection
1379  // DAG isel, it will want to copy the value to the vreg. However, there are
1380  // no uses, which goes counter to what selection DAG isel expects.
1381  if (!Op && !Address->use_empty() && isa<Instruction>(Address) &&
1382  (!isa<AllocaInst>(Address) ||
1383  !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))))
1384  Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address),
1385  false);
1386 
1387  if (Op) {
1389  "Expected inlined-at fields to agree");
1390  // A dbg.declare describes the address of a source variable, so lower it
1391  // into an indirect DBG_VALUE.
1392  auto *Expr = DI->getExpression();
1393  Expr = DIExpression::append(Expr, {dwarf::DW_OP_deref});
1394  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1395  TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ false,
1396  *Op, DI->getVariable(), Expr);
1397  } else {
1398  // We can't yet handle anything else here because it would require
1399  // generating code, thus altering codegen because of debug info.
1400  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1401  }
1402  return true;
1403  }
1404  case Intrinsic::dbg_value: {
1405  // This form of DBG_VALUE is target-independent.
1406  const DbgValueInst *DI = cast<DbgValueInst>(II);
1407  const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1408  const Value *V = DI->getValue();
1410  "Expected inlined-at fields to agree");
1411  if (!V) {
1412  // Currently the optimizer can produce this; insert an undef to
1413  // help debugging. Probably the optimizer should not do this.
1414  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
1415  DI->getVariable(), DI->getExpression());
1416  } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1417  if (CI->getBitWidth() > 64)
1418  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1419  .addCImm(CI)
1420  .addReg(0U)
1421  .addMetadata(DI->getVariable())
1422  .addMetadata(DI->getExpression());
1423  else
1424  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1425  .addImm(CI->getZExtValue())
1426  .addReg(0U)
1427  .addMetadata(DI->getVariable())
1428  .addMetadata(DI->getExpression());
1429  } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1430  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1431  .addFPImm(CF)
1432  .addReg(0U)
1433  .addMetadata(DI->getVariable())
1434  .addMetadata(DI->getExpression());
1435  } else if (unsigned Reg = lookUpRegForValue(V)) {
1436  // FIXME: This does not handle register-indirect values at offset 0.
1437  bool IsIndirect = false;
1438  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
1439  DI->getVariable(), DI->getExpression());
1440  } else {
1441  // We can't yet handle anything else here because it would require
1442  // generating code, thus altering codegen because of debug info.
1443  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1444  }
1445  return true;
1446  }
1447  case Intrinsic::dbg_label: {
1448  const DbgLabelInst *DI = cast<DbgLabelInst>(II);
1449  assert(DI->getLabel() && "Missing label");
1450  if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1451  LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1452  return true;
1453  }
1454 
1455  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1456  TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
1457  return true;
1458  }
1459  case Intrinsic::objectsize:
1460  llvm_unreachable("llvm.objectsize.* should have been lowered already");
1461 
1462  case Intrinsic::is_constant:
1463  llvm_unreachable("llvm.is.constant.* should have been lowered already");
1464 
1465  case Intrinsic::launder_invariant_group:
1466  case Intrinsic::strip_invariant_group:
1467  case Intrinsic::expect: {
1468  unsigned ResultReg = getRegForValue(II->getArgOperand(0));
1469  if (!ResultReg)
1470  return false;
1471  updateValueMap(II, ResultReg);
1472  return true;
1473  }
1474  case Intrinsic::experimental_stackmap:
1475  return selectStackmap(II);
1476  case Intrinsic::experimental_patchpoint_void:
1477  case Intrinsic::experimental_patchpoint_i64:
1478  return selectPatchpoint(II);
1479 
1480  case Intrinsic::xray_customevent:
1481  return selectXRayCustomEvent(II);
1482  case Intrinsic::xray_typedevent:
1483  return selectXRayTypedEvent(II);
1484  }
1485 
1486  return fastLowerIntrinsicCall(II);
1487 }
1488 
1489 bool FastISel::selectCast(const User *I, unsigned Opcode) {
1490  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1491  EVT DstVT = TLI.getValueType(DL, I->getType());
1492 
1493  if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other ||
1494  !DstVT.isSimple())
1495  // Unhandled type. Halt "fast" selection and bail.
1496  return false;
1497 
1498  // Check if the destination type is legal.
1499  if (!TLI.isTypeLegal(DstVT))
1500  return false;
1501 
1502  // Check if the source operand is legal.
1503  if (!TLI.isTypeLegal(SrcVT))
1504  return false;
1505 
1506  unsigned InputReg = getRegForValue(I->getOperand(0));
1507  if (!InputReg)
1508  // Unhandled operand. Halt "fast" selection and bail.
1509  return false;
1510 
1511  bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
1512 
1513  unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1514  Opcode, InputReg, InputRegIsKill);
1515  if (!ResultReg)
1516  return false;
1517 
1518  updateValueMap(I, ResultReg);
1519  return true;
1520 }
1521 
1523  // If the bitcast doesn't change the type, just use the operand value.
1524  if (I->getType() == I->getOperand(0)->getType()) {
1525  unsigned Reg = getRegForValue(I->getOperand(0));
1526  if (!Reg)
1527  return false;
1528  updateValueMap(I, Reg);
1529  return true;
1530  }
1531 
1532  // Bitcasts of other values become reg-reg copies or BITCAST operators.
1533  EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1534  EVT DstEVT = TLI.getValueType(DL, I->getType());
1535  if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1536  !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT))
1537  // Unhandled type. Halt "fast" selection and bail.
1538  return false;
1539 
1540  MVT SrcVT = SrcEVT.getSimpleVT();
1541  MVT DstVT = DstEVT.getSimpleVT();
1542  unsigned Op0 = getRegForValue(I->getOperand(0));
1543  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1544  return false;
1545  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
1546 
1547  // First, try to perform the bitcast by inserting a reg-reg copy.
1548  unsigned ResultReg = 0;
1549  if (SrcVT == DstVT) {
1550  const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
1551  const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
1552  // Don't attempt a cross-class copy. It will likely fail.
1553  if (SrcClass == DstClass) {
1554  ResultReg = createResultReg(DstClass);
1555  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1556  TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
1557  }
1558  }
1559 
1560  // If the reg-reg copy failed, select a BITCAST opcode.
1561  if (!ResultReg)
1562  ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
1563 
1564  if (!ResultReg)
1565  return false;
1566 
1567  updateValueMap(I, ResultReg);
1568  return true;
1569 }
1570 
1571 // Remove local value instructions starting from the instruction after
1572 // SavedLastLocalValue to the current function insert point.
1573 void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1574 {
1575  MachineInstr *CurLastLocalValue = getLastLocalValue();
1576  if (CurLastLocalValue != SavedLastLocalValue) {
1577  // Find the first local value instruction to be deleted.
1578  // This is the instruction after SavedLastLocalValue if it is non-NULL.
1579  // Otherwise it's the first instruction in the block.
1580  MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1581  if (SavedLastLocalValue)
1582  ++FirstDeadInst;
1583  else
1584  FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1585  setLastLocalValue(SavedLastLocalValue);
1586  removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1587  }
1588 }
1589 
1591  MachineInstr *SavedLastLocalValue = getLastLocalValue();
1592  // Just before the terminator instruction, insert instructions to
1593  // feed PHI nodes in successor blocks.
1594  if (I->isTerminator()) {
1595  if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1596  // PHI node handling may have generated local value instructions,
1597  // even though it failed to handle all PHI nodes.
1598  // We remove these instructions because SelectionDAGISel will generate
1599  // them again.
1600  removeDeadLocalValueCode(SavedLastLocalValue);
1601  return false;
1602  }
1603  }
1604 
1605  // FastISel does not handle any operand bundles except OB_funclet.
1607  for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i)
1608  if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1609  return false;
1610 
1611  DbgLoc = I->getDebugLoc();
1612 
1613  SavedInsertPt = FuncInfo.InsertPt;
1614 
1615  if (const auto *Call = dyn_cast<CallInst>(I)) {
1616  const Function *F = Call->getCalledFunction();
1617  LibFunc Func;
1618 
1619  // As a special case, don't handle calls to builtin library functions that
1620  // may be translated directly to target instructions.
1621  if (F && !F->hasLocalLinkage() && F->hasName() &&
1622  LibInfo->getLibFunc(F->getName(), Func) &&
1623  LibInfo->hasOptimizedCodeGen(Func))
1624  return false;
1625 
1626  // Don't handle Intrinsic::trap if a trap function is specified.
1627  if (F && F->getIntrinsicID() == Intrinsic::trap &&
1628  Call->hasFnAttr("trap-func-name"))
1629  return false;
1630  }
1631 
1632  // First, try doing target-independent selection.
1633  if (!SkipTargetIndependentISel) {
1634  if (selectOperator(I, I->getOpcode())) {
1635  ++NumFastIselSuccessIndependent;
1636  DbgLoc = DebugLoc();
1637  return true;
1638  }
1639  // Remove dead code.
1640  recomputeInsertPt();
1641  if (SavedInsertPt != FuncInfo.InsertPt)
1642  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1643  SavedInsertPt = FuncInfo.InsertPt;
1644  }
1645  // Next, try calling the target to attempt to handle the instruction.
1646  if (fastSelectInstruction(I)) {
1647  ++NumFastIselSuccessTarget;
1648  DbgLoc = DebugLoc();
1649  return true;
1650  }
1651  // Remove dead code.
1652  recomputeInsertPt();
1653  if (SavedInsertPt != FuncInfo.InsertPt)
1654  removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1655 
1656  DbgLoc = DebugLoc();
1657  // Undo phi node updates, because they will be added again by SelectionDAG.
1658  if (I->isTerminator()) {
1659  // PHI node handling may have generated local value instructions.
1660  // We remove them because SelectionDAGISel will generate them again.
1661  removeDeadLocalValueCode(SavedLastLocalValue);
1662  FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
1663  }
1664  return false;
1665 }
1666 
1667 /// Emit an unconditional branch to the given block, unless it is the immediate
1668 /// (fall-through) successor, and update the CFG.
1670  const DebugLoc &DbgLoc) {
1671  if (FuncInfo.MBB->getBasicBlock()->sizeWithoutDebug() > 1 &&
1672  FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
1673  // For more accurate line information if this is the only non-debug
1674  // instruction in the block then emit it, otherwise we have the
1675  // unconditional fall-through case, which needs no instructions.
1676  } else {
1677  // The unconditional branch case.
1678  TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1679  SmallVector<MachineOperand, 0>(), DbgLoc);
1680  }
1681  if (FuncInfo.BPI) {
1682  auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1683  FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
1684  FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
1685  } else
1686  FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
1687 }
1688 
1690  MachineBasicBlock *TrueMBB,
1691  MachineBasicBlock *FalseMBB) {
1692  // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1693  // happen in degenerate IR and MachineIR forbids to have a block twice in the
1694  // successor/predecessor lists.
1695  if (TrueMBB != FalseMBB) {
1696  if (FuncInfo.BPI) {
1697  auto BranchProbability =
1698  FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1699  FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
1700  } else
1701  FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
1702  }
1703 
1704  fastEmitBranch(FalseMBB, DbgLoc);
1705 }
1706 
1707 /// Emit an FNeg operation.
1708 bool FastISel::selectFNeg(const User *I, const Value *In) {
1709  unsigned OpReg = getRegForValue(In);
1710  if (!OpReg)
1711  return false;
1712  bool OpRegIsKill = hasTrivialKill(In);
1713 
1714  // If the target has ISD::FNEG, use it.
1715  EVT VT = TLI.getValueType(DL, I->getType());
1716  unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1717  OpReg, OpRegIsKill);
1718  if (ResultReg) {
1719  updateValueMap(I, ResultReg);
1720  return true;
1721  }
1722 
1723  // Bitcast the value to integer, twiddle the sign bit with xor,
1724  // and then bitcast it back to floating-point.
1725  if (VT.getSizeInBits() > 64)
1726  return false;
1727  EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1728  if (!TLI.isTypeLegal(IntVT))
1729  return false;
1730 
1731  unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1732  ISD::BITCAST, OpReg, OpRegIsKill);
1733  if (!IntReg)
1734  return false;
1735 
1736  unsigned IntResultReg = fastEmit_ri_(
1737  IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
1738  UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1739  if (!IntResultReg)
1740  return false;
1741 
1742  ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1743  IntResultReg, /*IsKill=*/true);
1744  if (!ResultReg)
1745  return false;
1746 
1747  updateValueMap(I, ResultReg);
1748  return true;
1749 }
1750 
1752  const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1753  if (!EVI)
1754  return false;
1755 
1756  // Make sure we only try to handle extracts with a legal result. But also
1757  // allow i1 because it's easy.
1758  EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1759  if (!RealVT.isSimple())
1760  return false;
1761  MVT VT = RealVT.getSimpleVT();
1762  if (!TLI.isTypeLegal(VT) && VT != MVT::i1)
1763  return false;
1764 
1765  const Value *Op0 = EVI->getOperand(0);
1766  Type *AggTy = Op0->getType();
1767 
1768  // Get the base result register.
1769  unsigned ResultReg;
1770  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(Op0);
1771  if (I != FuncInfo.ValueMap.end())
1772  ResultReg = I->second;
1773  else if (isa<Instruction>(Op0))
1774  ResultReg = FuncInfo.InitializeRegForValue(Op0);
1775  else
1776  return false; // fast-isel can't handle aggregate constants at the moment
1777 
1778  // Get the actual result register, which is an offset from the base register.
1779  unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1780 
1781  SmallVector<EVT, 4> AggValueVTs;
1782  ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1783 
1784  for (unsigned i = 0; i < VTIndex; i++)
1785  ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1786 
1787  updateValueMap(EVI, ResultReg);
1788  return true;
1789 }
1790 
1791 bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1792  switch (Opcode) {
1793  case Instruction::Add:
1794  return selectBinaryOp(I, ISD::ADD);
1795  case Instruction::FAdd:
1796  return selectBinaryOp(I, ISD::FADD);
1797  case Instruction::Sub:
1798  return selectBinaryOp(I, ISD::SUB);
1799  case Instruction::FSub: {
1800  // FNeg is currently represented in LLVM IR as a special case of FSub.
1801  Value *X;
1802  if (match(I, m_FNeg(m_Value(X))))
1803  return selectFNeg(I, X);
1804  return selectBinaryOp(I, ISD::FSUB);
1805  }
1806  case Instruction::Mul:
1807  return selectBinaryOp(I, ISD::MUL);
1808  case Instruction::FMul:
1809  return selectBinaryOp(I, ISD::FMUL);
1810  case Instruction::SDiv:
1811  return selectBinaryOp(I, ISD::SDIV);
1812  case Instruction::UDiv:
1813  return selectBinaryOp(I, ISD::UDIV);
1814  case Instruction::FDiv:
1815  return selectBinaryOp(I, ISD::FDIV);
1816  case Instruction::SRem:
1817  return selectBinaryOp(I, ISD::SREM);
1818  case Instruction::URem:
1819  return selectBinaryOp(I, ISD::UREM);
1820  case Instruction::FRem:
1821  return selectBinaryOp(I, ISD::FREM);
1822  case Instruction::Shl:
1823  return selectBinaryOp(I, ISD::SHL);
1824  case Instruction::LShr:
1825  return selectBinaryOp(I, ISD::SRL);
1826  case Instruction::AShr:
1827  return selectBinaryOp(I, ISD::SRA);
1828  case Instruction::And:
1829  return selectBinaryOp(I, ISD::AND);
1830  case Instruction::Or:
1831  return selectBinaryOp(I, ISD::OR);
1832  case Instruction::Xor:
1833  return selectBinaryOp(I, ISD::XOR);
1834 
1835  case Instruction::FNeg:
1836  return selectFNeg(I, I->getOperand(0));
1837 
1838  case Instruction::GetElementPtr:
1839  return selectGetElementPtr(I);
1840 
1841  case Instruction::Br: {
1842  const BranchInst *BI = cast<BranchInst>(I);
1843 
1844  if (BI->isUnconditional()) {
1845  const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1846  MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1847  fastEmitBranch(MSucc, BI->getDebugLoc());
1848  return true;
1849  }
1850 
1851  // Conditional branches are not handed yet.
1852  // Halt "fast" selection and bail.
1853  return false;
1854  }
1855 
1856  case Instruction::Unreachable:
1857  if (TM.Options.TrapUnreachable)
1858  return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
1859  else
1860  return true;
1861 
1862  case Instruction::Alloca:
1863  // FunctionLowering has the static-sized case covered.
1864  if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1865  return true;
1866 
1867  // Dynamic-sized alloca is not handled yet.
1868  return false;
1869 
1870  case Instruction::Call:
1871  // On AIX, call lowering uses the DAG-ISEL path currently so that the
1872  // callee of the direct function call instruction will be mapped to the
1873  // symbol for the function's entry point, which is distinct from the
1874  // function descriptor symbol. The latter is the symbol whose XCOFF symbol
1875  // name is the C-linkage name of the source level function.
1876  if (TM.getTargetTriple().isOSAIX())
1877  return false;
1878  return selectCall(I);
1879 
1880  case Instruction::BitCast:
1881  return selectBitCast(I);
1882 
1883  case Instruction::FPToSI:
1884  return selectCast(I, ISD::FP_TO_SINT);
1885  case Instruction::ZExt:
1886  return selectCast(I, ISD::ZERO_EXTEND);
1887  case Instruction::SExt:
1888  return selectCast(I, ISD::SIGN_EXTEND);
1889  case Instruction::Trunc:
1890  return selectCast(I, ISD::TRUNCATE);
1891  case Instruction::SIToFP:
1892  return selectCast(I, ISD::SINT_TO_FP);
1893 
1894  case Instruction::IntToPtr: // Deliberate fall-through.
1895  case Instruction::PtrToInt: {
1896  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1897  EVT DstVT = TLI.getValueType(DL, I->getType());
1898  if (DstVT.bitsGT(SrcVT))
1899  return selectCast(I, ISD::ZERO_EXTEND);
1900  if (DstVT.bitsLT(SrcVT))
1901  return selectCast(I, ISD::TRUNCATE);
1902  unsigned Reg = getRegForValue(I->getOperand(0));
1903  if (!Reg)
1904  return false;
1905  updateValueMap(I, Reg);
1906  return true;
1907  }
1908 
1909  case Instruction::ExtractValue:
1910  return selectExtractValue(I);
1911 
1912  case Instruction::PHI:
1913  llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1914 
1915  default:
1916  // Unhandled instruction. Halt "fast" selection and bail.
1917  return false;
1918  }
1919 }
1920 
1922  const TargetLibraryInfo *LibInfo,
1923  bool SkipTargetIndependentISel)
1924  : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1925  MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1926  TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1927  TII(*MF->getSubtarget().getInstrInfo()),
1928  TLI(*MF->getSubtarget().getTargetLowering()),
1929  TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1930  SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1931 
1932 FastISel::~FastISel() = default;
1933 
1934 bool FastISel::fastLowerArguments() { return false; }
1935 
1936 bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1937 
1939  return false;
1940 }
1941 
1942 unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1943 
1944 unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
1945  bool /*Op0IsKill*/) {
1946  return 0;
1947 }
1948 
1949 unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1950  bool /*Op0IsKill*/, unsigned /*Op1*/,
1951  bool /*Op1IsKill*/) {
1952  return 0;
1953 }
1954 
1955 unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1956  return 0;
1957 }
1958 
1959 unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1960  const ConstantFP * /*FPImm*/) {
1961  return 0;
1962 }
1963 
1964 unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1965  bool /*Op0IsKill*/, uint64_t /*Imm*/) {
1966  return 0;
1967 }
1968 
1969 /// This method is a wrapper of fastEmit_ri. It first tries to emit an
1970 /// instruction with an immediate operand using fastEmit_ri.
1971 /// If that fails, it materializes the immediate into a register and try
1972 /// fastEmit_rr instead.
1973 unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1974  bool Op0IsKill, uint64_t Imm, MVT ImmType) {
1975  // If this is a multiply by a power of two, emit this as a shift left.
1976  if (Opcode == ISD::MUL && isPowerOf2_64(Imm)) {
1977  Opcode = ISD::SHL;
1978  Imm = Log2_64(Imm);
1979  } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)) {
1980  // div x, 8 -> srl x, 3
1981  Opcode = ISD::SRL;
1982  Imm = Log2_64(Imm);
1983  }
1984 
1985  // Horrible hack (to be removed), check to make sure shift amounts are
1986  // in-range.
1987  if ((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
1988  Imm >= VT.getSizeInBits())
1989  return 0;
1990 
1991  // First check if immediate type is legal. If not, we can't use the ri form.
1992  unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
1993  if (ResultReg)
1994  return ResultReg;
1995  unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
1996  bool IsImmKill = true;
1997  if (!MaterialReg) {
1998  // This is a bit ugly/slow, but failing here means falling out of
1999  // fast-isel, which would be very slow.
2000  IntegerType *ITy =
2002  MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
2003  if (!MaterialReg)
2004  return 0;
2005  // FIXME: If the materialized register here has no uses yet then this
2006  // will be the first use and we should be able to mark it as killed.
2007  // However, the local value area for materialising constant expressions
2008  // grows down, not up, which means that any constant expressions we generate
2009  // later which also use 'Imm' could be after this instruction and therefore
2010  // after this kill.
2011  IsImmKill = false;
2012  }
2013  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
2014 }
2015 
2017  return MRI.createVirtualRegister(RC);
2018 }
2019 
2021  unsigned OpNum) {
2022  if (Register::isVirtualRegister(Op)) {
2023  const TargetRegisterClass *RegClass =
2024  TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
2025  if (!MRI.constrainRegClass(Op, RegClass)) {
2026  // If it's not legal to COPY between the register classes, something
2027  // has gone very wrong before we got here.
2028  unsigned NewOp = createResultReg(RegClass);
2030  TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
2031  return NewOp;
2032  }
2033  }
2034  return Op;
2035 }
2036 
2037 unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2038  const TargetRegisterClass *RC) {
2039  unsigned ResultReg = createResultReg(RC);
2040  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2041 
2042  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
2043  return ResultReg;
2044 }
2045 
2046 unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2047  const TargetRegisterClass *RC, unsigned Op0,
2048  bool Op0IsKill) {
2049  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2050 
2051  unsigned ResultReg = createResultReg(RC);
2052  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2053 
2054  if (II.getNumDefs() >= 1)
2055  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2056  .addReg(Op0, getKillRegState(Op0IsKill));
2057  else {
2059  .addReg(Op0, getKillRegState(Op0IsKill));
2061  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2062  }
2063 
2064  return ResultReg;
2065 }
2066 
2067 unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2068  const TargetRegisterClass *RC, unsigned Op0,
2069  bool Op0IsKill, unsigned Op1,
2070  bool Op1IsKill) {
2071  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2072 
2073  unsigned ResultReg = createResultReg(RC);
2074  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2075  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2076 
2077  if (II.getNumDefs() >= 1)
2078  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2079  .addReg(Op0, getKillRegState(Op0IsKill))
2080  .addReg(Op1, getKillRegState(Op1IsKill));
2081  else {
2083  .addReg(Op0, getKillRegState(Op0IsKill))
2084  .addReg(Op1, getKillRegState(Op1IsKill));
2086  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2087  }
2088  return ResultReg;
2089 }
2090 
2091 unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2092  const TargetRegisterClass *RC, unsigned Op0,
2093  bool Op0IsKill, unsigned Op1,
2094  bool Op1IsKill, unsigned Op2,
2095  bool Op2IsKill) {
2096  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2097 
2098  unsigned ResultReg = createResultReg(RC);
2099  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2100  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2101  Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
2102 
2103  if (II.getNumDefs() >= 1)
2104  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2105  .addReg(Op0, getKillRegState(Op0IsKill))
2106  .addReg(Op1, getKillRegState(Op1IsKill))
2107  .addReg(Op2, getKillRegState(Op2IsKill));
2108  else {
2110  .addReg(Op0, getKillRegState(Op0IsKill))
2111  .addReg(Op1, getKillRegState(Op1IsKill))
2112  .addReg(Op2, getKillRegState(Op2IsKill));
2114  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2115  }
2116  return ResultReg;
2117 }
2118 
2119 unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2120  const TargetRegisterClass *RC, unsigned Op0,
2121  bool Op0IsKill, uint64_t Imm) {
2122  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2123 
2124  unsigned ResultReg = createResultReg(RC);
2125  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2126 
2127  if (II.getNumDefs() >= 1)
2128  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2129  .addReg(Op0, getKillRegState(Op0IsKill))
2130  .addImm(Imm);
2131  else {
2133  .addReg(Op0, getKillRegState(Op0IsKill))
2134  .addImm(Imm);
2136  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2137  }
2138  return ResultReg;
2139 }
2140 
2141 unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2142  const TargetRegisterClass *RC, unsigned Op0,
2143  bool Op0IsKill, uint64_t Imm1,
2144  uint64_t Imm2) {
2145  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2146 
2147  unsigned ResultReg = createResultReg(RC);
2148  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2149 
2150  if (II.getNumDefs() >= 1)
2151  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2152  .addReg(Op0, getKillRegState(Op0IsKill))
2153  .addImm(Imm1)
2154  .addImm(Imm2);
2155  else {
2157  .addReg(Op0, getKillRegState(Op0IsKill))
2158  .addImm(Imm1)
2159  .addImm(Imm2);
2161  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2162  }
2163  return ResultReg;
2164 }
2165 
2166 unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2167  const TargetRegisterClass *RC,
2168  const ConstantFP *FPImm) {
2169  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2170 
2171  unsigned ResultReg = createResultReg(RC);
2172 
2173  if (II.getNumDefs() >= 1)
2174  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2175  .addFPImm(FPImm);
2176  else {
2178  .addFPImm(FPImm);
2180  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2181  }
2182  return ResultReg;
2183 }
2184 
2185 unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2186  const TargetRegisterClass *RC, unsigned Op0,
2187  bool Op0IsKill, unsigned Op1,
2188  bool Op1IsKill, uint64_t Imm) {
2189  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2190 
2191  unsigned ResultReg = createResultReg(RC);
2192  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2193  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2194 
2195  if (II.getNumDefs() >= 1)
2196  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2197  .addReg(Op0, getKillRegState(Op0IsKill))
2198  .addReg(Op1, getKillRegState(Op1IsKill))
2199  .addImm(Imm);
2200  else {
2202  .addReg(Op0, getKillRegState(Op0IsKill))
2203  .addReg(Op1, getKillRegState(Op1IsKill))
2204  .addImm(Imm);
2206  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2207  }
2208  return ResultReg;
2209 }
2210 
2211 unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2212  const TargetRegisterClass *RC, uint64_t Imm) {
2213  unsigned ResultReg = createResultReg(RC);
2214  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2215 
2216  if (II.getNumDefs() >= 1)
2217  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2218  .addImm(Imm);
2219  else {
2222  TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2223  }
2224  return ResultReg;
2225 }
2226 
2227 unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2228  bool Op0IsKill, uint32_t Idx) {
2229  unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2231  "Cannot yet extract from physregs");
2232  const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2234  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
2235  ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
2236  return ResultReg;
2237 }
2238 
2239 /// Emit MachineInstrs to compute the value of Op with all but the least
2240 /// significant bit set to zero.
2241 unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
2242  return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
2243 }
2244 
2245 /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2246 /// Emit code to ensure constants are copied into registers when needed.
2247 /// Remember the virtual registers that need to be added to the Machine PHI
2248 /// nodes as input. We cannot just directly add them, because expansion
2249 /// might result in multiple MBB's for one BB. As such, the start of the
2250 /// BB might correspond to a different MBB than the end.
2251 bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2252  const Instruction *TI = LLVMBB->getTerminator();
2253 
2256 
2257  // Check successor nodes' PHI nodes that expect a constant to be available
2258  // from this block.
2259  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
2260  const BasicBlock *SuccBB = TI->getSuccessor(succ);
2261  if (!isa<PHINode>(SuccBB->begin()))
2262  continue;
2263  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2264 
2265  // If this terminator has multiple identical successors (common for
2266  // switches), only handle each succ once.
2267  if (!SuccsHandled.insert(SuccMBB).second)
2268  continue;
2269 
2270  MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2271 
2272  // At this point we know that there is a 1-1 correspondence between LLVM PHI
2273  // nodes and Machine PHI nodes, but the incoming operands have not been
2274  // emitted yet.
2275  for (const PHINode &PN : SuccBB->phis()) {
2276  // Ignore dead phi's.
2277  if (PN.use_empty())
2278  continue;
2279 
2280  // Only handle legal types. Two interesting things to note here. First,
2281  // by bailing out early, we may leave behind some dead instructions,
2282  // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2283  // own moves. Second, this check is necessary because FastISel doesn't
2284  // use CreateRegs to create registers, so it always creates
2285  // exactly one register for each non-void instruction.
2286  EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2287  if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2288  // Handle integer promotions, though, because they're common and easy.
2289  if (!(VT == MVT::i1 || VT == MVT::i8 || VT == MVT::i16)) {
2291  return false;
2292  }
2293  }
2294 
2295  const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2296 
2297  // Set the DebugLoc for the copy. Prefer the location of the operand
2298  // if there is one; use the location of the PHI otherwise.
2299  DbgLoc = PN.getDebugLoc();
2300  if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2301  DbgLoc = Inst->getDebugLoc();
2302 
2303  unsigned Reg = getRegForValue(PHIOp);
2304  if (!Reg) {
2306  return false;
2307  }
2308  FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2309  DbgLoc = DebugLoc();
2310  }
2311  }
2312 
2313  return true;
2314 }
2315 
2316 bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2317  assert(LI->hasOneUse() &&
2318  "tryToFoldLoad expected a LoadInst with a single use");
2319  // We know that the load has a single use, but don't know what it is. If it
2320  // isn't one of the folded instructions, then we can't succeed here. Handle
2321  // this by scanning the single-use users of the load until we get to FoldInst.
2322  unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2323 
2324  const Instruction *TheUser = LI->user_back();
2325  while (TheUser != FoldInst && // Scan up until we find FoldInst.
2326  // Stay in the right block.
2327  TheUser->getParent() == FoldInst->getParent() &&
2328  --MaxUsers) { // Don't scan too far.
2329  // If there are multiple or no uses of this instruction, then bail out.
2330  if (!TheUser->hasOneUse())
2331  return false;
2332 
2333  TheUser = TheUser->user_back();
2334  }
2335 
2336  // If we didn't find the fold instruction, then we failed to collapse the
2337  // sequence.
2338  if (TheUser != FoldInst)
2339  return false;
2340 
2341  // Don't try to fold volatile loads. Target has to deal with alignment
2342  // constraints.
2343  if (LI->isVolatile())
2344  return false;
2345 
2346  // Figure out which vreg this is going into. If there is no assigned vreg yet
2347  // then there actually was no reference to it. Perhaps the load is referenced
2348  // by a dead instruction.
2349  unsigned LoadReg = getRegForValue(LI);
2350  if (!LoadReg)
2351  return false;
2352 
2353  // We can't fold if this vreg has no uses or more than one use. Multiple uses
2354  // may mean that the instruction got lowered to multiple MIs, or the use of
2355  // the loaded value ended up being multiple operands of the result.
2356  if (!MRI.hasOneUse(LoadReg))
2357  return false;
2358 
2360  MachineInstr *User = RI->getParent();
2361 
2362  // Set the insertion point properly. Folding the load can cause generation of
2363  // other random instructions (like sign extends) for addressing modes; make
2364  // sure they get inserted in a logical place before the new instruction.
2365  FuncInfo.InsertPt = User;
2366  FuncInfo.MBB = User->getParent();
2367 
2368  // Ask the target to try folding the load.
2369  return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2370 }
2371 
2373  // Must be an add.
2374  if (!isa<AddOperator>(Add))
2375  return false;
2376  // Type size needs to match.
2377  if (DL.getTypeSizeInBits(GEP->getType()) !=
2378  DL.getTypeSizeInBits(Add->getType()))
2379  return false;
2380  // Must be in the same basic block.
2381  if (isa<Instruction>(Add) &&
2382  FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2383  return false;
2384  // Must have a constant operand.
2385  return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2386 }
2387 
2390  const Value *Ptr;
2391  Type *ValTy;
2392  unsigned Alignment;
2394  bool IsVolatile;
2395 
2396  if (const auto *LI = dyn_cast<LoadInst>(I)) {
2397  Alignment = LI->getAlignment();
2398  IsVolatile = LI->isVolatile();
2399  Flags = MachineMemOperand::MOLoad;
2400  Ptr = LI->getPointerOperand();
2401  ValTy = LI->getType();
2402  } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2403  Alignment = SI->getAlignment();
2404  IsVolatile = SI->isVolatile();
2406  Ptr = SI->getPointerOperand();
2407  ValTy = SI->getValueOperand()->getType();
2408  } else
2409  return nullptr;
2410 
2411  bool IsNonTemporal = I->hasMetadata(LLVMContext::MD_nontemporal);
2412  bool IsInvariant = I->hasMetadata(LLVMContext::MD_invariant_load);
2413  bool IsDereferenceable = I->hasMetadata(LLVMContext::MD_dereferenceable);
2414  const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2415 
2416  AAMDNodes AAInfo;
2417  I->getAAMetadata(AAInfo);
2418 
2419  if (Alignment == 0) // Ensure that codegen never sees alignment 0.
2420  Alignment = DL.getABITypeAlignment(ValTy);
2421 
2422  unsigned Size = DL.getTypeStoreSize(ValTy);
2423 
2424  if (IsVolatile)
2426  if (IsNonTemporal)
2428  if (IsDereferenceable)
2430  if (IsInvariant)
2432 
2433  return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
2434  Alignment, AAInfo, Ranges);
2435 }
2436 
2438  // If both operands are the same, then try to optimize or fold the cmp.
2440  if (CI->getOperand(0) != CI->getOperand(1))
2441  return Predicate;
2442 
2443  switch (Predicate) {
2444  default: llvm_unreachable("Invalid predicate!");
2445  case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break;
2446  case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break;
2447  case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break;
2448  case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break;
2449  case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break;
2450  case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break;
2451  case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break;
2452  case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break;
2453  case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break;
2454  case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break;
2455  case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break;
2456  case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2457  case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break;
2458  case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2459  case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break;
2460  case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break;
2461 
2462  case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break;
2463  case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break;
2464  case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break;
2465  case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break;
2466  case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break;
2467  case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break;
2468  case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break;
2469  case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break;
2470  case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break;
2471  case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break;
2472  }
2473 
2474  return Predicate;
2475 }
void setHasStackMap(bool s=true)
IterTy arg_end() const
Definition: CallSite.h:588
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:596
unsigned fastEmitInst_rrr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, unsigned Op2, bool Op2IsKill)
Emit a MachineInstr with three register operands and a result register in the given register class...
Definition: FastISel.cpp:2091
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
void setByValAlign(unsigned A)
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:112
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:70
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:722
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
unsigned fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill)
Emit MachineInstrs to compute the value of Op with all but the least significant bit set to zero...
Definition: FastISel.cpp:2241
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
bool hasLocalLinkage() const
Definition: GlobalValue.h:445
This instruction extracts a struct member or array element value from an aggregate value...
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
bool lowerCall(const CallInst *I)
Definition: FastISel.cpp:1248
This represents the llvm.dbg.label instruction.
IterTy arg_begin() const
Definition: CallSite.h:584
This class represents lattice values for constants.
Definition: AllocatorList.h:23
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
bool selectFNeg(const User *I, const Value *In)
Emit an FNeg operation.
Definition: FastISel.cpp:1708
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
BasicBlock * getSuccessor(unsigned Idx) const
Return the specified successor. This instruction must be a terminator.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
bool selectXRayTypedEvent(const CallInst *II)
Definition: FastISel.cpp:1060
ImmutableCallSite * CS
Definition: FastISel.h:89
bool selectGetElementPtr(const User *I)
Definition: FastISel.cpp:675
void leaveLocalValueArea(SavePoint Old)
Reset InsertPt to the given old insert position.
Definition: FastISel.cpp:575
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:179
bool selectStackmap(const CallInst *I)
Definition: FastISel.cpp:790
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
This file contains the declarations for metadata subclasses.
virtual bool tryToFoldLoadIntoMI(MachineInstr *, unsigned, const LoadInst *)
The specified machine instr operand is a vreg, and that vreg is being provided by the specified load ...
Definition: FastISel.h:304
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
gep_type_iterator gep_type_end(const User *GEP)
unsigned less or equal
Definition: InstrTypes.h:758
unsigned less than
Definition: InstrTypes.h:757
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:738
MachineMemOperand * createMachineMemOperandFor(const Instruction *I) const
Create a machine mem operand from the given instruction.
Definition: FastISel.cpp:2389
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:743
bool isTerminator() const
Definition: Instruction.h:128
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:748
virtual unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode)
This method is called by target-independent code to request that an instruction with the given type a...
Definition: FastISel.cpp:1942
BasicBlock * getSuccessor(unsigned i) const
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
Metadata node.
Definition: Metadata.h:863
F(f)
An instruction for reading from memory.
Definition: Instructions.h:169
Hexagon Common GEP
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:618
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:144
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes...
virtual unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1955
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:477
FunctionType * getFunctionType() const
Definition: CallSite.h:328
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
virtual bool fastLowerCall(CallLoweringInfo &CLI)
This method is called by target-independent code to do target- specific call lowering.
Definition: FastISel.cpp:1936
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:130
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:289
CallingConv::ID getCallingConv() const
Get the calling convention of the call.
Definition: CallSite.h:320
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:273
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
Definition: FastISel.cpp:1590
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1241
unsigned fastEmitInst_rii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm1, uint64_t Imm2)
Emit a MachineInstr with one register operand and two immediate operands.
Definition: FastISel.cpp:2141
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1084
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:743
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:47
unsigned fastEmitInst_ri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, uint64_t Imm)
Emit a MachineInstr with a register operand, an immediate, and a result register in the given registe...
Definition: FastISel.cpp:2119
ArrayRef< unsigned > getIndices() const
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:742
unsigned fastEmitInst_rri(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill, uint64_t Imm)
Emit a MachineInstr with two register operands, an immediate, and a result register in the given regi...
Definition: FastISel.cpp:2185
CmpInst::Predicate optimizeCmpPredicate(const CmpInst *CI) const
Definition: FastISel.cpp:2437
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Definition: Instructions.h:233
A description of a memory reference used in the backend.
const HexagonInstrInfo * TII
TargetLoweringBase::ArgListTy ArgListTy
Definition: FastISel.h:69
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset, unsigned TargetFlags=0)
Shift and rotation operations.
Definition: ISDOpcodes.h:442
virtual const TargetRegisterClass * getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
Class to represent struct types.
Definition: DerivedTypes.h:238
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
DILabel * getLabel() const
bool canFoldAddIntoGEP(const User *GEP, const Value *Add)
Check if Add is an add that can be safely folded into GEP.
Definition: FastISel.cpp:2372
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned fastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill)
Emit a MachineInstr with one register operand and a result register in the given register class...
Definition: FastISel.cpp:2046
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:739
This file contains the simple types necessary to represent the attributes associated with functions a...
The memory access is dereferenceable (i.e., doesn&#39;t trap).
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
Definition: Instruction.h:224
static MachineOperand CreateRegMask(const uint32_t *Mask)
CreateRegMask - Creates a register mask operand referencing Mask.
virtual const TargetRegisterClass * getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx) const
Returns the largest legal sub-class of RC that supports the sub-register index Idx.
void setByValSize(unsigned S)
Type * getType() const
Return the type of the instruction that generated this call site.
Definition: CallSite.h:272
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:696
bool selectIntrinsicCall(const IntrinsicInst *II)
Definition: FastISel.cpp:1331
bool selectCast(const User *I, unsigned Opcode)
Definition: FastISel.cpp:1489
unsigned getSizeInBits() const
Context object for machine code objects.
Definition: MCContext.h:65
InstrTy * getInstruction() const
Definition: CallSite.h:96
Class to represent function types.
Definition: DerivedTypes.h:108
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
SmallVector< ISD::InputArg, 4 > Ins
Definition: FastISel.h:97
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:246
unsigned constrainOperandRegClass(const MCInstrDesc &II, unsigned Op, unsigned OpNum)
Try to constrain Op so that it is usable by argument OpNum of the provided MCInstrDesc.
Definition: FastISel.cpp:2020
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:510
bool selectOperator(const User *I, unsigned Opcode)
Do "fast" instruction selection for the given LLVM IR operator (Instruction or ConstantExpr), and append generated machine instructions to the current block.
Definition: FastISel.cpp:1791
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:455
ArchType getArch() const
getArch - Get the parsed architecture type of this triple.
Definition: Triple.h:296
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
bool selectExtractValue(const User *U)
Definition: FastISel.cpp:1751
unsigned getRegForValue(const Value *V)
Create a virtual register and arrange for it to be assigned the value for the given LLVM value...
Definition: FastISel.cpp:362
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
unsigned fastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass *RC)
Emit a MachineInstr with no operands and a result register in the given register class.
Definition: FastISel.cpp:2037
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:244
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringLiteral > StandardNames)
Initialize the set of available library functions based on the specified target triple.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:125
bool hasTrivialKill(const Value *V)
Test whether the given value has exactly one use.
Definition: FastISel.cpp:330
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
void setOrigAlign(unsigned A)
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > *Offsets=nullptr, uint64_t StartingOffset=0)
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:119
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
unsigned getNumSuccessors() const
Return the number of successors that this instruction has.
Value * getOperand(unsigned i) const
Definition: User.h:169
Class to represent pointers.
Definition: DerivedTypes.h:575
unsigned getKillRegState(bool B)
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:556
unsigned lookUpRegForValue(const Value *V)
Look up the value to see if its value is already cached in a register.
Definition: FastISel.cpp:475
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:19
void finishBasicBlock()
Flush the local value map and sink local values if possible.
Definition: FastISel.cpp:144
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:228
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:141
The memory access is volatile.
static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
Select the AArch64 opcode for the basic binary operation GenericOpc (such as G_OR or G_SDIV)...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void getAAMetadata(AAMDNodes &N, bool Merge=false) const
Fills the AAMDNodes structure with AA metadata from this instruction.
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
SmallVector< Register, 16 > OutRegs
Definition: FastISel.h:96
virtual ~FastISel()
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
unsigned const MachineRegisterInfo * MRI
CallLoweringInfo & setCallee(Type *ResultTy, FunctionType *FuncTy, const Value *Target, ArgListTy &&ArgsList, ImmutableCallSite &Call)
Definition: FastISel.h:104
Machine Value Type.
Value * getCalledValue() const
Definition: InstrTypes.h:1280
bool hasName() const
Definition: Value.h:252
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
const MachineInstrBuilder & addCImm(const ConstantInt *Val) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:287
Conditional or Unconditional Branch instruction.
Value * getAddress() const
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
Definition: FastISel.cpp:544
Value * getValue() const
SmallVector< ISD::ArgFlagsTy, 16 > OutFlags
Definition: FastISel.h:95
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
const MCPhysReg * ImplicitDefs
Definition: MCInstrDesc.h:189
DIExpression * getExpression() const
virtual unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1944
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:470
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:165
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We&#39;re checking to see if we can fold LI into FoldInst.
Definition: FastISel.cpp:2316
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
Definition: FastISel.cpp:146
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:732
std::pair< unsigned, bool > getRegForGEPIndex(const Value *Idx)
This is a wrapper around getRegForValue that also takes care of truncating or sign-extending the give...
Definition: FastISel.cpp:507
constexpr double e
Definition: MathExtras.h:57
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:799
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:741
self_iterator getIterator()
Definition: ilist_node.h:81
The memory access is non-temporal.
Class to represent integer types.
Definition: DerivedTypes.h:40
bool selectXRayCustomEvent(const CallInst *II)
Definition: FastISel.cpp:1041
const TargetRegisterInfo & TRI
Definition: FastISel.h:213
1 1 1 1 Always true (always folded)
Definition: InstrTypes.h:749
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:205
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static unsigned findSinkableLocalRegDef(MachineInstr &MI)
Return the defined register if this instruction defines exactly one virtual register and uses no othe...
Definition: FastISel.cpp:168
Extended Value Type.
Definition: ValueTypes.h:33
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:529
size_t size() const
Definition: SmallVector.h:52
This class contains a discriminated union of information about pointers in memory operands...
1 1 0 1 True if unordered, less than, or equal
Definition: InstrTypes.h:747
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
SmallVector< Value *, 16 > OutVals
Definition: FastISel.h:94
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call...
Definition: FastISel.cpp:1083
const TargetInstrInfo & TII
Definition: FastISel.h:211
MachineBasicBlock * MBB
MBB - The current block.
bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:512
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:43
signed greater than
Definition: InstrTypes.h:759
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:704
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
Definition: FastISel.cpp:530
The memory access writes data.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:50
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags...
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:736
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:244
virtual unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1949
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:389
virtual unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1964
MachineOperand class - Representation of each machine instruction operand.
1 1 0 0 True if unordered or less than
Definition: InstrTypes.h:746
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
bool selectCall(const User *I)
Definition: FastISel.cpp:1287
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:63
Provides information about what library functions are available for the current target.
unsigned arg_size() const
Definition: CallSite.h:226
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
void finishCondBranch(const BasicBlock *BranchBB, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB)
Emit an unconditional branch to FalseMBB, obtains the branch weight and adds TrueMBB and FalseMBB to ...
Definition: FastISel.cpp:1689
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:755
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:586
signed less than
Definition: InstrTypes.h:761
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:643
reg_iterator reg_begin(unsigned RegNo) const
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
unsigned fastEmitInst_rr(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill)
Emit a MachineInstr with two register operands and a result register in the given register class...
Definition: FastISel.cpp:2067
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:653
void updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs=1)
Update the value map to include the new mapping for this instruction, or insert an extra copy to get ...
Definition: FastISel.cpp:486
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:241
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:193
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
Definition: FastISel.cpp:130
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
signed less or equal
Definition: InstrTypes.h:762
bool selectBitCast(const User *I)
Definition: FastISel.cpp:1522
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Target - Wrapper for Target specific information.
virtual unsigned fastEmit_f(MVT VT, MVT RetVT, unsigned Opcode, const ConstantFP *FPImm)
This method is called by target-independent code to request that an instruction with the given type...
Definition: FastISel.cpp:1959
const DataLayout & DL
Definition: FastISel.h:210
bool selectBinaryOp(const User *I, unsigned ISDOpcode)
Select and emit code for a binary operator instruction, which has an opcode which directly correspond...
Definition: FastISel.cpp:584
void setDebugLoc(DebugLoc dl)
Replace current source information with new such.
amdgpu Simplify well known AMD library false FunctionCallee Callee
This file defines the FastISel class.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:240
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:500
bool isTailCall() const
DebugLoc DbgLoc
Definition: FastISel.h:208
Flags
Flags values. These may be or&#39;d together.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:256
The memory access reads data.
SavePoint enterLocalValueArea()
Prepare InsertPt to begin inserting instructions into the local value area and return the old insert ...
Definition: FastISel.cpp:566
Representation of each machine instruction.
Definition: MachineInstr.h:64
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:807
virtual bool fastLowerIntrinsicCall(const IntrinsicInst *II)
This method is called by target-independent code to do target- specific intrinsic lowering...
Definition: FastISel.cpp:1938
unsigned getOperandNo() const
getOperandNo - Return the operand # of this MachineOperand in its MachineInstr.
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match &#39;fneg X&#39; as &#39;fsub -0.0, X&#39;.
Definition: PatternMatch.h:814
bool selectPatchpoint(const CallInst *I)
Definition: FastISel.cpp:903
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1239
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:331
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:419
MachineRegisterInfo & MRI
Definition: FastISel.h:205
bool hasOneUse(unsigned RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:129
unsigned greater or equal
Definition: InstrTypes.h:756
This represents the llvm.dbg.value instruction.
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1344
bool lowerCallTo(const CallInst *CI, MCSymbol *Symbol, unsigned NumArgs)
Definition: FastISel.cpp:1105
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:44
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
Establish a view to a call site for examination.
Definition: CallSite.h:906
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1287
unsigned fastEmitInst_i(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm)
Emit a MachineInstr with a single immediate operand, and a result register in the given register clas...
Definition: FastISel.cpp:2211
static MachineOperand CreateImm(int64_t Val)
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call&#39;s live variable operands to a stackmap or patchpoint target node&#39;s ope...
The memory access always returns the same value (or traps).
static bool isRegUsedByPhiNodes(unsigned DefReg, FunctionLoweringInfo &FuncInfo)
Definition: FastISel.cpp:220
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:740
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
SmallVector< Register, 4 > InRegs
Definition: FastISel.h:98
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
void setPhysRegsDeadExcept(ArrayRef< Register > UsedRegs, const TargetRegisterInfo &TRI)
Mark every physreg used by this instruction as dead except those in the UsedRegs list.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:329
DILocalVariable * getVariable() const
bool isUnconditional() const
1 0 1 0 True if unordered or greater than
Definition: InstrTypes.h:744
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:477
const TargetLowering & TLI
Definition: FastISel.h:212
unsigned createResultReg(const TargetRegisterClass *RC)
Definition: FastISel.cpp:2016
CallLoweringInfo & setIsPatchPoint(bool Value=true)
Definition: FastISel.h:182
unsigned fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t Imm, MVT ImmType)
This method is a wrapper of fastEmit_ri.
Definition: FastISel.cpp:1973
unsigned fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, bool Op0IsKill, uint32_t Idx)
Emit a MachineInstr for an extract_subreg from a specified index of a superregister to a specified ty...
Definition: FastISel.cpp:2227
MachineBasicBlock::iterator InsertPt
Definition: FastISel.h:318
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
Definition: Value.h:396
FastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo, bool SkipTargetIndependentISel=false)
Definition: FastISel.cpp:1921
virtual bool fastLowerArguments()
This method is called by target-independent code to do target- specific argument lowering.
Definition: FastISel.cpp:1934
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:735
LLVM Value Representation.
Definition: Value.h:74
1 0 1 1 True if unordered, greater than, or equal
Definition: InstrTypes.h:745
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:69
static const Function * getParent(const Value *V)
static bool isCommutative(Instruction *I)
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable&#39;s name.
Definition: Mangler.cpp:111
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
IRTranslator LLVM IR MI
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:433
void fastEmitBranch(MachineBasicBlock *MSucc, const DebugLoc &DbgLoc)
Emit an unconditional branch to the given block, unless it is the immediate (fall-through) successor...
Definition: FastISel.cpp:1669
unsigned greater than
Definition: InstrTypes.h:755
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
static cl::opt< bool > SinkLocalValues("fast-isel-sink-local-values", cl::init(true), cl::Hidden, cl::desc("Sink local values in FastISel"))
unsigned fastEmitInst_f(unsigned MachineInstOpcode, const TargetRegisterClass *RC, const ConstantFP *FPImm)
Emit a MachineInstr with a floating point immediate, and a result register in the given register clas...
Definition: FastISel.cpp:2166
bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty...
Definition: Type.cpp:98
Conversion operators.
Definition: ISDOpcodes.h:497
FunctionLoweringInfo & FuncInfo
Definition: FastISel.h:203
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
Definition: Value.cpp:541
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:506
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:125
#define LLVM_DEBUG(X)
Definition: Debug.h:122
0 0 1 1 True if ordered and greater than or equal
Definition: InstrTypes.h:737
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:35
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
This represents the llvm.dbg.declare instruction.
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:611
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:63
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
static MachineOperand CreateFI(int Idx)
bool use_empty() const
Definition: Value.h:343
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:591
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Type * getElementType() const
Definition: DerivedTypes.h:594
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:970
0 0 0 0 Always false (always folded)
Definition: InstrTypes.h:734
signed greater or equal
Definition: InstrTypes.h:760
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:43
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
Definition: Instruction.h:66
gep_type_iterator gep_type_begin(const User *GEP)