LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - StatepointLowering.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 257 284 90.5 %
Date: 2018-10-20 13:21:21 Functions: 16 19 84.2 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file includes support code use by SelectionDAGBuilder when lowering a
      11             : // statepoint sequence in SelectionDAG IR.
      12             : //
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "StatepointLowering.h"
      16             : #include "SelectionDAGBuilder.h"
      17             : #include "llvm/ADT/ArrayRef.h"
      18             : #include "llvm/ADT/DenseMap.h"
      19             : #include "llvm/ADT/None.h"
      20             : #include "llvm/ADT/Optional.h"
      21             : #include "llvm/ADT/STLExtras.h"
      22             : #include "llvm/ADT/SmallVector.h"
      23             : #include "llvm/ADT/Statistic.h"
      24             : #include "llvm/CodeGen/FunctionLoweringInfo.h"
      25             : #include "llvm/CodeGen/GCMetadata.h"
      26             : #include "llvm/CodeGen/GCStrategy.h"
      27             : #include "llvm/CodeGen/ISDOpcodes.h"
      28             : #include "llvm/CodeGen/MachineFrameInfo.h"
      29             : #include "llvm/CodeGen/MachineFunction.h"
      30             : #include "llvm/CodeGen/MachineMemOperand.h"
      31             : #include "llvm/CodeGen/RuntimeLibcalls.h"
      32             : #include "llvm/CodeGen/SelectionDAG.h"
      33             : #include "llvm/CodeGen/SelectionDAGNodes.h"
      34             : #include "llvm/CodeGen/StackMaps.h"
      35             : #include "llvm/CodeGen/TargetLowering.h"
      36             : #include "llvm/CodeGen/TargetOpcodes.h"
      37             : #include "llvm/IR/CallingConv.h"
      38             : #include "llvm/IR/DerivedTypes.h"
      39             : #include "llvm/IR/Instruction.h"
      40             : #include "llvm/IR/Instructions.h"
      41             : #include "llvm/IR/LLVMContext.h"
      42             : #include "llvm/IR/Statepoint.h"
      43             : #include "llvm/IR/Type.h"
      44             : #include "llvm/Support/Casting.h"
      45             : #include "llvm/Support/MachineValueType.h"
      46             : #include "llvm/Target/TargetMachine.h"
      47             : #include "llvm/Target/TargetOptions.h"
      48             : #include <cassert>
      49             : #include <cstddef>
      50             : #include <cstdint>
      51             : #include <iterator>
      52             : #include <tuple>
      53             : #include <utility>
      54             : 
      55             : using namespace llvm;
      56             : 
      57             : #define DEBUG_TYPE "statepoint-lowering"
      58             : 
      59             : STATISTIC(NumSlotsAllocatedForStatepoints,
      60             :           "Number of stack slots allocated for statepoints");
      61             : STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
      62             : STATISTIC(StatepointMaxSlotsRequired,
      63             :           "Maximum number of stack slots required for a singe statepoint");
      64             : 
      65         284 : static void pushStackMapConstant(SmallVectorImpl<SDValue>& Ops,
      66             :                                  SelectionDAGBuilder &Builder, uint64_t Value) {
      67         284 :   SDLoc L = Builder.getCurSDLoc();
      68         284 :   Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
      69         284 :                                               MVT::i64));
      70         284 :   Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
      71         284 : }
      72             : 
      73          70 : void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
      74             :   // Consistency check
      75             :   assert(PendingGCRelocateCalls.empty() &&
      76             :          "Trying to visit statepoint before finished processing previous one");
      77          70 :   Locations.clear();
      78          70 :   NextSlotToAllocate = 0;
      79             :   // Need to resize this on each safepoint - we need the two to stay in sync and
      80             :   // the clear patterns of a SelectionDAGBuilder have no relation to
      81             :   // FunctionLoweringInfo.  Also need to ensure used bits get cleared.
      82          70 :   AllocatedStackSlots.clear();
      83          70 :   AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
      84          70 : }
      85             : 
      86     1269117 : void StatepointLoweringState::clear() {
      87     1269117 :   Locations.clear();
      88     1269116 :   AllocatedStackSlots.clear();
      89             :   assert(PendingGCRelocateCalls.empty() &&
      90             :          "cleared before statepoint sequence completed");
      91     1269116 : }
      92             : 
      93             : SDValue
      94          60 : StatepointLoweringState::allocateStackSlot(EVT ValueType,
      95             :                                            SelectionDAGBuilder &Builder) {
      96             :   NumSlotsAllocatedForStatepoints++;
      97          60 :   MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
      98             : 
      99             :   unsigned SpillSize = ValueType.getStoreSize();
     100             :   assert((SpillSize * 8) == ValueType.getSizeInBits() && "Size not in bytes?");
     101             : 
     102             :   // First look for a previously created stack slot which is not in
     103             :   // use (accounting for the fact arbitrary slots may already be
     104             :   // reserved), or to create a new stack slot and use it.
     105             : 
     106             :   const size_t NumSlots = AllocatedStackSlots.size();
     107             :   assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
     108             : 
     109             :   assert(AllocatedStackSlots.size() ==
     110             :          Builder.FuncInfo.StatepointStackSlots.size() &&
     111             :          "Broken invariant");
     112             : 
     113          85 :   for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
     114          72 :     if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
     115          26 :       const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
     116          13 :       if (MFI.getObjectSize(FI) == SpillSize) {
     117          11 :         AllocatedStackSlots.set(NextSlotToAllocate);
     118             :         // TODO: Is ValueType the right thing to use here?
     119          11 :         return Builder.DAG.getFrameIndex(FI, ValueType);
     120             :       }
     121             :     }
     122             :   }
     123             : 
     124             :   // Couldn't find a free slot, so create a new one:
     125             : 
     126          49 :   SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
     127          49 :   const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
     128             :   MFI.markAsStatepointSpillSlotObjectIndex(FI);
     129             : 
     130          49 :   Builder.FuncInfo.StatepointStackSlots.push_back(FI);
     131          98 :   AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
     132             :   assert(AllocatedStackSlots.size() ==
     133             :          Builder.FuncInfo.StatepointStackSlots.size() &&
     134             :          "Broken invariant");
     135             : 
     136             :   StatepointMaxSlotsRequired.updateMax(
     137             :       Builder.FuncInfo.StatepointStackSlots.size());
     138             : 
     139          49 :   return SpillSlot;
     140             : }
     141             : 
     142             : /// Utility function for reservePreviousStackSlotForValue. Tries to find
     143             : /// stack slot index to which we have spilled value for previous statepoints.
     144             : /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
     145         120 : static Optional<int> findPreviousSpillSlot(const Value *Val,
     146             :                                            SelectionDAGBuilder &Builder,
     147             :                                            int LookUpDepth) {
     148             :   // Can not look any further - give up now
     149         120 :   if (LookUpDepth <= 0)
     150             :     return None;
     151             : 
     152             :   // Spill location is known for gc relocates
     153             :   if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
     154             :     const auto &SpillMap =
     155           9 :         Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
     156             : 
     157           9 :     auto It = SpillMap.find(Relocate->getDerivedPtr());
     158           9 :     if (It == SpillMap.end())
     159             :       return None;
     160             : 
     161             :     return It->second;
     162             :   }
     163             : 
     164             :   // Look through bitcast instructions.
     165             :   if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
     166           3 :     return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
     167             : 
     168             :   // Look through phi nodes
     169             :   // All incoming values should have same known stack slot, otherwise result
     170             :   // is unknown.
     171             :   if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
     172             :     Optional<int> MergedResult = None;
     173             : 
     174           0 :     for (auto &IncomingValue : Phi->incoming_values()) {
     175             :       Optional<int> SpillSlot =
     176           0 :           findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
     177           0 :       if (!SpillSlot.hasValue())
     178             :         return None;
     179             : 
     180           0 :       if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
     181             :         return None;
     182             : 
     183             :       MergedResult = SpillSlot;
     184             :     }
     185             :     return MergedResult;
     186             :   }
     187             : 
     188             :   // TODO: We can do better for PHI nodes. In cases like this:
     189             :   //   ptr = phi(relocated_pointer, not_relocated_pointer)
     190             :   //   statepoint(ptr)
     191             :   // We will return that stack slot for ptr is unknown. And later we might
     192             :   // assign different stack slots for ptr and relocated_pointer. This limits
     193             :   // llvm's ability to remove redundant stores.
     194             :   // Unfortunately it's hard to accomplish in current infrastructure.
     195             :   // We use this function to eliminate spill store completely, while
     196             :   // in example we still need to emit store, but instead of any location
     197             :   // we need to use special "preferred" location.
     198             : 
     199             :   // TODO: handle simple updates.  If a value is modified and the original
     200             :   // value is no longer live, it would be nice to put the modified value in the
     201             :   // same slot.  This allows folding of the memory accesses for some
     202             :   // instructions types (like an increment).
     203             :   //   statepoint (i)
     204             :   //   i1 = i+1
     205             :   //   statepoint (i1)
     206             :   // However we need to be careful for cases like this:
     207             :   //   statepoint(i)
     208             :   //   i1 = i+1
     209             :   //   statepoint(i, i1)
     210             :   // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
     211             :   // put handling of simple modifications in this function like it's done
     212             :   // for bitcasts we might end up reserving i's slot for 'i+1' because order in
     213             :   // which we visit values is unspecified.
     214             : 
     215             :   // Don't know any information about this instruction
     216             :   return None;
     217             : }
     218             : 
     219             : /// Try to find existing copies of the incoming values in stack slots used for
     220             : /// statepoint spilling.  If we can find a spill slot for the incoming value,
     221             : /// mark that slot as allocated, and reuse the same slot for this safepoint.
     222             : /// This helps to avoid series of loads and stores that only serve to reshuffle
     223             : /// values on the stack between calls.
     224         205 : static void reservePreviousStackSlotForValue(const Value *IncomingValue,
     225             :                                              SelectionDAGBuilder &Builder) {
     226         205 :   SDValue Incoming = Builder.getValue(IncomingValue);
     227             : 
     228             :   if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
     229             :     // We won't need to spill this, so no need to check for previously
     230             :     // allocated stack slots
     231         196 :     return;
     232             :   }
     233             : 
     234         128 :   SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
     235         128 :   if (OldLocation.getNode())
     236             :     // Duplicates in input
     237             :     return;
     238             : 
     239             :   const int LookUpDepth = 6;
     240             :   Optional<int> Index =
     241         117 :       findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
     242         117 :   if (!Index.hasValue())
     243             :     return;
     244             : 
     245           9 :   const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
     246             : 
     247             :   auto SlotIt = find(StatepointSlots, *Index);
     248             :   assert(SlotIt != StatepointSlots.end() &&
     249             :          "Value spilled to the unknown stack slot");
     250             : 
     251             :   // This is one of our dedicated lowering slots
     252             :   const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
     253           9 :   if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
     254             :     // stack slot already assigned to someone else, can't use it!
     255             :     // TODO: currently we reserve space for gc arguments after doing
     256             :     // normal allocation for deopt arguments.  We should reserve for
     257             :     // _all_ deopt and gc arguments, then start allocating.  This
     258             :     // will prevent some moves being inserted when vm state changes,
     259             :     // but gc state doesn't between two calls.
     260             :     return;
     261             :   }
     262             :   // Reserve this stack slot
     263             :   Builder.StatepointLowering.reserveStackSlot(Offset);
     264             : 
     265             :   // Cache this slot so we find it when going through the normal
     266             :   // assignment loop.
     267             :   SDValue Loc =
     268          18 :       Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
     269           9 :   Builder.StatepointLowering.setLocation(Incoming, Loc);
     270             : }
     271             : 
     272             : /// Remove any duplicate (as SDValues) from the derived pointer pairs.  This
     273             : /// is not required for correctness.  It's purpose is to reduce the size of
     274             : /// StackMap section.  It has no effect on the number of spill slots required
     275             : /// or the actual lowering.
     276             : static void
     277          70 : removeDuplicateGCPtrs(SmallVectorImpl<const Value *> &Bases,
     278             :                       SmallVectorImpl<const Value *> &Ptrs,
     279             :                       SmallVectorImpl<const GCRelocateInst *> &Relocs,
     280             :                       SelectionDAGBuilder &Builder,
     281             :                       FunctionLoweringInfo::StatepointSpillMap &SSM) {
     282             :   DenseMap<SDValue, const Value *> Seen;
     283             : 
     284             :   SmallVector<const Value *, 64> NewBases, NewPtrs;
     285             :   SmallVector<const GCRelocateInst *, 64> NewRelocs;
     286         136 :   for (size_t i = 0, e = Ptrs.size(); i < e; i++) {
     287          66 :     SDValue SD = Builder.getValue(Ptrs[i]);
     288          66 :     auto SeenIt = Seen.find(SD);
     289             : 
     290          66 :     if (SeenIt == Seen.end()) {
     291             :       // Only add non-duplicates
     292          58 :       NewBases.push_back(Bases[i]);
     293          58 :       NewPtrs.push_back(Ptrs[i]);
     294          58 :       NewRelocs.push_back(Relocs[i]);
     295          58 :       Seen[SD] = Ptrs[i];
     296             :     } else {
     297             :       // Duplicate pointer found, note in SSM and move on:
     298           8 :       SSM.DuplicateMap[Ptrs[i]] = SeenIt->second;
     299             :     }
     300             :   }
     301             :   assert(Bases.size() >= NewBases.size());
     302             :   assert(Ptrs.size() >= NewPtrs.size());
     303             :   assert(Relocs.size() >= NewRelocs.size());
     304          70 :   Bases = NewBases;
     305          70 :   Ptrs = NewPtrs;
     306          70 :   Relocs = NewRelocs;
     307             :   assert(Ptrs.size() == Bases.size());
     308             :   assert(Ptrs.size() == Relocs.size());
     309          70 : }
     310             : 
     311             : /// Extract call from statepoint, lower it and return pointer to the
     312             : /// call node. Also update NodeMap so that getValue(statepoint) will
     313             : /// reference lowered call result
     314           0 : static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
     315             :     SelectionDAGBuilder::StatepointLoweringInfo &SI,
     316             :     SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
     317             :   SDValue ReturnValue, CallEndVal;
     318           0 :   std::tie(ReturnValue, CallEndVal) =
     319           0 :       Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
     320           0 :   SDNode *CallEnd = CallEndVal.getNode();
     321             : 
     322             :   // Get a call instruction from the call sequence chain.  Tail calls are not
     323             :   // allowed.  The following code is essentially reverse engineering X86's
     324             :   // LowerCallTo.
     325             :   //
     326             :   // We are expecting DAG to have the following form:
     327             :   //
     328             :   // ch = eh_label (only in case of invoke statepoint)
     329             :   //   ch, glue = callseq_start ch
     330             :   //   ch, glue = X86::Call ch, glue
     331             :   //   ch, glue = callseq_end ch, glue
     332             :   //   get_return_value ch, glue
     333             :   //
     334             :   // get_return_value can either be a sequence of CopyFromReg instructions
     335             :   // to grab the return value from the return register(s), or it can be a LOAD
     336             :   // to load a value returned by reference via a stack slot.
     337             : 
     338           0 :   bool HasDef = !SI.CLI.RetTy->isVoidTy();
     339           0 :   if (HasDef) {
     340           0 :     if (CallEnd->getOpcode() == ISD::LOAD)
     341           0 :       CallEnd = CallEnd->getOperand(0).getNode();
     342             :     else
     343           0 :       while (CallEnd->getOpcode() == ISD::CopyFromReg)
     344           0 :         CallEnd = CallEnd->getOperand(0).getNode();
     345             :   }
     346             : 
     347             :   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
     348           0 :   return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
     349             : }
     350             : 
     351             : /// Spill a value incoming to the statepoint. It might be either part of
     352             : /// vmstate
     353             : /// or gcstate. In both cases unconditionally spill it on the stack unless it
     354             : /// is a null constant. Return pair with first element being frame index
     355             : /// containing saved value and second element with outgoing chain from the
     356             : /// emitted store
     357             : static std::pair<SDValue, SDValue>
     358         128 : spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
     359             :                              SelectionDAGBuilder &Builder) {
     360         128 :   SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
     361             : 
     362             :   // Emit new store if we didn't do it for this ptr before
     363         128 :   if (!Loc.getNode()) {
     364          60 :     Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
     365         120 :                                                        Builder);
     366          60 :     int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
     367             :     // We use TargetFrameIndex so that isel will not select it into LEA
     368          60 :     Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
     369             : 
     370             :     // TODO: We can create TokenFactor node instead of
     371             :     //       chaining stores one after another, this may allow
     372             :     //       a bit more optimal scheduling for them
     373             : 
     374             : #ifndef NDEBUG
     375             :     // Right now we always allocate spill slots that are of the same
     376             :     // size as the value we're about to spill (the size of spillee can
     377             :     // vary since we spill vectors of pointers too).  At some point we
     378             :     // can consider allowing spills of smaller values to larger slots
     379             :     // (i.e. change the '==' in the assert below to a '>=').
     380             :     MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
     381             :     assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
     382             :            "Bad spill:  stack slot does not match!");
     383             : #endif
     384             : 
     385         120 :     Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
     386             :                                  MachinePointerInfo::getFixedStack(
     387         180 :                                      Builder.DAG.getMachineFunction(), Index));
     388             : 
     389          60 :     Builder.StatepointLowering.setLocation(Incoming, Loc);
     390             :   }
     391             : 
     392             :   assert(Loc.getNode());
     393         128 :   return std::make_pair(Loc, Chain);
     394             : }
     395             : 
     396             : /// Lower a single value incoming to a statepoint node.  This value can be
     397             : /// either a deopt value or a gc value, the handling is the same.  We special
     398             : /// case constants and allocas, then fall back to spilling if required.
     399         247 : static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
     400             :                                          SmallVectorImpl<SDValue> &Ops,
     401             :                                          SelectionDAGBuilder &Builder) {
     402         247 :   SDValue Chain = Builder.getRoot();
     403             : 
     404             :   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
     405             :     // If the original value was a constant, make sure it gets recorded as
     406             :     // such in the stackmap.  This is required so that the consumer can
     407             :     // parse any internal format to the deopt state.  It also handles null
     408             :     // pointers and other constant pointers in GC states.  Note the constant
     409             :     // vectors do not appear to actually hit this path and that anything larger
     410             :     // than an i64 value (not type!) will fail asserts here.
     411         148 :     pushStackMapConstant(Ops, Builder, C->getSExtValue());
     412             :   } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
     413             :     // This handles allocas as arguments to the statepoint (this is only
     414             :     // really meaningful for a deopt value.  For GC, we'd be trying to
     415             :     // relocate the address of the alloca itself?)
     416             :     assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
     417             :            "Incoming value is a frame index!");
     418           6 :     Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
     419           3 :                                                   Builder.getFrameIndexTy()));
     420         170 :   } else if (LiveInOnly) {
     421             :     // If this value is live in (not live-on-return, or live-through), we can
     422             :     // treat it the same way patchpoint treats it's "live in" values.  We'll
     423             :     // end up folding some of these into stack references, but they'll be
     424             :     // handled by the register allocator.  Note that we do not have the notion
     425             :     // of a late use so these values might be placed in registers which are
     426             :     // clobbered by the call.  This is fine for live-in.
     427          42 :     Ops.push_back(Incoming);
     428             :   } else {
     429             :     // Otherwise, locate a spill slot and explicitly spill it so it
     430             :     // can be found by the runtime later.  We currently do not support
     431             :     // tracking values through callee saved registers to their eventual
     432             :     // spill location.  This would be a useful optimization, but would
     433             :     // need to be optional since it requires a lot of complexity on the
     434             :     // runtime side which not all would support.
     435         128 :     auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
     436         128 :     Ops.push_back(Res.first);
     437         128 :     Chain = Res.second;
     438             :   }
     439             : 
     440         247 :   Builder.DAG.setRoot(Chain);
     441         247 : }
     442             : 
     443             : /// Lower deopt state and gc pointer arguments of the statepoint.  The actual
     444             : /// lowering is described in lowerIncomingStatepointValue.  This function is
     445             : /// responsible for lowering everything in the right position and playing some
     446             : /// tricks to avoid redundant stack manipulation where possible.  On
     447             : /// completion, 'Ops' will contain ready to use operands for machine code
     448             : /// statepoint. The chain nodes will have already been created and the DAG root
     449             : /// will be set to the last value spilled (if any were).
     450             : static void
     451          70 : lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
     452             :                         SelectionDAGBuilder::StatepointLoweringInfo &SI,
     453             :                         SelectionDAGBuilder &Builder) {
     454             :   // Lower the deopt and gc arguments for this statepoint.  Layout will be:
     455             :   // deopt argument length, deopt arguments.., gc arguments...
     456             : #ifndef NDEBUG
     457             :   if (auto *GFI = Builder.GFI) {
     458             :     // Check that each of the gc pointer and bases we've gotten out of the
     459             :     // safepoint is something the strategy thinks might be a pointer (or vector
     460             :     // of pointers) into the GC heap.  This is basically just here to help catch
     461             :     // errors during statepoint insertion. TODO: This should actually be in the
     462             :     // Verifier, but we can't get to the GCStrategy from there (yet).
     463             :     GCStrategy &S = GFI->getStrategy();
     464             :     for (const Value *V : SI.Bases) {
     465             :       auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
     466             :       if (Opt.hasValue()) {
     467             :         assert(Opt.getValue() &&
     468             :                "non gc managed base pointer found in statepoint");
     469             :       }
     470             :     }
     471             :     for (const Value *V : SI.Ptrs) {
     472             :       auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
     473             :       if (Opt.hasValue()) {
     474             :         assert(Opt.getValue() &&
     475             :                "non gc managed derived pointer found in statepoint");
     476             :       }
     477             :     }
     478             :     assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
     479             :   } else {
     480             :     assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
     481             :     assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
     482             :   }
     483             : #endif
     484             : 
     485             :   // Figure out what lowering strategy we're going to use for each part
     486             :   // Note: Is is conservatively correct to lower both "live-in" and "live-out"
     487             :   // as "live-through". A "live-through" variable is one which is "live-in",
     488             :   // "live-out", and live throughout the lifetime of the call (i.e. we can find
     489             :   // it from any PC within the transitive callee of the statepoint).  In
     490             :   // particular, if the callee spills callee preserved registers we may not
     491             :   // be able to find a value placed in that register during the call.  This is
     492             :   // fine for live-out, but not for live-through.  If we were willing to make
     493             :   // assumptions about the code generator producing the callee, we could
     494             :   // potentially allow live-through values in callee saved registers.
     495             :   const bool LiveInDeopt =
     496          70 :     SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
     497             : 
     498             :   auto isGCValue =[&](const Value *V) {
     499             :     return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
     500             :   };
     501             : 
     502             :   // Before we actually start lowering (and allocating spill slots for values),
     503             :   // reserve any stack slots which we judge to be profitable to reuse for a
     504             :   // particular value.  This is purely an optimization over the code below and
     505             :   // doesn't change semantics at all.  It is important for performance that we
     506             :   // reserve slots for both deopt and gc values before lowering either.
     507         201 :   for (const Value *V : SI.DeoptState) {
     508         131 :     if (!LiveInDeopt || isGCValue(V))
     509          89 :       reservePreviousStackSlotForValue(V, Builder);
     510             :   }
     511         128 :   for (unsigned i = 0; i < SI.Bases.size(); ++i) {
     512          58 :     reservePreviousStackSlotForValue(SI.Bases[i], Builder);
     513          58 :     reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
     514             :   }
     515             : 
     516             :   // First, prefix the list with the number of unique values to be
     517             :   // lowered.  Note that this is the number of *Values* not the
     518             :   // number of SDValues required to lower them.
     519          70 :   const int NumVMSArgs = SI.DeoptState.size();
     520          70 :   pushStackMapConstant(Ops, Builder, NumVMSArgs);
     521             : 
     522             :   // The vm state arguments are lowered in an opaque manner.  We do not know
     523             :   // what type of values are contained within.
     524         201 :   for (const Value *V : SI.DeoptState) {
     525         131 :     SDValue Incoming = Builder.getValue(V);
     526         131 :     const bool LiveInValue = LiveInDeopt && !isGCValue(V);
     527         131 :     lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder);
     528             :   }
     529             : 
     530             :   // Finally, go ahead and lower all the gc arguments.  There's no prefixed
     531             :   // length for this one.  After lowering, we'll have the base and pointer
     532             :   // arrays interwoven with each (lowered) base pointer immediately followed by
     533             :   // it's (lowered) derived pointer.  i.e
     534             :   // (base[0], ptr[0], base[1], ptr[1], ...)
     535         128 :   for (unsigned i = 0; i < SI.Bases.size(); ++i) {
     536          58 :     const Value *Base = SI.Bases[i];
     537          58 :     lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
     538             :                                  Ops, Builder);
     539             : 
     540          58 :     const Value *Ptr = SI.Ptrs[i];
     541          58 :     lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
     542             :                                  Ops, Builder);
     543             :   }
     544             : 
     545             :   // If there are any explicit spill slots passed to the statepoint, record
     546             :   // them, but otherwise do not do anything special.  These are user provided
     547             :   // allocas and give control over placement to the consumer.  In this case,
     548             :   // it is the contents of the slot which may get updated, not the pointer to
     549             :   // the alloca
     550         133 :   for (Value *V : SI.GCArgs) {
     551          63 :     SDValue Incoming = Builder.getValue(V);
     552             :     if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
     553             :       // This handles allocas as arguments to the statepoint
     554             :       assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
     555             :              "Incoming value is a frame index!");
     556           4 :       Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
     557           2 :                                                     Builder.getFrameIndexTy()));
     558             :     }
     559             :   }
     560             : 
     561             :   // Record computed locations for all lowered values.
     562             :   // This can not be embedded in lowering loops as we need to record *all*
     563             :   // values, while previous loops account only values with unique SDValues.
     564          70 :   const Instruction *StatepointInstr = SI.StatepointInstr;
     565          70 :   auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
     566             : 
     567         128 :   for (const GCRelocateInst *Relocate : SI.GCRelocates) {
     568          58 :     const Value *V = Relocate->getDerivedPtr();
     569          58 :     SDValue SDV = Builder.getValue(V);
     570          58 :     SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
     571             : 
     572          58 :     if (Loc.getNode()) {
     573          51 :       SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
     574             :     } else {
     575             :       // Record value as visited, but not spilled. This is case for allocas
     576             :       // and constants. For this values we can avoid emitting spill load while
     577             :       // visiting corresponding gc_relocate.
     578             :       // Actually we do not need to record them in this map at all.
     579             :       // We do this only to check that we are not relocating any unvisited
     580             :       // value.
     581           7 :       SpillMap.SlotMap[V] = None;
     582             : 
     583             :       // Default llvm mechanisms for exporting values which are used in
     584             :       // different basic blocks does not work for gc relocates.
     585             :       // Note that it would be incorrect to teach llvm that all relocates are
     586             :       // uses of the corresponding values so that it would automatically
     587             :       // export them. Relocates of the spilled values does not use original
     588             :       // value.
     589           7 :       if (Relocate->getParent() != StatepointInstr->getParent())
     590           3 :         Builder.ExportFromCurrentBlock(V);
     591             :     }
     592             :   }
     593          70 : }
     594             : 
     595          70 : SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
     596             :     SelectionDAGBuilder::StatepointLoweringInfo &SI) {
     597             :   // The basic scheme here is that information about both the original call and
     598             :   // the safepoint is encoded in the CallInst.  We create a temporary call and
     599             :   // lower it, then reverse engineer the calling sequence.
     600             : 
     601             :   NumOfStatepoints++;
     602             :   // Clear state
     603          70 :   StatepointLowering.startNewStatepoint(*this);
     604             : 
     605             : #ifndef NDEBUG
     606             :   // We schedule gc relocates before removeDuplicateGCPtrs since we _will_
     607             :   // encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
     608             :   for (auto *Reloc : SI.GCRelocates)
     609             :     if (Reloc->getParent() == SI.StatepointInstr->getParent())
     610             :       StatepointLowering.scheduleRelocCall(*Reloc);
     611             : #endif
     612             : 
     613             :   // Remove any redundant llvm::Values which map to the same SDValue as another
     614             :   // input.  Also has the effect of removing duplicates in the original
     615             :   // llvm::Value input list as well.  This is a useful optimization for
     616             :   // reducing the size of the StackMap section.  It has no other impact.
     617          70 :   removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
     618          70 :                         FuncInfo.StatepointSpillMaps[SI.StatepointInstr]);
     619             :   assert(SI.Bases.size() == SI.Ptrs.size() &&
     620             :          SI.Ptrs.size() == SI.GCRelocates.size());
     621             : 
     622             :   // Lower statepoint vmstate and gcstate arguments
     623             :   SmallVector<SDValue, 10> LoweredMetaArgs;
     624          70 :   lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this);
     625             : 
     626             :   // Now that we've emitted the spills, we need to update the root so that the
     627             :   // call sequence is ordered correctly.
     628          70 :   SI.CLI.setChain(getRoot());
     629             : 
     630             :   // Get call node, we will replace it later with statepoint
     631             :   SDValue ReturnVal;
     632             :   SDNode *CallNode;
     633             :   std::tie(ReturnVal, CallNode) =
     634          70 :       lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
     635             : 
     636             :   // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
     637             :   // nodes with all the appropriate arguments and return values.
     638             : 
     639             :   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
     640          70 :   SDValue Chain = CallNode->getOperand(0);
     641             : 
     642          70 :   SDValue Glue;
     643             :   bool CallHasIncomingGlue = CallNode->getGluedNode();
     644             :   if (CallHasIncomingGlue) {
     645             :     // Glue is always last operand
     646          10 :     Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
     647             :   }
     648             : 
     649             :   // Build the GC_TRANSITION_START node if necessary.
     650             :   //
     651             :   // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
     652             :   // order in which they appear in the call to the statepoint intrinsic. If
     653             :   // any of the operands is a pointer-typed, that operand is immediately
     654             :   // followed by a SRCVALUE for the pointer that may be used during lowering
     655             :   // (e.g. to form MachinePointerInfo values for loads/stores).
     656             :   const bool IsGCTransition =
     657          70 :       (SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) ==
     658             :       (uint64_t)StatepointFlags::GCTransition;
     659          70 :   if (IsGCTransition) {
     660             :     SmallVector<SDValue, 8> TSOps;
     661             : 
     662             :     // Add chain
     663           8 :     TSOps.push_back(Chain);
     664             : 
     665             :     // Add GC transition arguments
     666           9 :     for (const Value *V : SI.GCTransitionArgs) {
     667           1 :       TSOps.push_back(getValue(V));
     668           2 :       if (V->getType()->isPointerTy())
     669           1 :         TSOps.push_back(DAG.getSrcValue(V));
     670             :     }
     671             : 
     672             :     // Add glue if necessary
     673           8 :     if (CallHasIncomingGlue)
     674           2 :       TSOps.push_back(Glue);
     675             : 
     676          16 :     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
     677             : 
     678             :     SDValue GCTransitionStart =
     679          24 :         DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
     680             : 
     681           8 :     Chain = GCTransitionStart.getValue(0);
     682           8 :     Glue = GCTransitionStart.getValue(1);
     683             :   }
     684             : 
     685             :   // TODO: Currently, all of these operands are being marked as read/write in
     686             :   // PrologEpilougeInserter.cpp, we should special case the VMState arguments
     687             :   // and flags to be read-only.
     688             :   SmallVector<SDValue, 40> Ops;
     689             : 
     690             :   // Add the <id> and <numBytes> constants.
     691         140 :   Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
     692          70 :   Ops.push_back(
     693         140 :       DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
     694             : 
     695             :   // Calculate and push starting position of vmstate arguments
     696             :   // Get number of arguments incoming directly into call node
     697             :   unsigned NumCallRegArgs =
     698         140 :       CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
     699         140 :   Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
     700             : 
     701             :   // Add call target
     702          70 :   SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
     703          70 :   Ops.push_back(CallTarget);
     704             : 
     705             :   // Add call arguments
     706             :   // Get position of register mask in the call
     707             :   SDNode::op_iterator RegMaskIt;
     708          70 :   if (CallHasIncomingGlue)
     709          20 :     RegMaskIt = CallNode->op_end() - 2;
     710             :   else
     711         120 :     RegMaskIt = CallNode->op_end() - 1;
     712         140 :   Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
     713             : 
     714             :   // Add a constant argument for the calling convention
     715          70 :   pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
     716             : 
     717             :   // Add a constant argument for the flags
     718          70 :   uint64_t Flags = SI.StatepointFlags;
     719             :   assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
     720             :          "Unknown flag used");
     721          70 :   pushStackMapConstant(Ops, *this, Flags);
     722             : 
     723             :   // Insert all vmstate and gcstate arguments
     724          70 :   Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
     725             : 
     726             :   // Add register mask from call node
     727          70 :   Ops.push_back(*RegMaskIt);
     728             : 
     729             :   // Add chain
     730          70 :   Ops.push_back(Chain);
     731             : 
     732             :   // Same for the glue, but we add it only if original call had it
     733          70 :   if (Glue.getNode())
     734          16 :     Ops.push_back(Glue);
     735             : 
     736             :   // Compute return values.  Provide a glue output since we consume one as
     737             :   // input.  This allows someone else to chain off us as needed.
     738         140 :   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
     739             : 
     740             :   SDNode *StatepointMCNode =
     741         210 :       DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
     742             : 
     743             :   SDNode *SinkNode = StatepointMCNode;
     744             : 
     745             :   // Build the GC_TRANSITION_END node if necessary.
     746             :   //
     747             :   // See the comment above regarding GC_TRANSITION_START for the layout of
     748             :   // the operands to the GC_TRANSITION_END node.
     749          70 :   if (IsGCTransition) {
     750             :     SmallVector<SDValue, 8> TEOps;
     751             : 
     752             :     // Add chain
     753           8 :     TEOps.push_back(SDValue(StatepointMCNode, 0));
     754             : 
     755             :     // Add GC transition arguments
     756           9 :     for (const Value *V : SI.GCTransitionArgs) {
     757           1 :       TEOps.push_back(getValue(V));
     758           2 :       if (V->getType()->isPointerTy())
     759           1 :         TEOps.push_back(DAG.getSrcValue(V));
     760             :     }
     761             : 
     762             :     // Add glue
     763           8 :     TEOps.push_back(SDValue(StatepointMCNode, 1));
     764             : 
     765          16 :     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
     766             : 
     767             :     SDValue GCTransitionStart =
     768          24 :         DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
     769             : 
     770           8 :     SinkNode = GCTransitionStart.getNode();
     771             :   }
     772             : 
     773             :   // Replace original call
     774          70 :   DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
     775             :   // Remove original call node
     776          70 :   DAG.DeleteNode(CallNode);
     777             : 
     778             :   // DON'T set the root - under the assumption that it's already set past the
     779             :   // inserted node we created.
     780             : 
     781             :   // TODO: A better future implementation would be to emit a single variable
     782             :   // argument, variable return value STATEPOINT node here and then hookup the
     783             :   // return value of each gc.relocate to the respective output of the
     784             :   // previously emitted STATEPOINT value.  Unfortunately, this doesn't appear
     785             :   // to actually be possible today.
     786             : 
     787          70 :   return ReturnVal;
     788             : }
     789             : 
     790             : void
     791          67 : SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
     792             :                                      const BasicBlock *EHPadBB /*= nullptr*/) {
     793             :   assert(ISP.getCallSite().getCallingConv() != CallingConv::AnyReg &&
     794             :          "anyregcc is not supported on statepoints!");
     795             : 
     796             : #ifndef NDEBUG
     797             :   // If this is a malformed statepoint, report it early to simplify debugging.
     798             :   // This should catch any IR level mistake that's made when constructing or
     799             :   // transforming statepoints.
     800             :   ISP.verify();
     801             : 
     802             :   // Check that the associated GCStrategy expects to encounter statepoints.
     803             :   assert(GFI->getStrategy().useStatepoints() &&
     804             :          "GCStrategy does not expect to encounter statepoints");
     805             : #endif
     806             : 
     807          67 :   SDValue ActualCallee;
     808             : 
     809          67 :   if (ISP.getNumPatchBytes() > 0) {
     810             :     // If we've been asked to emit a nop sequence instead of a call instruction
     811             :     // for this statepoint then don't lower the call target, but use a constant
     812             :     // `null` instead.  Not lowering the call target lets statepoint clients get
     813             :     // away without providing a physical address for the symbolic call target at
     814             :     // link time.
     815             : 
     816           1 :     const auto &TLI = DAG.getTargetLoweringInfo();
     817           1 :     const auto &DL = DAG.getDataLayout();
     818             : 
     819           1 :     unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
     820           3 :     ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
     821             :   } else {
     822          66 :     ActualCallee = getValue(ISP.getCalledValue());
     823             :   }
     824             : 
     825         134 :   StatepointLoweringInfo SI(DAG);
     826         134 :   populateCallLoweringInfo(SI.CLI, ISP.getCallSite(),
     827             :                            ImmutableStatepoint::CallArgsBeginPos,
     828          67 :                            ISP.getNumCallArgs(), ActualCallee,
     829             :                            ISP.getActualReturnType(), false /* IsPatchPoint */);
     830             : 
     831         200 :   for (const GCRelocateInst *Relocate : ISP.getRelocates()) {
     832          66 :     SI.GCRelocates.push_back(Relocate);
     833          66 :     SI.Bases.push_back(Relocate->getBasePtr());
     834          66 :     SI.Ptrs.push_back(Relocate->getDerivedPtr());
     835             :   }
     836             : 
     837          67 :   SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
     838          67 :   SI.StatepointInstr = ISP.getInstruction();
     839          67 :   SI.GCTransitionArgs =
     840             :       ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
     841          67 :   SI.ID = ISP.getID();
     842          67 :   SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end());
     843          67 :   SI.StatepointFlags = ISP.getFlags();
     844          67 :   SI.NumPatchBytes = ISP.getNumPatchBytes();
     845          67 :   SI.EHPadBB = EHPadBB;
     846             : 
     847          67 :   SDValue ReturnValue = LowerAsSTATEPOINT(SI);
     848             : 
     849             :   // Export the result value if needed
     850          67 :   const GCResultInst *GCResult = ISP.getGCResult();
     851          67 :   Type *RetTy = ISP.getActualReturnType();
     852          67 :   if (!RetTy->isVoidTy() && GCResult) {
     853          48 :     if (GCResult->getParent() != ISP.getCallSite().getParent()) {
     854             :       // Result value will be used in a different basic block so we need to
     855             :       // export it now.  Default exporting mechanism will not work here because
     856             :       // statepoint call has a different type than the actual call. It means
     857             :       // that by default llvm will create export register of the wrong type
     858             :       // (always i32 in our case). So instead we need to create export register
     859             :       // with correct type manually.
     860             :       // TODO: To eliminate this problem we can remove gc.result intrinsics
     861             :       //       completely and make statepoint call to return a tuple.
     862           2 :       unsigned Reg = FuncInfo.CreateRegs(RetTy);
     863           4 :       RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
     864           2 :                        DAG.getDataLayout(), Reg, RetTy,
     865           2 :                        ISP.getCallSite().getCallingConv());
     866           2 :       SDValue Chain = DAG.getEntryNode();
     867             : 
     868           4 :       RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
     869           2 :       PendingExports.push_back(Chain);
     870           2 :       FuncInfo.ValueMap[ISP.getInstruction()] = Reg;
     871             :     } else {
     872             :       // Result value will be used in a same basic block. Don't export it or
     873             :       // perform any explicit register copies.
     874             :       // We'll replace the actuall call node shortly. gc_result will grab
     875             :       // this value.
     876          22 :       setValue(ISP.getInstruction(), ReturnValue);
     877             :     }
     878             :   } else {
     879             :     // The token value is never used from here on, just generate a poison value
     880         129 :     setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
     881             :   }
     882          67 : }
     883             : 
     884           3 : void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
     885             :     ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB,
     886             :     bool VarArgDisallowed, bool ForceVoidReturnTy) {
     887           6 :   StatepointLoweringInfo SI(DAG);
     888           3 :   unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin();
     889           6 :   populateCallLoweringInfo(
     890             :       SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee,
     891           0 :       ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(),
     892             :       false);
     893           3 :   if (!VarArgDisallowed)
     894           3 :     SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg();
     895             : 
     896           3 :   auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt);
     897             : 
     898             :   unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID;
     899             : 
     900           3 :   auto SD = parseStatepointDirectivesFromAttrs(CS.getAttributes());
     901           3 :   SI.ID = SD.StatepointID.getValueOr(DefaultID);
     902           3 :   SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
     903             : 
     904           3 :   SI.DeoptState =
     905             :       ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
     906           3 :   SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
     907           3 :   SI.EHPadBB = EHPadBB;
     908             : 
     909             :   // NB! The GC arguments are deliberately left empty.
     910             : 
     911           3 :   if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
     912             :     const Instruction *Inst = CS.getInstruction();
     913           0 :     ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal);
     914           0 :     setValue(Inst, ReturnVal);
     915             :   }
     916           3 : }
     917             : 
     918           3 : void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle(
     919             :     ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) {
     920           3 :   LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB,
     921             :                                    /* VarArgDisallowed = */ false,
     922             :                                    /* ForceVoidReturnTy  = */ false);
     923           3 : }
     924             : 
     925          24 : void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
     926             :   // The result value of the gc_result is simply the result of the actual
     927             :   // call.  We've already emitted this, so just grab the value.
     928          24 :   const Instruction *I = CI.getStatepoint();
     929             : 
     930          24 :   if (I->getParent() != CI.getParent()) {
     931             :     // Statepoint is in different basic block so we should have stored call
     932             :     // result in a virtual register.
     933             :     // We can not use default getValue() functionality to copy value from this
     934             :     // register because statepoint and actual call return types can be
     935             :     // different, and getValue() will use CopyFromReg of the wrong type,
     936             :     // which is always i32 in our case.
     937           2 :     PointerType *CalleeType = cast<PointerType>(
     938           2 :         ImmutableStatepoint(I).getCalledValue()->getType());
     939             :     Type *RetTy =
     940           2 :         cast<FunctionType>(CalleeType->getElementType())->getReturnType();
     941           2 :     SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
     942             : 
     943             :     assert(CopyFromReg.getNode());
     944           2 :     setValue(&CI, CopyFromReg);
     945             :   } else {
     946          22 :     setValue(&CI, getValue(I));
     947             :   }
     948          24 : }
     949             : 
     950          66 : void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
     951             : #ifndef NDEBUG
     952             :   // Consistency check
     953             :   // We skip this check for relocates not in the same basic block as their
     954             :   // statepoint. It would be too expensive to preserve validation info through
     955             :   // different basic blocks.
     956             :   if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
     957             :     StatepointLowering.relocCallVisited(Relocate);
     958             : 
     959             :   auto *Ty = Relocate.getType()->getScalarType();
     960             :   if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
     961             :     assert(*IsManaged && "Non gc managed pointer relocated!");
     962             : #endif
     963             : 
     964          66 :   const Value *DerivedPtr = Relocate.getDerivedPtr();
     965          66 :   SDValue SD = getValue(DerivedPtr);
     966             : 
     967          66 :   auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
     968          66 :   auto SlotIt = SpillMap.find(DerivedPtr);
     969             :   assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
     970          66 :   Optional<int> DerivedPtrLocation = SlotIt->second;
     971             : 
     972             :   // We didn't need to spill these special cases (constants and allocas).
     973             :   // See the handling in spillIncomingValueForStatepoint for detail.
     974          66 :   if (!DerivedPtrLocation) {
     975           8 :     setValue(&Relocate, SD);
     976             :     return;
     977             :   }
     978             : 
     979             :   SDValue SpillSlot =
     980          58 :       DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
     981             : 
     982             :   // Be conservative: flush all pending loads
     983             :   // TODO: Probably we can be less restrictive on this,
     984             :   // it may allow more scheduling opportunities.
     985          58 :   SDValue Chain = getRoot();
     986             : 
     987             :   SDValue SpillLoad =
     988          58 :       DAG.getLoad(DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
     989             :                                                            Relocate.getType()),
     990          58 :                   getCurSDLoc(), Chain, SpillSlot,
     991             :                   MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
     992         116 :                                                     *DerivedPtrLocation));
     993             : 
     994             :   // Again, be conservative, don't emit pending loads
     995          58 :   DAG.setRoot(SpillLoad.getValue(1));
     996             : 
     997             :   assert(SpillLoad.getNode());
     998          58 :   setValue(&Relocate, SpillLoad);
     999             : }
    1000             : 
    1001           0 : void SelectionDAGBuilder::LowerDeoptimizeCall(const CallInst *CI) {
    1002           0 :   const auto &TLI = DAG.getTargetLoweringInfo();
    1003             :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
    1004           0 :                                          TLI.getPointerTy(DAG.getDataLayout()));
    1005             : 
    1006             :   // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
    1007             :   // call.  We also do not lower the return value to any virtual register, and
    1008             :   // change the immediately following return to a trap instruction.
    1009           0 :   LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
    1010             :                                    /* VarArgDisallowed = */ true,
    1011             :                                    /* ForceVoidReturnTy = */ true);
    1012           0 : }
    1013             : 
    1014           0 : void SelectionDAGBuilder::LowerDeoptimizingReturn() {
    1015             :   // We do not lower the return value from llvm.deoptimize to any virtual
    1016             :   // register, and change the immediately following return to a trap
    1017             :   // instruction.
    1018           0 :   if (DAG.getTarget().Options.TrapUnreachable)
    1019             :     DAG.setRoot(
    1020           0 :         DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
    1021           0 : }

Generated by: LCOV version 1.13