LCOV - code coverage report
Current view: top level - lib/CodeGen/SelectionDAG - StatepointLowering.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 303 325 93.2 %
Date: 2017-09-14 15:23:50 Functions: 18 20 90.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- StatepointLowering.cpp - SDAGBuilder's statepoint code -----------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file includes support code use by SelectionDAGBuilder when lowering a
      11             : // statepoint sequence in SelectionDAG IR.
      12             : //
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "StatepointLowering.h"
      16             : #include "SelectionDAGBuilder.h"
      17             : #include "llvm/ADT/SmallSet.h"
      18             : #include "llvm/ADT/Statistic.h"
      19             : #include "llvm/CodeGen/FunctionLoweringInfo.h"
      20             : #include "llvm/CodeGen/GCMetadata.h"
      21             : #include "llvm/CodeGen/GCStrategy.h"
      22             : #include "llvm/CodeGen/MachineFrameInfo.h"
      23             : #include "llvm/CodeGen/SelectionDAG.h"
      24             : #include "llvm/CodeGen/StackMaps.h"
      25             : #include "llvm/IR/CallingConv.h"
      26             : #include "llvm/IR/Instructions.h"
      27             : #include "llvm/IR/IntrinsicInst.h"
      28             : #include "llvm/IR/Intrinsics.h"
      29             : #include "llvm/IR/Statepoint.h"
      30             : #include "llvm/Target/TargetLowering.h"
      31             : #include <algorithm>
      32             : using namespace llvm;
      33             : 
      34             : #define DEBUG_TYPE "statepoint-lowering"
      35             : 
      36             : STATISTIC(NumSlotsAllocatedForStatepoints,
      37             :           "Number of stack slots allocated for statepoints");
      38             : STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");
      39             : STATISTIC(StatepointMaxSlotsRequired,
      40             :           "Maximum number of stack slots required for a singe statepoint");
      41             : 
      42         281 : static void pushStackMapConstant(SmallVectorImpl<SDValue>& Ops,
      43             :                                  SelectionDAGBuilder &Builder, uint64_t Value) {
      44         562 :   SDLoc L = Builder.getCurSDLoc();
      45         562 :   Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,
      46         281 :                                               MVT::i64));
      47         843 :   Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));
      48         281 : }
      49             : 
      50          69 : void StatepointLoweringState::startNewStatepoint(SelectionDAGBuilder &Builder) {
      51             :   // Consistency check
      52             :   assert(PendingGCRelocateCalls.empty() &&
      53             :          "Trying to visit statepoint before finished processing previous one");
      54          69 :   Locations.clear();
      55          69 :   NextSlotToAllocate = 0;
      56             :   // Need to resize this on each safepoint - we need the two to stay in sync and
      57             :   // the clear patterns of a SelectionDAGBuilder have no relation to
      58             :   // FunctionLoweringInfo.  Also need to ensure used bits get cleared.
      59          69 :   AllocatedStackSlots.clear();
      60         138 :   AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());
      61          69 : }
      62             : 
      63      280070 : void StatepointLoweringState::clear() {
      64      280070 :   Locations.clear();
      65      280070 :   AllocatedStackSlots.clear();
      66             :   assert(PendingGCRelocateCalls.empty() &&
      67             :          "cleared before statepoint sequence completed");
      68      280070 : }
      69             : 
      70             : SDValue
      71          60 : StatepointLoweringState::allocateStackSlot(EVT ValueType,
      72             :                                            SelectionDAGBuilder &Builder) {
      73          60 :   NumSlotsAllocatedForStatepoints++;
      74          60 :   MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
      75             : 
      76          60 :   unsigned SpillSize = ValueType.getSizeInBits() / 8;
      77             :   assert((SpillSize * 8) == ValueType.getSizeInBits() && "Size not in bytes?");
      78             : 
      79             :   // First look for a previously created stack slot which is not in
      80             :   // use (accounting for the fact arbitrary slots may already be
      81             :   // reserved), or to create a new stack slot and use it.
      82             : 
      83         120 :   const size_t NumSlots = AllocatedStackSlots.size();
      84             :   assert(NextSlotToAllocate <= NumSlots && "Broken invariant");
      85             : 
      86             :   assert(AllocatedStackSlots.size() ==
      87             :          Builder.FuncInfo.StatepointStackSlots.size() &&
      88             :          "Broken invariant");
      89             : 
      90         110 :   for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
      91          72 :     if (!AllocatedStackSlots.test(NextSlotToAllocate)) {
      92          26 :       const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];
      93          13 :       if (MFI.getObjectSize(FI) == SpillSize) {
      94          11 :         AllocatedStackSlots.set(NextSlotToAllocate);
      95             :         // TODO: Is ValueType the right thing to use here?
      96          11 :         return Builder.DAG.getFrameIndex(FI, ValueType);
      97             :       }
      98             :     }
      99             :   }
     100             : 
     101             :   // Couldn't find a free slot, so create a new one:
     102             : 
     103          49 :   SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);
     104          49 :   const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
     105          98 :   MFI.markAsStatepointSpillSlotObjectIndex(FI);
     106             : 
     107          49 :   Builder.FuncInfo.StatepointStackSlots.push_back(FI);
     108          98 :   AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);
     109             :   assert(AllocatedStackSlots.size() ==
     110             :          Builder.FuncInfo.StatepointStackSlots.size() &&
     111             :          "Broken invariant");
     112             : 
     113          49 :   StatepointMaxSlotsRequired.updateMax(
     114          98 :       Builder.FuncInfo.StatepointStackSlots.size());
     115             : 
     116          49 :   return SpillSlot;
     117             : }
     118             : 
     119             : /// Utility function for reservePreviousStackSlotForValue. Tries to find
     120             : /// stack slot index to which we have spilled value for previous statepoints.
     121             : /// LookUpDepth specifies maximum DFS depth this function is allowed to look.
     122         120 : static Optional<int> findPreviousSpillSlot(const Value *Val,
     123             :                                            SelectionDAGBuilder &Builder,
     124             :                                            int LookUpDepth) {
     125             :   // Can not look any further - give up now
     126         120 :   if (LookUpDepth <= 0)
     127             :     return None;
     128             : 
     129             :   // Spill location is known for gc relocates
     130           9 :   if (const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
     131             :     const auto &SpillMap =
     132          18 :         Builder.FuncInfo.StatepointSpillMaps[Relocate->getStatepoint()];
     133             : 
     134           9 :     auto It = SpillMap.find(Relocate->getDerivedPtr());
     135           9 :     if (It == SpillMap.end())
     136             :       return None;
     137             : 
     138           9 :     return It->second;
     139             :   }
     140             : 
     141             :   // Look through bitcast instructions.
     142           3 :   if (const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
     143           6 :     return findPreviousSpillSlot(Cast->getOperand(0), Builder, LookUpDepth - 1);
     144             : 
     145             :   // Look through phi nodes
     146             :   // All incoming values should have same known stack slot, otherwise result
     147             :   // is unknown.
     148           0 :   if (const PHINode *Phi = dyn_cast<PHINode>(Val)) {
     149           0 :     Optional<int> MergedResult = None;
     150             : 
     151           0 :     for (auto &IncomingValue : Phi->incoming_values()) {
     152             :       Optional<int> SpillSlot =
     153           0 :           findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth - 1);
     154           0 :       if (!SpillSlot.hasValue())
     155           0 :         return None;
     156             : 
     157           0 :       if (MergedResult.hasValue() && *MergedResult != *SpillSlot)
     158             :         return None;
     159             : 
     160           0 :       MergedResult = SpillSlot;
     161             :     }
     162             :     return MergedResult;
     163             :   }
     164             : 
     165             :   // TODO: We can do better for PHI nodes. In cases like this:
     166             :   //   ptr = phi(relocated_pointer, not_relocated_pointer)
     167             :   //   statepoint(ptr)
     168             :   // We will return that stack slot for ptr is unknown. And later we might
     169             :   // assign different stack slots for ptr and relocated_pointer. This limits
     170             :   // llvm's ability to remove redundant stores.
     171             :   // Unfortunately it's hard to accomplish in current infrastructure.
     172             :   // We use this function to eliminate spill store completely, while
     173             :   // in example we still need to emit store, but instead of any location
     174             :   // we need to use special "preferred" location.
     175             : 
     176             :   // TODO: handle simple updates.  If a value is modified and the original
     177             :   // value is no longer live, it would be nice to put the modified value in the
     178             :   // same slot.  This allows folding of the memory accesses for some
     179             :   // instructions types (like an increment).
     180             :   //   statepoint (i)
     181             :   //   i1 = i+1
     182             :   //   statepoint (i1)
     183             :   // However we need to be careful for cases like this:
     184             :   //   statepoint(i)
     185             :   //   i1 = i+1
     186             :   //   statepoint(i, i1)
     187             :   // Here we want to reserve spill slot for 'i', but not for 'i+1'. If we just
     188             :   // put handling of simple modifications in this function like it's done
     189             :   // for bitcasts we might end up reserving i's slot for 'i+1' because order in
     190             :   // which we visit values is unspecified.
     191             : 
     192             :   // Don't know any information about this instruction
     193             :   return None;
     194             : }
     195             : 
     196             : /// Try to find existing copies of the incoming values in stack slots used for
     197             : /// statepoint spilling.  If we can find a spill slot for the incoming value,
     198             : /// mark that slot as allocated, and reuse the same slot for this safepoint.
     199             : /// This helps to avoid series of loads and stores that only serve to reshuffle
     200             : /// values on the stack between calls.
     201         205 : static void reservePreviousStackSlotForValue(const Value *IncomingValue,
     202             :                                              SelectionDAGBuilder &Builder) {
     203             : 
     204         205 :   SDValue Incoming = Builder.getValue(IncomingValue);
     205             : 
     206         259 :   if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
     207             :     // We won't need to spill this, so no need to check for previously
     208             :     // allocated stack slots
     209         196 :     return;
     210             :   }
     211             : 
     212         128 :   SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);
     213         128 :   if (OldLocation.getNode())
     214             :     // Duplicates in input
     215             :     return;
     216             : 
     217         117 :   const int LookUpDepth = 6;
     218             :   Optional<int> Index =
     219         126 :       findPreviousSpillSlot(IncomingValue, Builder, LookUpDepth);
     220         117 :   if (!Index.hasValue())
     221             :     return;
     222             : 
     223           9 :   const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;
     224             : 
     225          18 :   auto SlotIt = find(StatepointSlots, *Index);
     226             :   assert(SlotIt != StatepointSlots.end() &&
     227             :          "Value spilled to the unknown stack slot");
     228             : 
     229             :   // This is one of our dedicated lowering slots
     230          18 :   const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
     231          18 :   if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {
     232             :     // stack slot already assigned to someone else, can't use it!
     233             :     // TODO: currently we reserve space for gc arguments after doing
     234             :     // normal allocation for deopt arguments.  We should reserve for
     235             :     // _all_ deopt and gc arguments, then start allocating.  This
     236             :     // will prevent some moves being inserted when vm state changes,
     237             :     // but gc state doesn't between two calls.
     238             :     return;
     239             :   }
     240             :   // Reserve this stack slot
     241          18 :   Builder.StatepointLowering.reserveStackSlot(Offset);
     242             : 
     243             :   // Cache this slot so we find it when going through the normal
     244             :   // assignment loop.
     245             :   SDValue Loc =
     246          36 :       Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());
     247          18 :   Builder.StatepointLowering.setLocation(Incoming, Loc);
     248             : }
     249             : 
     250             : /// Remove any duplicate (as SDValues) from the derived pointer pairs.  This
     251             : /// is not required for correctness.  It's purpose is to reduce the size of
     252             : /// StackMap section.  It has no effect on the number of spill slots required
     253             : /// or the actual lowering.
     254             : static void
     255          69 : removeDuplicateGCPtrs(SmallVectorImpl<const Value *> &Bases,
     256             :                       SmallVectorImpl<const Value *> &Ptrs,
     257             :                       SmallVectorImpl<const GCRelocateInst *> &Relocs,
     258             :                       SelectionDAGBuilder &Builder,
     259             :                       FunctionLoweringInfo::StatepointSpillMap &SSM) {
     260         138 :   DenseMap<SDValue, const Value *> Seen;
     261             : 
     262         276 :   SmallVector<const Value *, 64> NewBases, NewPtrs;
     263         138 :   SmallVector<const GCRelocateInst *, 64> NewRelocs;
     264         204 :   for (size_t i = 0, e = Ptrs.size(); i < e; i++) {
     265         132 :     SDValue SD = Builder.getValue(Ptrs[i]);
     266          66 :     auto SeenIt = Seen.find(SD);
     267             : 
     268         132 :     if (SeenIt == Seen.end()) {
     269             :       // Only add non-duplicates
     270         116 :       NewBases.push_back(Bases[i]);
     271         116 :       NewPtrs.push_back(Ptrs[i]);
     272         116 :       NewRelocs.push_back(Relocs[i]);
     273         174 :       Seen[SD] = Ptrs[i];
     274             :     } else {
     275             :       // Duplicate pointer found, note in SSM and move on:
     276          24 :       SSM.DuplicateMap[Ptrs[i]] = SeenIt->second;
     277             :     }
     278             :   }
     279             :   assert(Bases.size() >= NewBases.size());
     280             :   assert(Ptrs.size() >= NewPtrs.size());
     281             :   assert(Relocs.size() >= NewRelocs.size());
     282          69 :   Bases = NewBases;
     283          69 :   Ptrs = NewPtrs;
     284          69 :   Relocs = NewRelocs;
     285             :   assert(Ptrs.size() == Bases.size());
     286             :   assert(Ptrs.size() == Relocs.size());
     287          69 : }
     288             : 
     289             : /// Extract call from statepoint, lower it and return pointer to the
     290             : /// call node. Also update NodeMap so that getValue(statepoint) will
     291             : /// reference lowered call result
     292          69 : static std::pair<SDValue, SDNode *> lowerCallFromStatepointLoweringInfo(
     293             :     SelectionDAGBuilder::StatepointLoweringInfo &SI,
     294             :     SelectionDAGBuilder &Builder, SmallVectorImpl<SDValue> &PendingExports) {
     295             : 
     296             :   SDValue ReturnValue, CallEndVal;
     297          69 :   std::tie(ReturnValue, CallEndVal) =
     298         207 :       Builder.lowerInvokable(SI.CLI, SI.EHPadBB);
     299          69 :   SDNode *CallEnd = CallEndVal.getNode();
     300             : 
     301             :   // Get a call instruction from the call sequence chain.  Tail calls are not
     302             :   // allowed.  The following code is essentially reverse engineering X86's
     303             :   // LowerCallTo.
     304             :   //
     305             :   // We are expecting DAG to have the following form:
     306             :   //
     307             :   // ch = eh_label (only in case of invoke statepoint)
     308             :   //   ch, glue = callseq_start ch
     309             :   //   ch, glue = X86::Call ch, glue
     310             :   //   ch, glue = callseq_end ch, glue
     311             :   //   get_return_value ch, glue
     312             :   //
     313             :   // get_return_value can either be a sequence of CopyFromReg instructions
     314             :   // to grab the return value from the return register(s), or it can be a LOAD
     315             :   // to load a value returned by reference via a stack slot.
     316             : 
     317         138 :   bool HasDef = !SI.CLI.RetTy->isVoidTy();
     318          69 :   if (HasDef) {
     319          26 :     if (CallEnd->getOpcode() == ISD::LOAD)
     320           0 :       CallEnd = CallEnd->getOperand(0).getNode();
     321             :     else
     322          53 :       while (CallEnd->getOpcode() == ISD::CopyFromReg)
     323          54 :         CallEnd = CallEnd->getOperand(0).getNode();
     324             :   }
     325             : 
     326             :   assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");
     327         207 :   return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());
     328             : }
     329             : 
     330             : /// Spill a value incoming to the statepoint. It might be either part of
     331             : /// vmstate
     332             : /// or gcstate. In both cases unconditionally spill it on the stack unless it
     333             : /// is a null constant. Return pair with first element being frame index
     334             : /// containing saved value and second element with outgoing chain from the
     335             : /// emitted store
     336             : static std::pair<SDValue, SDValue>
     337         128 : spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
     338             :                              SelectionDAGBuilder &Builder) {
     339         128 :   SDValue Loc = Builder.StatepointLowering.getLocation(Incoming);
     340             : 
     341             :   // Emit new store if we didn't do it for this ptr before
     342         128 :   if (!Loc.getNode()) {
     343          60 :     Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),
     344          60 :                                                        Builder);
     345          60 :     int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
     346             :     // We use TargetFrameIndex so that isel will not select it into LEA
     347         180 :     Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
     348             : 
     349             :     // TODO: We can create TokenFactor node instead of
     350             :     //       chaining stores one after another, this may allow
     351             :     //       a bit more optimal scheduling for them
     352             : 
     353             : #ifndef NDEBUG
     354             :     // Right now we always allocate spill slots that are of the same
     355             :     // size as the value we're about to spill (the size of spillee can
     356             :     // vary since we spill vectors of pointers too).  At some point we
     357             :     // can consider allowing spills of smaller values to larger slots
     358             :     // (i.e. change the '==' in the assert below to a '>=').
     359             :     MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();
     360             :     assert((MFI.getObjectSize(Index) * 8) == Incoming.getValueSizeInBits() &&
     361             :            "Bad spill:  stack slot does not match!");
     362             : #endif
     363             : 
     364         240 :     Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,
     365             :                                  MachinePointerInfo::getFixedStack(
     366         120 :                                      Builder.DAG.getMachineFunction(), Index));
     367             : 
     368          60 :     Builder.StatepointLowering.setLocation(Incoming, Loc);
     369             :   }
     370             : 
     371             :   assert(Loc.getNode());
     372         128 :   return std::make_pair(Loc, Chain);
     373             : }
     374             : 
     375             : /// Lower a single value incoming to a statepoint node.  This value can be
     376             : /// either a deopt value or a gc value, the handling is the same.  We special
     377             : /// case constants and allocas, then fall back to spilling if required.
     378         247 : static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
     379             :                                          SmallVectorImpl<SDValue> &Ops,
     380             :                                          SelectionDAGBuilder &Builder) {
     381         247 :   SDValue Chain = Builder.getRoot();
     382             : 
     383          74 :   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
     384             :     // If the original value was a constant, make sure it gets recorded as
     385             :     // such in the stackmap.  This is required so that the consumer can
     386             :     // parse any internal format to the deopt state.  It also handles null
     387             :     // pointers and other constant pointers in GC states.  Note the constant
     388             :     // vectors do not appear to actually hit this path and that anything larger
     389             :     // than an i64 value (not type!) will fail asserts here.
     390          74 :     pushStackMapConstant(Ops, Builder, C->getSExtValue());
     391           3 :   } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
     392             :     // This handles allocas as arguments to the statepoint (this is only
     393             :     // really meaningful for a deopt value.  For GC, we'd be trying to
     394             :     // relocate the address of the alloca itself?)
     395             :     assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
     396             :            "Incoming value is a frame index!");
     397           6 :     Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
     398           6 :                                                   Builder.getFrameIndexTy()));
     399         170 :   } else if (LiveInOnly) {
     400             :     // If this value is live in (not live-on-return, or live-through), we can
     401             :     // treat it the same way patchpoint treats it's "live in" values.  We'll 
     402             :     // end up folding some of these into stack references, but they'll be 
     403             :     // handled by the register allocator.  Note that we do not have the notion
     404             :     // of a late use so these values might be placed in registers which are 
     405             :     // clobbered by the call.  This is fine for live-in.
     406          42 :     Ops.push_back(Incoming);
     407             :   } else {
     408             :     // Otherwise, locate a spill slot and explicitly spill it so it
     409             :     // can be found by the runtime later.  We currently do not support
     410             :     // tracking values through callee saved registers to their eventual
     411             :     // spill location.  This would be a useful optimization, but would
     412             :     // need to be optional since it requires a lot of complexity on the
     413             :     // runtime side which not all would support.
     414         128 :     auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder);
     415         128 :     Ops.push_back(Res.first);
     416         128 :     Chain = Res.second;
     417             :   }
     418             : 
     419         247 :   Builder.DAG.setRoot(Chain);
     420         247 : }
     421             : 
     422             : /// Lower deopt state and gc pointer arguments of the statepoint.  The actual
     423             : /// lowering is described in lowerIncomingStatepointValue.  This function is
     424             : /// responsible for lowering everything in the right position and playing some
     425             : /// tricks to avoid redundant stack manipulation where possible.  On
     426             : /// completion, 'Ops' will contain ready to use operands for machine code
     427             : /// statepoint. The chain nodes will have already been created and the DAG root
     428             : /// will be set to the last value spilled (if any were).
     429             : static void
     430          69 : lowerStatepointMetaArgs(SmallVectorImpl<SDValue> &Ops,
     431             :                         SelectionDAGBuilder::StatepointLoweringInfo &SI,
     432             :                         SelectionDAGBuilder &Builder) {
     433             :   // Lower the deopt and gc arguments for this statepoint.  Layout will be:
     434             :   // deopt argument length, deopt arguments.., gc arguments...
     435             : #ifndef NDEBUG
     436             :   if (auto *GFI = Builder.GFI) {
     437             :     // Check that each of the gc pointer and bases we've gotten out of the
     438             :     // safepoint is something the strategy thinks might be a pointer (or vector
     439             :     // of pointers) into the GC heap.  This is basically just here to help catch
     440             :     // errors during statepoint insertion. TODO: This should actually be in the
     441             :     // Verifier, but we can't get to the GCStrategy from there (yet).
     442             :     GCStrategy &S = GFI->getStrategy();
     443             :     for (const Value *V : SI.Bases) {
     444             :       auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
     445             :       if (Opt.hasValue()) {
     446             :         assert(Opt.getValue() &&
     447             :                "non gc managed base pointer found in statepoint");
     448             :       }
     449             :     }
     450             :     for (const Value *V : SI.Ptrs) {
     451             :       auto Opt = S.isGCManagedPointer(V->getType()->getScalarType());
     452             :       if (Opt.hasValue()) {
     453             :         assert(Opt.getValue() &&
     454             :                "non gc managed derived pointer found in statepoint");
     455             :       }
     456             :     }
     457             :     assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");
     458             :   } else {
     459             :     assert(SI.Bases.empty() && "No gc specified, so cannot relocate pointers!");
     460             :     assert(SI.Ptrs.empty() && "No gc specified, so cannot relocate pointers!");
     461             :   }
     462             : #endif
     463             : 
     464             :   // Figure out what lowering strategy we're going to use for each part
     465             :   // Note: Is is conservatively correct to lower both "live-in" and "live-out"
     466             :   // as "live-through". A "live-through" variable is one which is "live-in",
     467             :   // "live-out", and live throughout the lifetime of the call (i.e. we can find
     468             :   // it from any PC within the transitive callee of the statepoint).  In
     469             :   // particular, if the callee spills callee preserved registers we may not
     470             :   // be able to find a value placed in that register during the call.  This is
     471             :   // fine for live-out, but not for live-through.  If we were willing to make
     472             :   // assumptions about the code generator producing the callee, we could
     473             :   // potentially allow live-through values in callee saved registers.
     474          69 :   const bool LiveInDeopt =
     475          69 :     SI.StatepointFlags & (uint64_t)StatepointFlags::DeoptLiveIn;
     476             : 
     477          84 :   auto isGCValue =[&](const Value *V) {
     478         252 :     return is_contained(SI.Ptrs, V) || is_contained(SI.Bases, V);
     479         153 :   };
     480             :   
     481             :   // Before we actually start lowering (and allocating spill slots for values),
     482             :   // reserve any stack slots which we judge to be profitable to reuse for a
     483             :   // particular value.  This is purely an optimization over the code below and
     484             :   // doesn't change semantics at all.  It is important for performance that we
     485             :   // reserve slots for both deopt and gc values before lowering either.
     486         269 :   for (const Value *V : SI.DeoptState) {
     487         131 :     if (!LiveInDeopt || isGCValue(V))
     488          89 :       reservePreviousStackSlotForValue(V, Builder);
     489             :   }
     490         312 :   for (unsigned i = 0; i < SI.Bases.size(); ++i) {
     491         116 :     reservePreviousStackSlotForValue(SI.Bases[i], Builder);
     492         116 :     reservePreviousStackSlotForValue(SI.Ptrs[i], Builder);
     493             :   }
     494             : 
     495             :   // First, prefix the list with the number of unique values to be
     496             :   // lowered.  Note that this is the number of *Values* not the
     497             :   // number of SDValues required to lower them.
     498          69 :   const int NumVMSArgs = SI.DeoptState.size();
     499          69 :   pushStackMapConstant(Ops, Builder, NumVMSArgs);
     500             : 
     501             :   // The vm state arguments are lowered in an opaque manner.  We do not know
     502             :   // what type of values are contained within.
     503         269 :   for (const Value *V : SI.DeoptState) {
     504         131 :     SDValue Incoming = Builder.getValue(V);
     505         131 :     const bool LiveInValue = LiveInDeopt && !isGCValue(V);
     506         131 :     lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder);
     507             :   }
     508             : 
     509             :   // Finally, go ahead and lower all the gc arguments.  There's no prefixed
     510             :   // length for this one.  After lowering, we'll have the base and pointer
     511             :   // arrays interwoven with each (lowered) base pointer immediately followed by
     512             :   // it's (lowered) derived pointer.  i.e
     513             :   // (base[0], ptr[0], base[1], ptr[1], ...)
     514         312 :   for (unsigned i = 0; i < SI.Bases.size(); ++i) {
     515         116 :     const Value *Base = SI.Bases[i];
     516          58 :     lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false,
     517             :                                  Ops, Builder);
     518             : 
     519         116 :     const Value *Ptr = SI.Ptrs[i];
     520          58 :     lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false,
     521             :                                  Ops, Builder);
     522             :   }
     523             : 
     524             :   // If there are any explicit spill slots passed to the statepoint, record
     525             :   // them, but otherwise do not do anything special.  These are user provided
     526             :   // allocas and give control over placement to the consumer.  In this case,
     527             :   // it is the contents of the slot which may get updated, not the pointer to
     528             :   // the alloca
     529         201 :   for (Value *V : SI.GCArgs) {
     530          63 :     SDValue Incoming = Builder.getValue(V);
     531           2 :     if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Incoming)) {
     532             :       // This handles allocas as arguments to the statepoint
     533             :       assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&
     534             :              "Incoming value is a frame index!");
     535           4 :       Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),
     536           4 :                                                     Builder.getFrameIndexTy()));
     537             :     }
     538             :   }
     539             : 
     540             :   // Record computed locations for all lowered values.
     541             :   // This can not be embedded in lowering loops as we need to record *all*
     542             :   // values, while previous loops account only values with unique SDValues.
     543          69 :   const Instruction *StatepointInstr = SI.StatepointInstr;
     544         138 :   auto &SpillMap = Builder.FuncInfo.StatepointSpillMaps[StatepointInstr];
     545             : 
     546         265 :   for (const GCRelocateInst *Relocate : SI.GCRelocates) {
     547          58 :     const Value *V = Relocate->getDerivedPtr();
     548          58 :     SDValue SDV = Builder.getValue(V);
     549          58 :     SDValue Loc = Builder.StatepointLowering.getLocation(SDV);
     550             : 
     551          58 :     if (Loc.getNode()) {
     552         102 :       SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
     553             :     } else {
     554             :       // Record value as visited, but not spilled. This is case for allocas
     555             :       // and constants. For this values we can avoid emitting spill load while
     556             :       // visiting corresponding gc_relocate.
     557             :       // Actually we do not need to record them in this map at all.
     558             :       // We do this only to check that we are not relocating any unvisited
     559             :       // value.
     560          28 :       SpillMap.SlotMap[V] = None;
     561             : 
     562             :       // Default llvm mechanisms for exporting values which are used in
     563             :       // different basic blocks does not work for gc relocates.
     564             :       // Note that it would be incorrect to teach llvm that all relocates are
     565             :       // uses of the corresponding values so that it would automatically
     566             :       // export them. Relocates of the spilled values does not use original
     567             :       // value.
     568           7 :       if (Relocate->getParent() != StatepointInstr->getParent())
     569           3 :         Builder.ExportFromCurrentBlock(V);
     570             :     }
     571             :   }
     572          69 : }
     573             : 
     574          69 : SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
     575             :     SelectionDAGBuilder::StatepointLoweringInfo &SI) {
     576             :   // The basic scheme here is that information about both the original call and
     577             :   // the safepoint is encoded in the CallInst.  We create a temporary call and
     578             :   // lower it, then reverse engineer the calling sequence.
     579             : 
     580          69 :   NumOfStatepoints++;
     581             :   // Clear state
     582          69 :   StatepointLowering.startNewStatepoint(*this);
     583             : 
     584             : #ifndef NDEBUG
     585             :   // We schedule gc relocates before removeDuplicateGCPtrs since we _will_
     586             :   // encounter the duplicate gc relocates we elide in removeDuplicateGCPtrs.
     587             :   for (auto *Reloc : SI.GCRelocates)
     588             :     if (Reloc->getParent() == SI.StatepointInstr->getParent())
     589             :       StatepointLowering.scheduleRelocCall(*Reloc);
     590             : #endif
     591             : 
     592             :   // Remove any redundant llvm::Values which map to the same SDValue as another
     593             :   // input.  Also has the effect of removing duplicates in the original
     594             :   // llvm::Value input list as well.  This is a useful optimization for
     595             :   // reducing the size of the StackMap section.  It has no other impact.
     596         207 :   removeDuplicateGCPtrs(SI.Bases, SI.Ptrs, SI.GCRelocates, *this,
     597          69 :                         FuncInfo.StatepointSpillMaps[SI.StatepointInstr]);
     598             :   assert(SI.Bases.size() == SI.Ptrs.size() &&
     599             :          SI.Ptrs.size() == SI.GCRelocates.size());
     600             : 
     601             :   // Lower statepoint vmstate and gcstate arguments
     602         138 :   SmallVector<SDValue, 10> LoweredMetaArgs;
     603          69 :   lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this);
     604             : 
     605             :   // Now that we've emitted the spills, we need to update the root so that the
     606             :   // call sequence is ordered correctly.
     607         138 :   SI.CLI.setChain(getRoot());
     608             : 
     609             :   // Get call node, we will replace it later with statepoint
     610             :   SDValue ReturnVal;
     611             :   SDNode *CallNode;
     612          69 :   std::tie(ReturnVal, CallNode) =
     613         207 :       lowerCallFromStatepointLoweringInfo(SI, *this, PendingExports);
     614             : 
     615             :   // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
     616             :   // nodes with all the appropriate arguments and return values.
     617             : 
     618             :   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
     619         138 :   SDValue Chain = CallNode->getOperand(0);
     620             : 
     621          69 :   SDValue Glue;
     622          78 :   bool CallHasIncomingGlue = CallNode->getGluedNode();
     623             :   if (CallHasIncomingGlue) {
     624             :     // Glue is always last operand
     625          18 :     Glue = CallNode->getOperand(CallNode->getNumOperands() - 1);
     626             :   }
     627             : 
     628             :   // Build the GC_TRANSITION_START node if necessary.
     629             :   //
     630             :   // The operands to the GC_TRANSITION_{START,END} nodes are laid out in the
     631             :   // order in which they appear in the call to the statepoint intrinsic. If
     632             :   // any of the operands is a pointer-typed, that operand is immediately
     633             :   // followed by a SRCVALUE for the pointer that may be used during lowering
     634             :   // (e.g. to form MachinePointerInfo values for loads/stores).
     635          69 :   const bool IsGCTransition =
     636          69 :       (SI.StatepointFlags & (uint64_t)StatepointFlags::GCTransition) ==
     637             :       (uint64_t)StatepointFlags::GCTransition;
     638          69 :   if (IsGCTransition) {
     639          16 :     SmallVector<SDValue, 8> TSOps;
     640             : 
     641             :     // Add chain
     642           8 :     TSOps.push_back(Chain);
     643             : 
     644             :     // Add GC transition arguments
     645          17 :     for (const Value *V : SI.GCTransitionArgs) {
     646           1 :       TSOps.push_back(getValue(V));
     647           2 :       if (V->getType()->isPointerTy())
     648           1 :         TSOps.push_back(DAG.getSrcValue(V));
     649             :     }
     650             : 
     651             :     // Add glue if necessary
     652           8 :     if (CallHasIncomingGlue)
     653           2 :       TSOps.push_back(Glue);
     654             : 
     655          24 :     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
     656             : 
     657             :     SDValue GCTransitionStart =
     658          32 :         DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);
     659             : 
     660          16 :     Chain = GCTransitionStart.getValue(0);
     661          16 :     Glue = GCTransitionStart.getValue(1);
     662             :   }
     663             : 
     664             :   // TODO: Currently, all of these operands are being marked as read/write in
     665             :   // PrologEpilougeInserter.cpp, we should special case the VMState arguments
     666             :   // and flags to be read-only.
     667         138 :   SmallVector<SDValue, 40> Ops;
     668             : 
     669             :   // Add the <id> and <numBytes> constants.
     670         345 :   Ops.push_back(DAG.getTargetConstant(SI.ID, getCurSDLoc(), MVT::i64));
     671          69 :   Ops.push_back(
     672         414 :       DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));
     673             : 
     674             :   // Calculate and push starting position of vmstate arguments
     675             :   // Get number of arguments incoming directly into call node
     676             :   unsigned NumCallRegArgs =
     677         138 :       CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);
     678         345 :   Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));
     679             : 
     680             :   // Add call target
     681         207 :   SDValue CallTarget = SDValue(CallNode->getOperand(1).getNode(), 0);
     682          69 :   Ops.push_back(CallTarget);
     683             : 
     684             :   // Add call arguments
     685             :   // Get position of register mask in the call
     686             :   SDNode::op_iterator RegMaskIt;
     687          69 :   if (CallHasIncomingGlue)
     688          18 :     RegMaskIt = CallNode->op_end() - 2;
     689             :   else
     690         120 :     RegMaskIt = CallNode->op_end() - 1;
     691         138 :   Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);
     692             : 
     693             :   // Add a constant argument for the calling convention
     694          69 :   pushStackMapConstant(Ops, *this, SI.CLI.CallConv);
     695             : 
     696             :   // Add a constant argument for the flags
     697          69 :   uint64_t Flags = SI.StatepointFlags;
     698             :   assert(((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0) &&
     699             :          "Unknown flag used");
     700          69 :   pushStackMapConstant(Ops, *this, Flags);
     701             : 
     702             :   // Insert all vmstate and gcstate arguments
     703         207 :   Ops.insert(Ops.end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
     704             : 
     705             :   // Add register mask from call node
     706          69 :   Ops.push_back(*RegMaskIt);
     707             : 
     708             :   // Add chain
     709          69 :   Ops.push_back(Chain);
     710             : 
     711             :   // Same for the glue, but we add it only if original call had it
     712          69 :   if (Glue.getNode())
     713          15 :     Ops.push_back(Glue);
     714             : 
     715             :   // Compute return values.  Provide a glue output since we consume one as
     716             :   // input.  This allows someone else to chain off us as needed.
     717         207 :   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
     718             : 
     719             :   SDNode *StatepointMCNode =
     720         276 :       DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
     721             : 
     722          69 :   SDNode *SinkNode = StatepointMCNode;
     723             : 
     724             :   // Build the GC_TRANSITION_END node if necessary.
     725             :   //
     726             :   // See the comment above regarding GC_TRANSITION_START for the layout of
     727             :   // the operands to the GC_TRANSITION_END node.
     728          69 :   if (IsGCTransition) {
     729          16 :     SmallVector<SDValue, 8> TEOps;
     730             : 
     731             :     // Add chain
     732           8 :     TEOps.push_back(SDValue(StatepointMCNode, 0));
     733             : 
     734             :     // Add GC transition arguments
     735          17 :     for (const Value *V : SI.GCTransitionArgs) {
     736           1 :       TEOps.push_back(getValue(V));
     737           2 :       if (V->getType()->isPointerTy())
     738           1 :         TEOps.push_back(DAG.getSrcValue(V));
     739             :     }
     740             : 
     741             :     // Add glue
     742           8 :     TEOps.push_back(SDValue(StatepointMCNode, 1));
     743             : 
     744          24 :     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
     745             : 
     746             :     SDValue GCTransitionStart =
     747          32 :         DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);
     748             : 
     749           8 :     SinkNode = GCTransitionStart.getNode();
     750             :   }
     751             : 
     752             :   // Replace original call
     753          69 :   DAG.ReplaceAllUsesWith(CallNode, SinkNode); // This may update Root
     754             :   // Remove original call node
     755          69 :   DAG.DeleteNode(CallNode);
     756             : 
     757             :   // DON'T set the root - under the assumption that it's already set past the
     758             :   // inserted node we created.
     759             : 
     760             :   // TODO: A better future implementation would be to emit a single variable
     761             :   // argument, variable return value STATEPOINT node here and then hookup the
     762             :   // return value of each gc.relocate to the respective output of the
     763             :   // previously emitted STATEPOINT value.  Unfortunately, this doesn't appear
     764             :   // to actually be possible today.
     765             : 
     766         138 :   return ReturnVal;
     767             : }
     768             : 
     769             : void
     770          66 : SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
     771             :                                      const BasicBlock *EHPadBB /*= nullptr*/) {
     772             :   assert(ISP.getCallSite().getCallingConv() != CallingConv::AnyReg &&
     773             :          "anyregcc is not supported on statepoints!");
     774             : 
     775             : #ifndef NDEBUG
     776             :   // If this is a malformed statepoint, report it early to simplify debugging.
     777             :   // This should catch any IR level mistake that's made when constructing or
     778             :   // transforming statepoints.
     779             :   ISP.verify();
     780             : 
     781             :   // Check that the associated GCStrategy expects to encounter statepoints.
     782             :   assert(GFI->getStrategy().useStatepoints() &&
     783             :          "GCStrategy does not expect to encounter statepoints");
     784             : #endif
     785             : 
     786          66 :   SDValue ActualCallee;
     787             : 
     788          66 :   if (ISP.getNumPatchBytes() > 0) {
     789             :     // If we've been asked to emit a nop sequence instead of a call instruction
     790             :     // for this statepoint then don't lower the call target, but use a constant
     791             :     // `null` instead.  Not lowering the call target lets statepoint clients get
     792             :     // away without providing a physical address for the symbolic call target at
     793             :     // link time.
     794             : 
     795           1 :     const auto &TLI = DAG.getTargetLoweringInfo();
     796           2 :     const auto &DL = DAG.getDataLayout();
     797             : 
     798           2 :     unsigned AS = ISP.getCalledValue()->getType()->getPointerAddressSpace();
     799           5 :     ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
     800             :   } else {
     801          65 :     ActualCallee = getValue(ISP.getCalledValue());
     802             :   }
     803             : 
     804         132 :   StatepointLoweringInfo SI(DAG);
     805         198 :   populateCallLoweringInfo(SI.CLI, ISP.getCallSite(),
     806             :                            ImmutableStatepoint::CallArgsBeginPos,
     807          66 :                            ISP.getNumCallArgs(), ActualCallee,
     808             :                            ISP.getActualReturnType(), false /* IsPatchPoint */);
     809             : 
     810         396 :   for (const GCRelocateInst *Relocate : ISP.getRelocates()) {
     811          66 :     SI.GCRelocates.push_back(Relocate);
     812          66 :     SI.Bases.push_back(Relocate->getBasePtr());
     813          66 :     SI.Ptrs.push_back(Relocate->getDerivedPtr());
     814             :   }
     815             : 
     816         198 :   SI.GCArgs = ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
     817          66 :   SI.StatepointInstr = ISP.getInstruction();
     818          66 :   SI.GCTransitionArgs =
     819         198 :       ArrayRef<const Use>(ISP.gc_args_begin(), ISP.gc_args_end());
     820          66 :   SI.ID = ISP.getID();
     821         198 :   SI.DeoptState = ArrayRef<const Use>(ISP.deopt_begin(), ISP.deopt_end());
     822          66 :   SI.StatepointFlags = ISP.getFlags();
     823          66 :   SI.NumPatchBytes = ISP.getNumPatchBytes();
     824          66 :   SI.EHPadBB = EHPadBB;
     825             : 
     826          66 :   SDValue ReturnValue = LowerAsSTATEPOINT(SI);
     827             : 
     828             :   // Export the result value if needed
     829          66 :   const GCResultInst *GCResult = ISP.getGCResult();
     830          66 :   Type *RetTy = ISP.getActualReturnType();
     831          66 :   if (!RetTy->isVoidTy() && GCResult) {
     832          72 :     if (GCResult->getParent() != ISP.getCallSite().getParent()) {
     833             :       // Result value will be used in a different basic block so we need to
     834             :       // export it now.  Default exporting mechanism will not work here because
     835             :       // statepoint call has a different type than the actual call. It means
     836             :       // that by default llvm will create export register of the wrong type
     837             :       // (always i32 in our case). So instead we need to create export register
     838             :       // with correct type manually.
     839             :       // TODO: To eliminate this problem we can remove gc.result intrinsics
     840             :       //       completely and make statepoint call to return a tuple.
     841           2 :       unsigned Reg = FuncInfo.CreateRegs(RetTy);
     842           4 :       RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
     843           8 :                        DAG.getDataLayout(), Reg, RetTy, true);
     844           4 :       SDValue Chain = DAG.getEntryNode();
     845             : 
     846           6 :       RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
     847           2 :       PendingExports.push_back(Chain);
     848           4 :       FuncInfo.ValueMap[ISP.getInstruction()] = Reg;
     849             :     } else {
     850             :       // Result value will be used in a same basic block. Don't export it or
     851             :       // perform any explicit register copies.
     852             :       // We'll replace the actuall call node shortly. gc_result will grab
     853             :       // this value.
     854          22 :       setValue(ISP.getInstruction(), ReturnValue);
     855             :     }
     856             :   } else {
     857             :     // The token value is never used from here on, just generate a poison value
     858         210 :     setValue(ISP.getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
     859             :   }
     860          66 : }
     861             : 
     862           3 : void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
     863             :     ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB,
     864             :     bool VarArgDisallowed, bool ForceVoidReturnTy) {
     865           6 :   StatepointLoweringInfo SI(DAG);
     866           9 :   unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin();
     867           6 :   populateCallLoweringInfo(
     868             :       SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee,
     869           0 :       ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(),
     870             :       false);
     871           3 :   if (!VarArgDisallowed)
     872           6 :     SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg();
     873             : 
     874           9 :   auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt);
     875             : 
     876           3 :   unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID;
     877             : 
     878           6 :   auto SD = parseStatepointDirectivesFromAttrs(CS.getAttributes());
     879           3 :   SI.ID = SD.StatepointID.getValueOr(DefaultID);
     880           6 :   SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
     881             : 
     882           3 :   SI.DeoptState =
     883           6 :       ArrayRef<const Use>(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());
     884           3 :   SI.StatepointFlags = static_cast<uint64_t>(StatepointFlags::None);
     885           3 :   SI.EHPadBB = EHPadBB;
     886             : 
     887             :   // NB! The GC arguments are deliberately left empty.
     888             : 
     889           3 :   if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
     890           0 :     const Instruction *Inst = CS.getInstruction();
     891           0 :     ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal);
     892             :     setValue(Inst, ReturnVal);
     893             :   }
     894           3 : }
     895             : 
     896           3 : void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle(
     897             :     ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) {
     898           3 :   LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB,
     899             :                                    /* VarArgDisallowed = */ false,
     900             :                                    /* ForceVoidReturnTy  = */ false);
     901           3 : }
     902             : 
     903          24 : void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {
     904             :   // The result value of the gc_result is simply the result of the actual
     905             :   // call.  We've already emitted this, so just grab the value.
     906          24 :   const Instruction *I = CI.getStatepoint();
     907             : 
     908          24 :   if (I->getParent() != CI.getParent()) {
     909             :     // Statepoint is in different basic block so we should have stored call
     910             :     // result in a virtual register.
     911             :     // We can not use default getValue() functionality to copy value from this
     912             :     // register because statepoint and actual call return types can be
     913             :     // different, and getValue() will use CopyFromReg of the wrong type,
     914             :     // which is always i32 in our case.
     915           2 :     PointerType *CalleeType = cast<PointerType>(
     916           4 :         ImmutableStatepoint(I).getCalledValue()->getType());
     917             :     Type *RetTy =
     918           6 :         cast<FunctionType>(CalleeType->getElementType())->getReturnType();
     919           2 :     SDValue CopyFromReg = getCopyFromRegs(I, RetTy);
     920             : 
     921             :     assert(CopyFromReg.getNode());
     922           4 :     setValue(&CI, CopyFromReg);
     923             :   } else {
     924          22 :     setValue(&CI, getValue(I));
     925             :   }
     926          24 : }
     927             : 
     928          66 : void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
     929             : #ifndef NDEBUG
     930             :   // Consistency check
     931             :   // We skip this check for relocates not in the same basic block as their
     932             :   // statepoint. It would be too expensive to preserve validation info through
     933             :   // different basic blocks.
     934             :   if (Relocate.getStatepoint()->getParent() == Relocate.getParent())
     935             :     StatepointLowering.relocCallVisited(Relocate);
     936             : 
     937             :   auto *Ty = Relocate.getType()->getScalarType();
     938             :   if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
     939             :     assert(*IsManaged && "Non gc managed pointer relocated!");
     940             : #endif
     941             : 
     942          66 :   const Value *DerivedPtr = Relocate.getDerivedPtr();
     943          66 :   SDValue SD = getValue(DerivedPtr);
     944             : 
     945         132 :   auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.getStatepoint()];
     946          66 :   auto SlotIt = SpillMap.find(DerivedPtr);
     947             :   assert(SlotIt != SpillMap.end() && "Relocating not lowered gc value");
     948         190 :   Optional<int> DerivedPtrLocation = SlotIt->second;
     949             : 
     950             :   // We didn't need to spill these special cases (constants and allocas).
     951             :   // See the handling in spillIncomingValueForStatepoint for detail.
     952          66 :   if (!DerivedPtrLocation) {
     953          16 :     setValue(&Relocate, SD);
     954           8 :     return;
     955             :   }
     956             : 
     957             :   SDValue SpillSlot =
     958         232 :       DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
     959             : 
     960             :   // Be conservative: flush all pending loads
     961             :   // TODO: Probably we can be less restrictive on this,
     962             :   // it may allow more scheduling opportunities.
     963          58 :   SDValue Chain = getRoot();
     964             : 
     965             :   SDValue SpillLoad =
     966         116 :       DAG.getLoad(DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
     967             :                                                            Relocate.getType()),
     968         116 :                   getCurSDLoc(), Chain, SpillSlot,
     969             :                   MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
     970         290 :                                                     *DerivedPtrLocation));
     971             : 
     972             :   // Again, be conservative, don't emit pending loads
     973         116 :   DAG.setRoot(SpillLoad.getValue(1));
     974             : 
     975             :   assert(SpillLoad.getNode());
     976         116 :   setValue(&Relocate, SpillLoad);
     977             : }
     978             : 
     979           0 : void SelectionDAGBuilder::LowerDeoptimizeCall(const CallInst *CI) {
     980           0 :   const auto &TLI = DAG.getTargetLoweringInfo();
     981             :   SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
     982           0 :                                          TLI.getPointerTy(DAG.getDataLayout()));
     983             : 
     984             :   // We don't lower calls to __llvm_deoptimize as varargs, but as a regular
     985             :   // call.  We also do not lower the return value to any virtual register, and
     986             :   // change the immediately following return to a trap instruction.
     987           0 :   LowerCallSiteWithDeoptBundleImpl(CI, Callee, /* EHPadBB = */ nullptr,
     988             :                                    /* VarArgDisallowed = */ true,
     989             :                                    /* ForceVoidReturnTy = */ true);
     990           0 : }
     991             : 
     992           0 : void SelectionDAGBuilder::LowerDeoptimizingReturn() {
     993             :   // We do not lower the return value from llvm.deoptimize to any virtual
     994             :   // register, and change the immediately following return to a trap
     995             :   // instruction.
     996           0 :   if (DAG.getTarget().Options.TrapUnreachable)
     997           0 :     DAG.setRoot(
     998           0 :         DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
     999           0 : }

Generated by: LCOV version 1.13