LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - SIMemoryLegalizer.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 203 229 88.6 %
Date: 2017-09-14 15:23:50 Functions: 23 24 95.8 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- SIMemoryLegalizer.cpp ----------------------------------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// \brief Memory legalizer - implements memory model. More information can be
      12             : /// found here:
      13             : ///   http://llvm.org/docs/AMDGPUUsage.html#memory-model
      14             : //
      15             : //===----------------------------------------------------------------------===//
      16             : 
      17             : #include "AMDGPU.h"
      18             : #include "AMDGPUMachineModuleInfo.h"
      19             : #include "AMDGPUSubtarget.h"
      20             : #include "SIDefines.h"
      21             : #include "SIInstrInfo.h"
      22             : #include "Utils/AMDGPUBaseInfo.h"
      23             : #include "llvm/ADT/None.h"
      24             : #include "llvm/ADT/Optional.h"
      25             : #include "llvm/CodeGen/MachineBasicBlock.h"
      26             : #include "llvm/CodeGen/MachineFunction.h"
      27             : #include "llvm/CodeGen/MachineFunctionPass.h"
      28             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      29             : #include "llvm/CodeGen/MachineMemOperand.h"
      30             : #include "llvm/CodeGen/MachineModuleInfo.h"
      31             : #include "llvm/CodeGen/MachineOperand.h"
      32             : #include "llvm/IR/DebugLoc.h"
      33             : #include "llvm/IR/DiagnosticInfo.h"
      34             : #include "llvm/IR/Function.h"
      35             : #include "llvm/IR/LLVMContext.h"
      36             : #include "llvm/MC/MCInstrDesc.h"
      37             : #include "llvm/Pass.h"
      38             : #include "llvm/Support/AtomicOrdering.h"
      39             : #include <cassert>
      40             : #include <list>
      41             : 
      42             : using namespace llvm;
      43             : using namespace llvm::AMDGPU;
      44             : 
      45             : #define DEBUG_TYPE "si-memory-legalizer"
      46             : #define PASS_NAME "SI Memory Legalizer"
      47             : 
      48             : namespace {
      49             : 
      50             : class SIMemOpInfo final {
      51             : private:
      52             :   SyncScope::ID SSID = SyncScope::System;
      53             :   AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
      54             :   AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
      55             :   bool IsNonTemporal = false;
      56             : 
      57             :   SIMemOpInfo(SyncScope::ID SSID, AtomicOrdering Ordering)
      58             :       : SSID(SSID), Ordering(Ordering) {}
      59             : 
      60             :   SIMemOpInfo(SyncScope::ID SSID, AtomicOrdering Ordering,
      61             :               AtomicOrdering FailureOrdering, bool IsNonTemporal = false)
      62             :       : SSID(SSID), Ordering(Ordering), FailureOrdering(FailureOrdering),
      63       43800 :         IsNonTemporal(IsNonTemporal) {}
      64             : 
      65             :   /// \returns Info constructed from \p MI, which has at least machine memory
      66             :   /// operand.
      67             :   static Optional<SIMemOpInfo> constructFromMIWithMMO(
      68             :       const MachineBasicBlock::iterator &MI);
      69             : 
      70             : public:
      71             :   /// \returns Synchronization scope ID of the machine instruction used to
      72             :   /// create this SIMemOpInfo.
      73             :   SyncScope::ID getSSID() const {
      74             :     return SSID;
      75             :   }
      76             :   /// \returns Ordering constraint of the machine instruction used to
      77             :   /// create this SIMemOpInfo.
      78             :   AtomicOrdering getOrdering() const {
      79             :     return Ordering;
      80             :   }
      81             :   /// \returns Failure ordering constraint of the machine instruction used to
      82             :   /// create this SIMemOpInfo.
      83             :   AtomicOrdering getFailureOrdering() const {
      84             :     return FailureOrdering;
      85             :   }
      86             :   /// \returns True if memory access of the machine instruction used to
      87             :   /// create this SIMemOpInfo is non-temporal, false otherwise.
      88             :   bool isNonTemporal() const {
      89             :     return IsNonTemporal;
      90             :   }
      91             : 
      92             :   /// \returns True if ordering constraint of the machine instruction used to
      93             :   /// create this SIMemOpInfo is unordered or higher, false otherwise.
      94             :   bool isAtomic() const {
      95             :     return Ordering != AtomicOrdering::NotAtomic;
      96             :   }
      97             : 
      98             :   /// \returns Load info if \p MI is a load operation, "None" otherwise.
      99             :   static Optional<SIMemOpInfo> getLoadInfo(
     100             :       const MachineBasicBlock::iterator &MI);
     101             :   /// \returns Store info if \p MI is a store operation, "None" otherwise.
     102             :   static Optional<SIMemOpInfo> getStoreInfo(
     103             :       const MachineBasicBlock::iterator &MI);
     104             :   /// \returns Atomic fence info if \p MI is an atomic fence operation,
     105             :   /// "None" otherwise.
     106             :   static Optional<SIMemOpInfo> getAtomicFenceInfo(
     107             :       const MachineBasicBlock::iterator &MI);
     108             :   /// \returns Atomic cmpxchg info if \p MI is an atomic cmpxchg operation,
     109             :   /// "None" otherwise.
     110             :   static Optional<SIMemOpInfo> getAtomicCmpxchgInfo(
     111             :       const MachineBasicBlock::iterator &MI);
     112             :   /// \returns Atomic rmw info if \p MI is an atomic rmw operation,
     113             :   /// "None" otherwise.
     114             :   static Optional<SIMemOpInfo> getAtomicRmwInfo(
     115             :       const MachineBasicBlock::iterator &MI);
     116             : 
     117             :   /// \brief Reports unknown synchronization scope used in \p MI to LLVM
     118             :   /// context.
     119             :   static void reportUnknownSyncScope(
     120             :       const MachineBasicBlock::iterator &MI);
     121             : };
     122             : 
     123        4392 : class SIMemoryLegalizer final : public MachineFunctionPass {
     124             : private:
     125             :   /// \brief Machine module info.
     126             :   const AMDGPUMachineModuleInfo *MMI = nullptr;
     127             : 
     128             :   /// \brief Instruction info.
     129             :   const SIInstrInfo *TII = nullptr;
     130             : 
     131             :   /// \brief Immediate for "vmcnt(0)".
     132             :   unsigned Vmcnt0Immediate = 0;
     133             : 
     134             :   /// \brief Opcode for cache invalidation instruction (L1).
     135             :   unsigned Wbinvl1Opcode = 0;
     136             : 
     137             :   /// \brief List of atomic pseudo instructions.
     138             :   std::list<MachineBasicBlock::iterator> AtomicPseudoMIs;
     139             : 
     140             :   /// \brief Sets named bit (BitName) to "true" if present in \p MI. Returns
     141             :   /// true if \p MI is modified, false otherwise.
     142             :   template <uint16_t BitName>
     143        3183 :   bool enableNamedBit(const MachineBasicBlock::iterator &MI) const {
     144        6366 :     int BitIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), BitName);
     145        3183 :     if (BitIdx == -1)
     146             :       return false;
     147             : 
     148        6186 :     MachineOperand &Bit = MI->getOperand(BitIdx);
     149        3093 :     if (Bit.getImm() != 0)
     150             :       return false;
     151             : 
     152        6158 :     Bit.setImm(1);
     153             :     return true;
     154             :   }
     155             : 
     156             :   /// \brief Sets GLC bit to "true" if present in \p MI. Returns true if \p MI
     157             :   /// is modified, false otherwise.
     158             :   bool enableGLCBit(const MachineBasicBlock::iterator &MI) const {
     159        1670 :     return enableNamedBit<AMDGPU::OpName::glc>(MI);
     160             :   }
     161             : 
     162             :   /// \brief Sets SLC bit to "true" if present in \p MI. Returns true if \p MI
     163             :   /// is modified, false otherwise.
     164             :   bool enableSLCBit(const MachineBasicBlock::iterator &MI) const {
     165        1513 :     return enableNamedBit<AMDGPU::OpName::slc>(MI);
     166             :   }
     167             : 
     168             :   /// \brief Inserts "buffer_wbinvl1_vol" instruction \p Before or after \p MI.
     169             :   /// Always returns true.
     170             :   bool insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI,
     171             :                               bool Before = true) const;
     172             :   /// \brief Inserts "s_waitcnt vmcnt(0)" instruction \p Before or after \p MI.
     173             :   /// Always returns true.
     174             :   bool insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI,
     175             :                            bool Before = true) const;
     176             : 
     177             :   /// \brief Removes all processed atomic pseudo instructions from the current
     178             :   /// function. Returns true if current function is modified, false otherwise.
     179             :   bool removeAtomicPseudoMIs();
     180             : 
     181             :   /// \brief Expands load operation \p MI. Returns true if instructions are
     182             :   /// added/deleted or \p MI is modified, false otherwise.
     183             :   bool expandLoad(const SIMemOpInfo &MOI,
     184             :                   MachineBasicBlock::iterator &MI);
     185             :   /// \brief Expands store operation \p MI. Returns true if instructions are
     186             :   /// added/deleted or \p MI is modified, false otherwise.
     187             :   bool expandStore(const SIMemOpInfo &MOI,
     188             :                    MachineBasicBlock::iterator &MI);
     189             :   /// \brief Expands atomic fence operation \p MI. Returns true if
     190             :   /// instructions are added/deleted or \p MI is modified, false otherwise.
     191             :   bool expandAtomicFence(const SIMemOpInfo &MOI,
     192             :                          MachineBasicBlock::iterator &MI);
     193             :   /// \brief Expands atomic cmpxchg operation \p MI. Returns true if
     194             :   /// instructions are added/deleted or \p MI is modified, false otherwise.
     195             :   bool expandAtomicCmpxchg(const SIMemOpInfo &MOI,
     196             :                            MachineBasicBlock::iterator &MI);
     197             :   /// \brief Expands atomic rmw operation \p MI. Returns true if
     198             :   /// instructions are added/deleted or \p MI is modified, false otherwise.
     199             :   bool expandAtomicRmw(const SIMemOpInfo &MOI,
     200             :                        MachineBasicBlock::iterator &MI);
     201             : 
     202             : public:
     203             :   static char ID;
     204             : 
     205        2944 :   SIMemoryLegalizer() : MachineFunctionPass(ID) {}
     206             : 
     207        1464 :   void getAnalysisUsage(AnalysisUsage &AU) const override {
     208        1464 :     AU.setPreservesCFG();
     209        1464 :     MachineFunctionPass::getAnalysisUsage(AU);
     210        1464 :   }
     211             : 
     212        1464 :   StringRef getPassName() const override {
     213        1464 :     return PASS_NAME;
     214             :   }
     215             : 
     216             :   bool runOnMachineFunction(MachineFunction &MF) override;
     217             : };
     218             : 
     219             : } // end namespace anonymous
     220             : 
     221             : /* static */
     222       43812 : Optional<SIMemOpInfo> SIMemOpInfo::constructFromMIWithMMO(
     223             :     const MachineBasicBlock::iterator &MI) {
     224             :   assert(MI->getNumMemOperands() > 0);
     225             : 
     226       43812 :   const MachineFunction *MF = MI->getParent()->getParent();
     227             :   const AMDGPUMachineModuleInfo *MMI =
     228       43812 :       &MF->getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>();
     229             : 
     230       43812 :   SyncScope::ID SSID = SyncScope::SingleThread;
     231       43812 :   AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
     232       43812 :   AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
     233       43812 :   bool IsNonTemporal = true;
     234             : 
     235             :   // Validator should check whether or not MMOs cover the entire set of
     236             :   // locations accessed by the memory instruction.
     237      131434 :   for (const auto &MMO : MI->memoperands()) {
     238             :     const auto &IsSyncScopeInclusion =
     239      131454 :         MMI->isSyncScopeInclusion(SSID, MMO->getSyncScopeID());
     240       43822 :     if (!IsSyncScopeInclusion) {
     241          12 :       reportUnknownSyncScope(MI);
     242          24 :       return None;
     243             :     }
     244             : 
     245       87619 :     SSID = IsSyncScopeInclusion.getValue() ? SSID : MMO->getSyncScopeID();
     246       43810 :     Ordering =
     247      175240 :         isStrongerThan(Ordering, MMO->getOrdering()) ?
     248       43810 :             Ordering : MMO->getOrdering();
     249       43810 :     FailureOrdering =
     250      175240 :         isStrongerThan(FailureOrdering, MMO->getFailureOrdering()) ?
     251       43810 :             FailureOrdering : MMO->getFailureOrdering();
     252             : 
     253       87620 :     if (!(MMO->getFlags() & MachineMemOperand::MONonTemporal))
     254       42295 :       IsNonTemporal = false;
     255             :   }
     256             : 
     257      131400 :   return SIMemOpInfo(SSID, Ordering, FailureOrdering, IsNonTemporal);
     258             : }
     259             : 
     260             : /* static */
     261       44029 : Optional<SIMemOpInfo> SIMemOpInfo::getLoadInfo(
     262             :     const MachineBasicBlock::iterator &MI) {
     263             :   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
     264             : 
     265       62401 :   if (!(MI->mayLoad() && !MI->mayStore()))
     266       26945 :     return None;
     267             : 
     268             :   // Be conservative if there are no memory operands.
     269       17084 :   if (MI->getNumMemOperands() == 0)
     270          14 :     return SIMemOpInfo(SyncScope::System,
     271          14 :                        AtomicOrdering::SequentiallyConsistent);
     272             : 
     273       17070 :   return SIMemOpInfo::constructFromMIWithMMO(MI);
     274             : }
     275             : 
     276             : /* static */
     277       26947 : Optional<SIMemOpInfo> SIMemOpInfo::getStoreInfo(
     278             :     const MachineBasicBlock::iterator &MI) {
     279             :   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
     280             : 
     281       52604 :   if (!(!MI->mayLoad() && MI->mayStore()))
     282        1353 :     return None;
     283             : 
     284             :   // Be conservative if there are no memory operands.
     285       25594 :   if (MI->getNumMemOperands() == 0)
     286          98 :     return SIMemOpInfo(SyncScope::System,
     287          98 :                        AtomicOrdering::SequentiallyConsistent);
     288             : 
     289       25496 :   return SIMemOpInfo::constructFromMIWithMMO(MI);
     290             : }
     291             : 
     292             : /* static */
     293             : Optional<SIMemOpInfo> SIMemOpInfo::getAtomicFenceInfo(
     294             :     const MachineBasicBlock::iterator &MI) {
     295             :   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
     296             : 
     297        2710 :   if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE)
     298        1292 :     return None;
     299             : 
     300             :   SyncScope::ID SSID =
     301          63 :       static_cast<SyncScope::ID>(MI->getOperand(1).getImm());
     302             :   AtomicOrdering Ordering =
     303          63 :       static_cast<AtomicOrdering>(MI->getOperand(0).getImm());
     304         126 :   return SIMemOpInfo(SSID, Ordering);
     305             : }
     306             : 
     307             : /* static */
     308        1292 : Optional<SIMemOpInfo> SIMemOpInfo::getAtomicCmpxchgInfo(
     309             :     const MachineBasicBlock::iterator &MI) {
     310             :   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
     311             : 
     312        2582 :   if (!(MI->mayLoad() && MI->mayStore()))
     313           4 :     return None;
     314             : 
     315             :   // Be conservative if there are no memory operands.
     316        1288 :   if (MI->getNumMemOperands() == 0)
     317          46 :     return SIMemOpInfo(SyncScope::System,
     318             :                        AtomicOrdering::SequentiallyConsistent,
     319          46 :                        AtomicOrdering::SequentiallyConsistent);
     320             : 
     321        1242 :   return SIMemOpInfo::constructFromMIWithMMO(MI);
     322             : }
     323             : 
     324             : /* static */
     325           8 : Optional<SIMemOpInfo> SIMemOpInfo::getAtomicRmwInfo(
     326             :     const MachineBasicBlock::iterator &MI) {
     327             :   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
     328             : 
     329          14 :   if (!(MI->mayLoad() && MI->mayStore()))
     330           4 :     return None;
     331             : 
     332             :   // Be conservative if there are no memory operands.
     333           4 :   if (MI->getNumMemOperands() == 0)
     334           0 :     return SIMemOpInfo(SyncScope::System,
     335           0 :                        AtomicOrdering::SequentiallyConsistent);
     336             : 
     337           4 :   return SIMemOpInfo::constructFromMIWithMMO(MI);
     338             : }
     339             : 
     340             : /* static */
     341          14 : void SIMemOpInfo::reportUnknownSyncScope(
     342             :     const MachineBasicBlock::iterator &MI) {
     343          14 :   DiagnosticInfoUnsupported Diag(*MI->getParent()->getParent()->getFunction(),
     344          56 :                                  "Unsupported synchronization scope");
     345          14 :   LLVMContext *CTX = &MI->getParent()->getParent()->getFunction()->getContext();
     346          14 :   CTX->diagnose(Diag);
     347          14 : }
     348             : 
     349        1168 : bool SIMemoryLegalizer::insertBufferWbinvl1Vol(MachineBasicBlock::iterator &MI,
     350             :                                                bool Before) const {
     351        1168 :   MachineBasicBlock &MBB = *MI->getParent();
     352        3504 :   DebugLoc DL = MI->getDebugLoc();
     353             : 
     354        1168 :   if (!Before)
     355             :     ++MI;
     356             : 
     357        2336 :   BuildMI(MBB, MI, DL, TII->get(Wbinvl1Opcode));
     358             : 
     359        1168 :   if (!Before)
     360             :     --MI;
     361             : 
     362        2336 :   return true;
     363             : }
     364             : 
     365        2466 : bool SIMemoryLegalizer::insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI,
     366             :                                             bool Before) const {
     367        2466 :   MachineBasicBlock &MBB = *MI->getParent();
     368        7398 :   DebugLoc DL = MI->getDebugLoc();
     369             : 
     370        2466 :   if (!Before)
     371             :     ++MI;
     372             : 
     373        7398 :   BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Vmcnt0Immediate);
     374             : 
     375        2466 :   if (!Before)
     376             :     --MI;
     377             : 
     378        4932 :   return true;
     379             : }
     380             : 
     381       14832 : bool SIMemoryLegalizer::removeAtomicPseudoMIs() {
     382       29664 :   if (AtomicPseudoMIs.empty())
     383             :     return false;
     384             : 
     385         183 :   for (auto &MI : AtomicPseudoMIs)
     386          61 :     MI->eraseFromParent();
     387             : 
     388         122 :   AtomicPseudoMIs.clear();
     389          61 :   return true;
     390             : }
     391             : 
     392       17082 : bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,
     393             :                                    MachineBasicBlock::iterator &MI) {
     394             :   assert(MI->mayLoad() && !MI->mayStore());
     395             : 
     396       17082 :   bool Changed = false;
     397             : 
     398       17082 :   if (MOI.isAtomic()) {
     399         132 :     if (MOI.getSSID() == SyncScope::System ||
     400          33 :         MOI.getSSID() == MMI->getAgentSSID()) {
     401         146 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     402          71 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     403         134 :         Changed |= enableGLCBit(MI);
     404             : 
     405          75 :       if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     406          63 :         Changed |= insertWaitcntVmcnt0(MI);
     407             : 
     408         146 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     409          71 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
     410          67 :         Changed |= insertWaitcntVmcnt0(MI, false);
     411          67 :         Changed |= insertBufferWbinvl1Vol(MI, false);
     412             :       }
     413             : 
     414             :       return Changed;
     415             :     }
     416             : 
     417          40 :     if (MOI.getSSID() == SyncScope::SingleThread ||
     418          48 :         MOI.getSSID() == MMI->getWorkgroupSSID() ||
     419           8 :         MOI.getSSID() == MMI->getWavefrontSSID()) {
     420             :       return Changed;
     421             :     }
     422             : 
     423           0 :     llvm_unreachable("Unsupported synchronization scope");
     424             :   }
     425             : 
     426             :   // Atomic instructions do not have the nontemporal attribute.
     427       16983 :   if (MOI.isNonTemporal()) {
     428        2978 :     Changed |= enableGLCBit(MI);
     429        2978 :     Changed |= enableSLCBit(MI);
     430        1489 :     return Changed;
     431             :   }
     432             : 
     433             :   return Changed;
     434             : }
     435             : 
     436       25592 : bool SIMemoryLegalizer::expandStore(const SIMemOpInfo &MOI,
     437             :                                     MachineBasicBlock::iterator &MI) {
     438             :   assert(!MI->mayLoad() && MI->mayStore());
     439             : 
     440       25592 :   bool Changed = false;
     441             : 
     442       25592 :   if (MOI.isAtomic()) {
     443         220 :     if (MOI.getSSID() == SyncScope::System ||
     444          35 :         MOI.getSSID() == MMI->getAgentSSID()) {
     445         314 :       if (MOI.getOrdering() == AtomicOrdering::Release ||
     446         155 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     447         151 :         Changed |= insertWaitcntVmcnt0(MI);
     448             : 
     449             :       return Changed;
     450             :     }
     451             : 
     452          44 :     if (MOI.getSSID() == SyncScope::SingleThread ||
     453          53 :         MOI.getSSID() == MMI->getWorkgroupSSID() ||
     454           9 :         MOI.getSSID() == MMI->getWavefrontSSID()) {
     455             :       return Changed;
     456             :     }
     457             : 
     458           0 :     llvm_unreachable("Unsupported synchronization scope");
     459             :   }
     460             : 
     461             :   // Atomic instructions do not have the nontemporal attribute.
     462       25407 :   if (MOI.isNonTemporal()) {
     463          48 :     Changed |= enableGLCBit(MI);
     464          48 :     Changed |= enableSLCBit(MI);
     465          24 :     return Changed;
     466             :   }
     467             : 
     468             :   return Changed;
     469             : }
     470             : 
     471          63 : bool SIMemoryLegalizer::expandAtomicFence(const SIMemOpInfo &MOI,
     472             :                                           MachineBasicBlock::iterator &MI) {
     473             :   assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE);
     474             : 
     475          63 :   bool Changed = false;
     476             : 
     477          63 :   if (MOI.isAtomic()) {
     478         113 :     if (MOI.getSSID() == SyncScope::System ||
     479          50 :         MOI.getSSID() == MMI->getAgentSSID()) {
     480          43 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     481          30 :           MOI.getOrdering() == AtomicOrdering::Release ||
     482          43 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     483           6 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     484          25 :         Changed |= insertWaitcntVmcnt0(MI);
     485             : 
     486          43 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     487          37 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     488          12 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     489          19 :         Changed |= insertBufferWbinvl1Vol(MI);
     490             : 
     491          50 :       AtomicPseudoMIs.push_back(MI);
     492          25 :       return Changed;
     493             :     }
     494             : 
     495          64 :     if (MOI.getSSID() == SyncScope::SingleThread ||
     496          78 :         MOI.getSSID() == MMI->getWorkgroupSSID() ||
     497          14 :         MOI.getSSID() == MMI->getWavefrontSSID()) {
     498          72 :       AtomicPseudoMIs.push_back(MI);
     499          36 :       return Changed;
     500             :     }
     501             : 
     502           2 :     SIMemOpInfo::reportUnknownSyncScope(MI);
     503             :   }
     504             : 
     505             :   return Changed;
     506             : }
     507             : 
     508        1284 : bool SIMemoryLegalizer::expandAtomicCmpxchg(const SIMemOpInfo &MOI,
     509             :                                             MachineBasicBlock::iterator &MI) {
     510             :   assert(MI->mayLoad() && MI->mayStore());
     511             : 
     512        1284 :   bool Changed = false;
     513             : 
     514        1284 :   if (MOI.isAtomic()) {
     515        1308 :     if (MOI.getSSID() == SyncScope::System ||
     516         120 :         MOI.getSSID() == MMI->getAgentSSID()) {
     517        2184 :       if (MOI.getOrdering() == AtomicOrdering::Release ||
     518        2160 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     519        2192 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
     520          20 :           MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
     521        1078 :         Changed |= insertWaitcntVmcnt0(MI);
     522             : 
     523        2184 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     524        2160 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     525        1094 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
     526        1134 :           MOI.getFailureOrdering() == AtomicOrdering::Acquire ||
     527          16 :           MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) {
     528        1082 :         Changed |= insertWaitcntVmcnt0(MI, false);
     529        1082 :         Changed |= insertBufferWbinvl1Vol(MI, false);
     530             :       }
     531             : 
     532             :       return Changed;
     533             :     }
     534             : 
     535         150 :     if (MOI.getSSID() == SyncScope::SingleThread ||
     536         180 :         MOI.getSSID() == MMI->getWorkgroupSSID() ||
     537          30 :         MOI.getSSID() == MMI->getWavefrontSSID()) {
     538         180 :       Changed |= enableGLCBit(MI);
     539          90 :       return Changed;
     540             :     }
     541             : 
     542           0 :     llvm_unreachable("Unsupported synchronization scope");
     543             :   }
     544             : 
     545             :   return Changed;
     546             : }
     547             : 
     548           0 : bool SIMemoryLegalizer::expandAtomicRmw(const SIMemOpInfo &MOI,
     549             :                                         MachineBasicBlock::iterator &MI) {
     550             :   assert(MI->mayLoad() && MI->mayStore());
     551             : 
     552           0 :   bool Changed = false;
     553             : 
     554           0 :   if (MOI.isAtomic()) {
     555           0 :     if (MOI.getSSID() == SyncScope::System ||
     556           0 :         MOI.getSSID() == MMI->getAgentSSID()) {
     557           0 :       if (MOI.getOrdering() == AtomicOrdering::Release ||
     558           0 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     559           0 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     560           0 :         Changed |= insertWaitcntVmcnt0(MI);
     561             : 
     562           0 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     563           0 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     564           0 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
     565           0 :         Changed |= insertWaitcntVmcnt0(MI, false);
     566           0 :         Changed |= insertBufferWbinvl1Vol(MI, false);
     567             :       }
     568             : 
     569             :       return Changed;
     570             :     }
     571             : 
     572           0 :     if (MOI.getSSID() == SyncScope::SingleThread ||
     573           0 :         MOI.getSSID() == MMI->getWorkgroupSSID() ||
     574           0 :         MOI.getSSID() == MMI->getWavefrontSSID()) {
     575           0 :       Changed |= enableGLCBit(MI);
     576           0 :       return Changed;
     577             :     }
     578             : 
     579           0 :     llvm_unreachable("Unsupported synchronization scope");
     580             :   }
     581             : 
     582             :   return Changed;
     583             : }
     584             : 
     585       14832 : bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
     586       14832 :   bool Changed = false;
     587       14832 :   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
     588       29664 :   const IsaInfo::IsaVersion IV = IsaInfo::getIsaVersion(ST.getFeatureBits());
     589             : 
     590       14832 :   MMI = &MF.getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>();
     591       14832 :   TII = ST.getInstrInfo();
     592             : 
     593       14832 :   Vmcnt0Immediate =
     594       14832 :       AMDGPU::encodeWaitcnt(IV, 0, getExpcntBitMask(IV), getLgkmcntBitMask(IV));
     595       14832 :   Wbinvl1Opcode = ST.getGeneration() <= AMDGPUSubtarget::SOUTHERN_ISLANDS ?
     596             :       AMDGPU::BUFFER_WBINVL1 : AMDGPU::BUFFER_WBINVL1_VOL;
     597             : 
     598       61236 :   for (auto &MBB : MF) {
     599      652578 :     for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) {
     600      301179 :       if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic))
     601      257150 :         continue;
     602             : 
     603       44029 :       if (const auto &MOI = SIMemOpInfo::getLoadInfo(MI))
     604       17082 :         Changed |= expandLoad(MOI.getValue(), MI);
     605       26947 :       else if (const auto &MOI = SIMemOpInfo::getStoreInfo(MI))
     606       25592 :         Changed |= expandStore(MOI.getValue(), MI);
     607        1355 :       else if (const auto &MOI = SIMemOpInfo::getAtomicFenceInfo(MI))
     608          63 :         Changed |= expandAtomicFence(MOI.getValue(), MI);
     609        1292 :       else if (const auto &MOI = SIMemOpInfo::getAtomicCmpxchgInfo(MI))
     610        1284 :         Changed |= expandAtomicCmpxchg(MOI.getValue(), MI);
     611           8 :       else if (const auto &MOI = SIMemOpInfo::getAtomicRmwInfo(MI))
     612           0 :         Changed |= expandAtomicRmw(MOI.getValue(), MI);
     613             :     }
     614             :   }
     615             : 
     616       14832 :   Changed |= removeAtomicPseudoMIs();
     617       14832 :   return Changed;
     618             : }
     619             : 
     620      312538 : INITIALIZE_PASS(SIMemoryLegalizer, DEBUG_TYPE, PASS_NAME, false, false)
     621             : 
     622             : char SIMemoryLegalizer::ID = 0;
     623             : char &llvm::SIMemoryLegalizerID = SIMemoryLegalizer::ID;
     624             : 
     625        1468 : FunctionPass *llvm::createSIMemoryLegalizerPass() {
     626        2936 :   return new SIMemoryLegalizer();
     627             : }

Generated by: LCOV version 1.13