LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - SIMemoryLegalizer.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 157 160 98.1 %
Date: 2018-02-23 15:42:53 Functions: 22 22 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- SIMemoryLegalizer.cpp ----------------------------------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// \brief Memory legalizer - implements memory model. More information can be
      12             : /// found here:
      13             : ///   http://llvm.org/docs/AMDGPUUsage.html#memory-model
      14             : //
      15             : //===----------------------------------------------------------------------===//
      16             : 
      17             : #include "AMDGPU.h"
      18             : #include "AMDGPUMachineModuleInfo.h"
      19             : #include "AMDGPUSubtarget.h"
      20             : #include "SIDefines.h"
      21             : #include "SIInstrInfo.h"
      22             : #include "Utils/AMDGPUBaseInfo.h"
      23             : #include "llvm/ADT/None.h"
      24             : #include "llvm/ADT/Optional.h"
      25             : #include "llvm/CodeGen/MachineBasicBlock.h"
      26             : #include "llvm/CodeGen/MachineFunction.h"
      27             : #include "llvm/CodeGen/MachineFunctionPass.h"
      28             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      29             : #include "llvm/CodeGen/MachineMemOperand.h"
      30             : #include "llvm/CodeGen/MachineModuleInfo.h"
      31             : #include "llvm/CodeGen/MachineOperand.h"
      32             : #include "llvm/IR/DebugLoc.h"
      33             : #include "llvm/IR/DiagnosticInfo.h"
      34             : #include "llvm/IR/Function.h"
      35             : #include "llvm/IR/LLVMContext.h"
      36             : #include "llvm/MC/MCInstrDesc.h"
      37             : #include "llvm/Pass.h"
      38             : #include "llvm/Support/AtomicOrdering.h"
      39             : #include <cassert>
      40             : #include <list>
      41             : 
      42             : using namespace llvm;
      43             : using namespace llvm::AMDGPU;
      44             : 
      45             : #define DEBUG_TYPE "si-memory-legalizer"
      46             : #define PASS_NAME "SI Memory Legalizer"
      47             : 
      48             : namespace {
      49             : 
      50             : class SIMemOpInfo final {
      51             : private:
      52             :   SyncScope::ID SSID = SyncScope::System;
      53             :   AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
      54             :   AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
      55             :   bool IsNonTemporal = false;
      56             : 
      57             :   SIMemOpInfo(SyncScope::ID SSID, AtomicOrdering Ordering)
      58             :       : SSID(SSID), Ordering(Ordering) {}
      59             : 
      60             :   SIMemOpInfo(SyncScope::ID SSID, AtomicOrdering Ordering,
      61             :               AtomicOrdering FailureOrdering, bool IsNonTemporal = false)
      62             :       : SSID(SSID), Ordering(Ordering), FailureOrdering(FailureOrdering),
      63       46050 :         IsNonTemporal(IsNonTemporal) {}
      64             : 
      65             :   /// \returns Info constructed from \p MI, which has at least machine memory
      66             :   /// operand.
      67             :   static Optional<SIMemOpInfo> constructFromMIWithMMO(
      68             :       const MachineBasicBlock::iterator &MI);
      69             : 
      70             : public:
      71             :   /// \returns Synchronization scope ID of the machine instruction used to
      72             :   /// create this SIMemOpInfo.
      73             :   SyncScope::ID getSSID() const {
      74             :     return SSID;
      75             :   }
      76             :   /// \returns Ordering constraint of the machine instruction used to
      77             :   /// create this SIMemOpInfo.
      78             :   AtomicOrdering getOrdering() const {
      79             :     return Ordering;
      80             :   }
      81             :   /// \returns Failure ordering constraint of the machine instruction used to
      82             :   /// create this SIMemOpInfo.
      83             :   AtomicOrdering getFailureOrdering() const {
      84             :     return FailureOrdering;
      85             :   }
      86             :   /// \returns True if memory access of the machine instruction used to
      87             :   /// create this SIMemOpInfo is non-temporal, false otherwise.
      88             :   bool isNonTemporal() const {
      89             :     return IsNonTemporal;
      90             :   }
      91             : 
      92             :   /// \returns True if ordering constraint of the machine instruction used to
      93             :   /// create this SIMemOpInfo is unordered or higher, false otherwise.
      94             :   bool isAtomic() const {
      95             :     return Ordering != AtomicOrdering::NotAtomic;
      96             :   }
      97             : 
      98             :   /// \returns Load info if \p MI is a load operation, "None" otherwise.
      99             :   static Optional<SIMemOpInfo> getLoadInfo(
     100             :       const MachineBasicBlock::iterator &MI);
     101             :   /// \returns Store info if \p MI is a store operation, "None" otherwise.
     102             :   static Optional<SIMemOpInfo> getStoreInfo(
     103             :       const MachineBasicBlock::iterator &MI);
     104             :   /// \returns Atomic fence info if \p MI is an atomic fence operation,
     105             :   /// "None" otherwise.
     106             :   static Optional<SIMemOpInfo> getAtomicFenceInfo(
     107             :       const MachineBasicBlock::iterator &MI);
     108             :   /// \returns Atomic cmpxchg/rmw info if \p MI is an atomic cmpxchg or
     109             :   /// rmw operation, "None" otherwise.
     110             :   static Optional<SIMemOpInfo> getAtomicCmpxchgOrRmwInfo(
     111             :       const MachineBasicBlock::iterator &MI);
     112             : 
     113             :   /// \brief Reports unknown synchronization scope used in \p MI to LLVM
     114             :   /// context.
     115             :   static void reportUnknownSyncScope(
     116             :       const MachineBasicBlock::iterator &MI);
     117             : };
     118             : 
     119        5046 : class SIMemoryLegalizer final : public MachineFunctionPass {
     120             : private:
     121             :   /// \brief Machine module info.
     122             :   const AMDGPUMachineModuleInfo *MMI = nullptr;
     123             : 
     124             :   /// \brief Instruction info.
     125             :   const SIInstrInfo *TII = nullptr;
     126             : 
     127             :   /// \brief Immediate for "vmcnt(0)".
     128             :   unsigned Vmcnt0Immediate = 0;
     129             : 
     130             :   /// \brief Opcode for cache invalidation instruction (L1).
     131             :   unsigned VmemSIMDCacheInvalidateOpc = 0;
     132             : 
     133             :   /// \brief List of atomic pseudo instructions.
     134             :   std::list<MachineBasicBlock::iterator> AtomicPseudoMIs;
     135             : 
     136             :   /// \brief Sets named bit (BitName) to "true" if present in \p MI. Returns
     137             :   /// true if \p MI is modified, false otherwise.
     138             :   template <uint16_t BitName>
     139         263 :   bool enableNamedBit(const MachineBasicBlock::iterator &MI) const {
     140         526 :     int BitIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), BitName);
     141         263 :     if (BitIdx == -1)
     142             :       return false;
     143             : 
     144         173 :     MachineOperand &Bit = MI->getOperand(BitIdx);
     145         173 :     if (Bit.getImm() != 0)
     146             :       return false;
     147             : 
     148             :     Bit.setImm(1);
     149             :     return true;
     150             :   }
     151             : 
     152             :   /// \brief Sets GLC bit to "true" if present in \p MI. Returns true if \p MI
     153             :   /// is modified, false otherwise.
     154             :   bool enableGLCBit(const MachineBasicBlock::iterator &MI) const {
     155         218 :     return enableNamedBit<AMDGPU::OpName::glc>(MI);
     156             :   }
     157             : 
     158             :   /// \brief Sets SLC bit to "true" if present in \p MI. Returns true if \p MI
     159             :   /// is modified, false otherwise.
     160             :   bool enableSLCBit(const MachineBasicBlock::iterator &MI) const {
     161          45 :     return enableNamedBit<AMDGPU::OpName::slc>(MI);
     162             :   }
     163             : 
     164             :   /// \brief Inserts "buffer_wbinvl1_vol" instruction \p Before or after \p MI.
     165             :   /// Always returns true.
     166             :   bool insertVmemSIMDCacheInvalidate(MachineBasicBlock::iterator &MI,
     167             :                                      bool Before = true) const;
     168             :   /// \brief Inserts "s_waitcnt vmcnt(0)" instruction \p Before or after \p MI.
     169             :   /// Always returns true.
     170             :   bool insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI,
     171             :                            bool Before = true) const;
     172             : 
     173             :   /// \brief Removes all processed atomic pseudo instructions from the current
     174             :   /// function. Returns true if current function is modified, false otherwise.
     175             :   bool removeAtomicPseudoMIs();
     176             : 
     177             :   /// \brief Expands load operation \p MI. Returns true if instructions are
     178             :   /// added/deleted or \p MI is modified, false otherwise.
     179             :   bool expandLoad(const SIMemOpInfo &MOI,
     180             :                   MachineBasicBlock::iterator &MI);
     181             :   /// \brief Expands store operation \p MI. Returns true if instructions are
     182             :   /// added/deleted or \p MI is modified, false otherwise.
     183             :   bool expandStore(const SIMemOpInfo &MOI,
     184             :                    MachineBasicBlock::iterator &MI);
     185             :   /// \brief Expands atomic fence operation \p MI. Returns true if
     186             :   /// instructions are added/deleted or \p MI is modified, false otherwise.
     187             :   bool expandAtomicFence(const SIMemOpInfo &MOI,
     188             :                          MachineBasicBlock::iterator &MI);
     189             :   /// \brief Expands atomic cmpxchg or rmw operation \p MI. Returns true if
     190             :   /// instructions are added/deleted or \p MI is modified, false otherwise.
     191             :   bool expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,
     192             :                                 MachineBasicBlock::iterator &MI);
     193             : 
     194             : public:
     195             :   static char ID;
     196             : 
     197        1690 :   SIMemoryLegalizer() : MachineFunctionPass(ID) {}
     198             : 
     199        1683 :   void getAnalysisUsage(AnalysisUsage &AU) const override {
     200        1683 :     AU.setPreservesCFG();
     201        1683 :     MachineFunctionPass::getAnalysisUsage(AU);
     202        1683 :   }
     203             : 
     204        1683 :   StringRef getPassName() const override {
     205        1683 :     return PASS_NAME;
     206             :   }
     207             : 
     208             :   bool runOnMachineFunction(MachineFunction &MF) override;
     209             : };
     210             : 
     211             : } // end namespace anonymous
     212             : 
     213             : /* static */
     214       46058 : Optional<SIMemOpInfo> SIMemOpInfo::constructFromMIWithMMO(
     215             :     const MachineBasicBlock::iterator &MI) {
     216             :   assert(MI->getNumMemOperands() > 0);
     217             : 
     218       46058 :   const MachineFunction *MF = MI->getParent()->getParent();
     219             :   const AMDGPUMachineModuleInfo *MMI =
     220       46058 :       &MF->getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>();
     221             : 
     222             :   SyncScope::ID SSID = SyncScope::SingleThread;
     223             :   AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
     224             :   AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
     225             :   bool IsNonTemporal = true;
     226             : 
     227             :   // Validator should check whether or not MMOs cover the entire set of
     228             :   // locations accessed by the memory instruction.
     229      138274 :   for (const auto &MMO : MI->memoperands()) {
     230             :     const auto &IsSyncScopeInclusion =
     231       92232 :         MMI->isSyncScopeInclusion(SSID, MMO->getSyncScopeID());
     232       46116 :     if (!IsSyncScopeInclusion) {
     233           8 :       reportUnknownSyncScope(MI);
     234             :       return None;
     235             :     }
     236             : 
     237       46108 :     SSID = IsSyncScopeInclusion.getValue() ? SSID : MMO->getSyncScopeID();
     238             :     Ordering =
     239       92216 :         isStrongerThan(Ordering, MMO->getOrdering()) ?
     240             :             Ordering : MMO->getOrdering();
     241             :     FailureOrdering =
     242       46108 :         isStrongerThan(FailureOrdering, MMO->getFailureOrdering()) ?
     243             :             FailureOrdering : MMO->getFailureOrdering();
     244             : 
     245       92216 :     if (!(MMO->getFlags() & MachineMemOperand::MONonTemporal))
     246             :       IsNonTemporal = false;
     247             :   }
     248             : 
     249             :   return SIMemOpInfo(SSID, Ordering, FailureOrdering, IsNonTemporal);
     250             : }
     251             : 
     252             : /* static */
     253       46159 : Optional<SIMemOpInfo> SIMemOpInfo::getLoadInfo(
     254             :     const MachineBasicBlock::iterator &MI) {
     255             :   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
     256             : 
     257       65377 :   if (!(MI->mayLoad() && !MI->mayStore()))
     258             :     return None;
     259             : 
     260             :   // Be conservative if there are no memory operands.
     261       17882 :   if (MI->getNumMemOperands() == 0)
     262             :     return SIMemOpInfo(SyncScope::System,
     263             :                        AtomicOrdering::SequentiallyConsistent);
     264             : 
     265       17868 :   return SIMemOpInfo::constructFromMIWithMMO(MI);
     266             : }
     267             : 
     268             : /* static */
     269       28279 : Optional<SIMemOpInfo> SIMemOpInfo::getStoreInfo(
     270             :     const MachineBasicBlock::iterator &MI) {
     271             :   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
     272             : 
     273       55220 :   if (!(!MI->mayLoad() && MI->mayStore()))
     274             :     return None;
     275             : 
     276             :   // Be conservative if there are no memory operands.
     277       26866 :   if (MI->getNumMemOperands() == 0)
     278             :     return SIMemOpInfo(SyncScope::System,
     279             :                        AtomicOrdering::SequentiallyConsistent);
     280             : 
     281       26854 :   return SIMemOpInfo::constructFromMIWithMMO(MI);
     282             : }
     283             : 
     284             : /* static */
     285             : Optional<SIMemOpInfo> SIMemOpInfo::getAtomicFenceInfo(
     286             :     const MachineBasicBlock::iterator &MI) {
     287             :   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
     288             : 
     289        2830 :   if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE)
     290             :     return None;
     291             : 
     292             :   SyncScope::ID SSID =
     293          75 :       static_cast<SyncScope::ID>(MI->getOperand(1).getImm());
     294             :   AtomicOrdering Ordering =
     295          75 :       static_cast<AtomicOrdering>(MI->getOperand(0).getImm());
     296             :   return SIMemOpInfo(SSID, Ordering);
     297             : }
     298             : 
     299             : /* static */
     300        1340 : Optional<SIMemOpInfo> SIMemOpInfo::getAtomicCmpxchgOrRmwInfo(
     301             :     const MachineBasicBlock::iterator &MI) {
     302             :   assert(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic);
     303             : 
     304        2678 :   if (!(MI->mayLoad() && MI->mayStore()))
     305             :     return None;
     306             : 
     307             :   // Be conservative if there are no memory operands.
     308        1336 :   if (MI->getNumMemOperands() == 0)
     309             :     return SIMemOpInfo(SyncScope::System,
     310             :                        AtomicOrdering::SequentiallyConsistent,
     311             :                        AtomicOrdering::SequentiallyConsistent);
     312             : 
     313        1336 :   return SIMemOpInfo::constructFromMIWithMMO(MI);
     314             : }
     315             : 
     316             : /* static */
     317          10 : void SIMemOpInfo::reportUnknownSyncScope(
     318             :     const MachineBasicBlock::iterator &MI) {
     319             :   DiagnosticInfoUnsupported Diag(MI->getParent()->getParent()->getFunction(),
     320          10 :                                  "Unsupported synchronization scope");
     321          10 :   LLVMContext *CTX = &MI->getParent()->getParent()->getFunction().getContext();
     322          10 :   CTX->diagnose(Diag);
     323          10 : }
     324             : 
     325        1130 : bool SIMemoryLegalizer::insertVmemSIMDCacheInvalidate(
     326             :   MachineBasicBlock::iterator &MI, bool Before) const {
     327        1130 :   MachineBasicBlock &MBB = *MI->getParent();
     328             :   DebugLoc DL = MI->getDebugLoc();
     329             : 
     330        1130 :   if (!Before)
     331             :     ++MI;
     332             : 
     333        2260 :   BuildMI(MBB, MI, DL, TII->get(VmemSIMDCacheInvalidateOpc));
     334             : 
     335        1130 :   if (!Before)
     336             :     --MI;
     337             : 
     338        1130 :   return true;
     339             : }
     340             : 
     341        2308 : bool SIMemoryLegalizer::insertWaitcntVmcnt0(MachineBasicBlock::iterator &MI,
     342             :                                             bool Before) const {
     343        2308 :   MachineBasicBlock &MBB = *MI->getParent();
     344             :   DebugLoc DL = MI->getDebugLoc();
     345             : 
     346        2308 :   if (!Before)
     347             :     ++MI;
     348             : 
     349        6924 :   BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Vmcnt0Immediate);
     350             : 
     351        2308 :   if (!Before)
     352             :     --MI;
     353             : 
     354        2308 :   return true;
     355             : }
     356             : 
     357       16582 : bool SIMemoryLegalizer::removeAtomicPseudoMIs() {
     358       16582 :   if (AtomicPseudoMIs.empty())
     359             :     return false;
     360             : 
     361         140 :   for (auto &MI : AtomicPseudoMIs)
     362          73 :     MI->eraseFromParent();
     363             : 
     364             :   AtomicPseudoMIs.clear();
     365          67 :   return true;
     366             : }
     367             : 
     368       17880 : bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,
     369             :                                    MachineBasicBlock::iterator &MI) {
     370             :   assert(MI->mayLoad() && !MI->mayStore());
     371             : 
     372             :   bool Changed = false;
     373             : 
     374       17880 :   if (MOI.isAtomic()) {
     375         204 :     if (MOI.getSSID() == SyncScope::System ||
     376          65 :         MOI.getSSID() == MMI->getAgentSSID()) {
     377          83 :       if (MOI.getOrdering() == AtomicOrdering::Monotonic ||
     378         166 :           MOI.getOrdering() == AtomicOrdering::Acquire ||
     379             :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     380             :         Changed |= enableGLCBit(MI);
     381             : 
     382          91 :       if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     383          67 :         Changed |= insertWaitcntVmcnt0(MI);
     384             : 
     385          91 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     386             :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
     387          75 :         Changed |= insertWaitcntVmcnt0(MI, false);
     388          75 :         Changed |= insertVmemSIMDCacheInvalidate(MI, false);
     389             :       }
     390             : 
     391             :       return Changed;
     392             :     }
     393             : 
     394          32 :     if (MOI.getSSID() == SyncScope::SingleThread ||
     395          96 :         MOI.getSSID() == MMI->getWorkgroupSSID() ||
     396          16 :         MOI.getSSID() == MMI->getWavefrontSSID()) {
     397             :       return Changed;
     398             :     }
     399             : 
     400           0 :     llvm_unreachable("Unsupported synchronization scope");
     401             :   }
     402             : 
     403             :   // Atomic instructions do not have the nontemporal attribute.
     404       17741 :   if (MOI.isNonTemporal()) {
     405             :     Changed |= enableGLCBit(MI);
     406          21 :     Changed |= enableSLCBit(MI);
     407          21 :     return Changed;
     408             :   }
     409             : 
     410             :   return Changed;
     411             : }
     412             : 
     413       26864 : bool SIMemoryLegalizer::expandStore(const SIMemOpInfo &MOI,
     414             :                                     MachineBasicBlock::iterator &MI) {
     415             :   assert(!MI->mayLoad() && MI->mayStore());
     416             : 
     417             :   bool Changed = false;
     418             : 
     419       26864 :   if (MOI.isAtomic()) {
     420         206 :     if (MOI.getSSID() == SyncScope::System ||
     421          67 :         MOI.getSSID() == MMI->getAgentSSID()) {
     422         170 :       if (MOI.getOrdering() == AtomicOrdering::Release ||
     423             :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     424          73 :         Changed |= insertWaitcntVmcnt0(MI);
     425             : 
     426             :       return Changed;
     427             :     }
     428             : 
     429          34 :     if (MOI.getSSID() == SyncScope::SingleThread ||
     430         101 :         MOI.getSSID() == MMI->getWorkgroupSSID() ||
     431          17 :         MOI.getSSID() == MMI->getWavefrontSSID()) {
     432             :       return Changed;
     433             :     }
     434             : 
     435           0 :     llvm_unreachable("Unsupported synchronization scope");
     436             :   }
     437             : 
     438             :   // Atomic instructions do not have the nontemporal attribute.
     439       26725 :   if (MOI.isNonTemporal()) {
     440             :     Changed |= enableGLCBit(MI);
     441          24 :     Changed |= enableSLCBit(MI);
     442          24 :     return Changed;
     443             :   }
     444             : 
     445             :   return Changed;
     446             : }
     447             : 
     448          75 : bool SIMemoryLegalizer::expandAtomicFence(const SIMemOpInfo &MOI,
     449             :                                           MachineBasicBlock::iterator &MI) {
     450             :   assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE);
     451             : 
     452             :   bool Changed = false;
     453             : 
     454          75 :   if (MOI.isAtomic()) {
     455         137 :     if (MOI.getSSID() == SyncScope::System ||
     456          62 :         MOI.getSSID() == MMI->getAgentSSID()) {
     457          18 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     458          12 :           MOI.getOrdering() == AtomicOrdering::Release ||
     459          31 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     460             :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     461          25 :         Changed |= insertWaitcntVmcnt0(MI);
     462             : 
     463          43 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     464          37 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     465             :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
     466          19 :         Changed |= insertVmemSIMDCacheInvalidate(MI);
     467             : 
     468          25 :       AtomicPseudoMIs.push_back(MI);
     469          25 :       return Changed;
     470             :     }
     471             : 
     472          38 :     if (MOI.getSSID() == SyncScope::SingleThread ||
     473         102 :         MOI.getSSID() == MMI->getWorkgroupSSID() ||
     474          14 :         MOI.getSSID() == MMI->getWavefrontSSID()) {
     475          48 :       AtomicPseudoMIs.push_back(MI);
     476          48 :       return Changed;
     477             :     }
     478             : 
     479           2 :     SIMemOpInfo::reportUnknownSyncScope(MI);
     480             :   }
     481             : 
     482             :   return Changed;
     483             : }
     484             : 
     485        1332 : bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,
     486             :   MachineBasicBlock::iterator &MI) {
     487             :   assert(MI->mayLoad() && MI->mayStore());
     488             : 
     489             :   bool Changed = false;
     490             : 
     491        1332 :   if (MOI.isAtomic()) {
     492        1262 :     if (MOI.getSSID() == SyncScope::System ||
     493         120 :         MOI.getSSID() == MMI->getAgentSSID()) {
     494        1040 :       if (MOI.getOrdering() == AtomicOrdering::Release ||
     495        1028 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     496        1072 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
     497          20 :           MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
     498        1032 :         Changed |= insertWaitcntVmcnt0(MI);
     499             : 
     500        2092 :       if (MOI.getOrdering() == AtomicOrdering::Acquire ||
     501        1028 :           MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
     502          20 :           MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
     503        1088 :           MOI.getFailureOrdering() == AtomicOrdering::Acquire ||
     504             :           MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) {
     505        1036 :         Changed |= insertWaitcntVmcnt0(MI, false);
     506        1036 :         Changed |= insertVmemSIMDCacheInvalidate(MI, false);
     507             :       }
     508             : 
     509             :       return Changed;
     510             :     }
     511             : 
     512          60 :     if (MOI.getSSID() == SyncScope::SingleThread ||
     513         180 :         MOI.getSSID() == MMI->getWorkgroupSSID() ||
     514          30 :         MOI.getSSID() == MMI->getWavefrontSSID()) {
     515             :       Changed |= enableGLCBit(MI);
     516          90 :       return Changed;
     517             :     }
     518             : 
     519           0 :     llvm_unreachable("Unsupported synchronization scope");
     520             :   }
     521             : 
     522             :   return Changed;
     523             : }
     524             : 
     525       16582 : bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
     526             :   bool Changed = false;
     527       16582 :   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
     528       16582 :   const IsaInfo::IsaVersion IV = IsaInfo::getIsaVersion(ST.getFeatureBits());
     529             : 
     530       16582 :   MMI = &MF.getMMI().getObjFileInfo<AMDGPUMachineModuleInfo>();
     531       16582 :   TII = ST.getInstrInfo();
     532             : 
     533       16582 :   Vmcnt0Immediate =
     534       16582 :       AMDGPU::encodeWaitcnt(IV, 0, getExpcntBitMask(IV), getLgkmcntBitMask(IV));
     535       16582 :   VmemSIMDCacheInvalidateOpc =
     536       16582 :      ST.getGeneration() <= AMDGPUSubtarget::SOUTHERN_ISLANDS ?
     537             :        AMDGPU::BUFFER_WBINVL1 : AMDGPU::BUFFER_WBINVL1_VOL;
     538             : 
     539       35204 :   for (auto &MBB : MF) {
     540      317226 :     for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) {
     541      279982 :       if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic))
     542      233823 :         continue;
     543             : 
     544       46159 :       if (const auto &MOI = SIMemOpInfo::getLoadInfo(MI))
     545       17880 :         Changed |= expandLoad(MOI.getValue(), MI);
     546       28279 :       else if (const auto &MOI = SIMemOpInfo::getStoreInfo(MI))
     547       26864 :         Changed |= expandStore(MOI.getValue(), MI);
     548        1415 :       else if (const auto &MOI = SIMemOpInfo::getAtomicFenceInfo(MI))
     549          75 :         Changed |= expandAtomicFence(MOI.getValue(), MI);
     550        1340 :       else if (const auto &MOI = SIMemOpInfo::getAtomicCmpxchgOrRmwInfo(MI))
     551        1332 :         Changed |= expandAtomicCmpxchgOrRmw(MOI.getValue(), MI);
     552             :     }
     553             :   }
     554             : 
     555       16582 :   Changed |= removeAtomicPseudoMIs();
     556       16582 :   return Changed;
     557             : }
     558             : 
     559      279728 : INITIALIZE_PASS(SIMemoryLegalizer, DEBUG_TYPE, PASS_NAME, false, false)
     560             : 
     561             : char SIMemoryLegalizer::ID = 0;
     562             : char &llvm::SIMemoryLegalizerID = SIMemoryLegalizer::ID;
     563             : 
     564        1686 : FunctionPass *llvm::createSIMemoryLegalizerPass() {
     565        3372 :   return new SIMemoryLegalizer();
     566             : }

Generated by: LCOV version 1.13