32#include "llvm/IR/IntrinsicsAMDGPU.h"
37#define DEBUG_TYPE "amdgpu-atomic-optimizer"
44struct ReplacementInfo {
68class AMDGPUAtomicOptimizerImpl
81 Value *
const Identity)
const;
84 std::pair<Value *, Value *>
90 bool ValDivergent,
bool IsLDS)
const;
93 AMDGPUAtomicOptimizerImpl() =
delete;
98 :
F(
F), UA(UA),
DL(
F.getDataLayout()), DTU(DTU), ST(ST),
100 ScanImpl(ScanImpl) {}
110char AMDGPUAtomicOptimizer::ID = 0;
114bool AMDGPUAtomicOptimizer::runOnFunction(
Function &
F) {
115 if (skipFunction(
F)) {
120 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
123 getAnalysisIfAvailable<DominatorTreeWrapperPass>();
125 DomTreeUpdater::UpdateStrategy::Lazy);
131 return AMDGPUAtomicOptimizerImpl(
F, UA, DTU, ST, ScanImpl).run();
139 DomTreeUpdater::UpdateStrategy::Lazy);
142 bool IsChanged = AMDGPUAtomicOptimizerImpl(
F, UA, DTU, ST, ScanImpl).run();
153bool AMDGPUAtomicOptimizerImpl::run() {
161 if (ToReplace.empty())
164 for (
auto &[
I,
Op, ValIdx, ValDivergent, IsLDS] : ToReplace)
165 optimizeAtomic(*
I,
Op, ValIdx, ValDivergent, IsLDS);
171 switch (Ty->getTypeID()) {
176 unsigned Size = Ty->getIntegerBitWidth();
184void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &
I) {
185 if (
I.getType()->isVectorTy() ||
I.isVolatile())
189 switch (
I.getPointerAddressSpace()) {
220 !(
I.getType()->isFloatTy() ||
I.getType()->isDoubleTy())) {
224 const unsigned PtrIdx = 0;
225 const unsigned ValIdx = 1;
240 if (ScanImpl == ScanOptions::DPP && !ST.hasDPP())
251 if (IsLDS && ValDivergent && ScanImpl == ScanOptions::Iterative &&
253 TargetLowering::AtomicExpansionKind::None)
259 ToReplace.push_back({&
I,
Op, ValIdx, ValDivergent, IsLDS});
262void AMDGPUAtomicOptimizerImpl::visitIntrinsicInst(IntrinsicInst &
I) {
263 if (
I.getType()->isVectorTy())
268 switch (
I.getIntrinsicID()) {
271 case Intrinsic::amdgcn_struct_buffer_atomic_add:
272 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
273 case Intrinsic::amdgcn_raw_buffer_atomic_add:
274 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
277 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
278 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
279 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
280 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
283 case Intrinsic::amdgcn_struct_buffer_atomic_and:
284 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
285 case Intrinsic::amdgcn_raw_buffer_atomic_and:
286 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
289 case Intrinsic::amdgcn_struct_buffer_atomic_or:
290 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
291 case Intrinsic::amdgcn_raw_buffer_atomic_or:
292 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
295 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
296 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
297 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
298 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
301 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
302 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
303 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
304 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
307 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
308 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
309 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
310 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
313 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
314 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
315 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
316 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
319 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
320 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
321 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
322 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
327 const AMDGPU::BufferIntrinsicInfo &BufInfo =
342 if (ScanImpl == ScanOptions::DPP && !ST.hasDPP())
351 for (
unsigned Idx = 1;
Idx <
I.getNumOperands();
Idx++) {
360 ToReplace.push_back({&
I,
Op, ValIdx, ValDivergent,
false});
373 return B.CreateBinOp(Instruction::Add,
LHS,
RHS);
377 return B.CreateBinOp(Instruction::Sub,
LHS,
RHS);
381 return B.CreateBinOp(Instruction::And,
LHS,
RHS);
383 return B.CreateBinOp(Instruction::Or,
LHS,
RHS);
385 return B.CreateBinOp(Instruction::Xor,
LHS,
RHS);
400 return B.CreateMaxNum(
LHS,
RHS);
402 return B.CreateMinNum(
LHS,
RHS);
412 Value *Identity)
const {
413 Type *AtomicTy =
V->getType();
416 M, Intrinsic::amdgcn_update_dpp, AtomicTy);
421 B.CreateCall(UpdateDPP,
422 {Identity, V, B.getInt32(DPP::ROW_SHR0 | 1 << Idx),
423 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()}));
425 if (ST.hasDPPBroadcasts()) {
429 B.CreateCall(UpdateDPP,
430 {Identity, V, B.getInt32(DPP::BCAST15), B.getInt32(0xa),
431 B.getInt32(0xf), B.getFalse()}));
434 B.CreateCall(UpdateDPP,
435 {Identity, V, B.getInt32(DPP::BCAST31), B.getInt32(0xc),
436 B.getInt32(0xf), B.getFalse()}));
443 assert(ST.hasPermlane16Insts());
445 B.CreateIntrinsic(AtomicTy, Intrinsic::amdgcn_permlanex16,
447 B.getInt32(-1),
B.getFalse(),
B.getFalse()});
449 Value *UpdateDPPCall =
B.CreateCall(
451 B.getInt32(0xa),
B.getInt32(0xf),
B.getFalse()});
456 Value *
const Lane31 =
B.CreateIntrinsic(
457 AtomicTy, Intrinsic::amdgcn_readlane, {
V,
B.getInt32(31)});
459 Value *UpdateDPPCall =
B.CreateCall(
461 B.getInt32(0xc),
B.getInt32(0xf),
B.getFalse()});
472 Value *Identity)
const {
473 Type *AtomicTy =
V->getType();
476 M, Intrinsic::amdgcn_update_dpp, AtomicTy);
477 if (ST.hasDPPWavefrontShifts()) {
479 V =
B.CreateCall(UpdateDPP,
481 B.getInt32(0xf),
B.getFalse()});
484 M, Intrinsic::amdgcn_readlane, AtomicTy);
486 M, Intrinsic::amdgcn_writelane, AtomicTy);
491 V =
B.CreateCall(UpdateDPP,
493 B.getInt32(0xf),
B.getInt32(0xf),
B.getFalse()});
496 V =
B.CreateCall(WriteLane, {
B.CreateCall(ReadLane, {Old,
B.getInt32(15)}),
503 {
B.CreateCall(ReadLane, {Old,
B.getInt32(31)}),
B.getInt32(32),
V});
508 {
B.CreateCall(ReadLane, {Old,
B.getInt32(47)}),
B.getInt32(48),
V});
520std::pair<Value *, Value *> AMDGPUAtomicOptimizerImpl::buildScanIteratively(
522 Instruction &
I, BasicBlock *ComputeLoop, BasicBlock *ComputeEnd)
const {
523 auto *Ty =
I.getType();
525 auto *EntryBB =
I.getParent();
528 B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy,
B.getTrue());
531 B.SetInsertPoint(ComputeLoop);
535 auto *OldValuePhi =
B.CreatePHI(Ty, 2,
"OldValuePhi");
537 auto *ActiveBits =
B.CreatePHI(WaveTy, 2,
"ActiveBits");
538 ActiveBits->addIncoming(Ballot, EntryBB);
542 B.CreateIntrinsic(Intrinsic::cttz, WaveTy, {ActiveBits,
B.getTrue()});
544 auto *LaneIdxInt =
B.CreateTrunc(FF1,
B.getInt32Ty());
547 Value *LaneValue =
B.CreateIntrinsic(
V->getType(), Intrinsic::amdgcn_readlane,
552 Value *OldValue =
B.CreateIntrinsic(
V->getType(), Intrinsic::amdgcn_writelane,
553 {Accumulator, LaneIdxInt, OldValuePhi});
554 OldValuePhi->addIncoming(OldValue, ComputeLoop);
558 Accumulator->addIncoming(NewAccumulator, ComputeLoop);
562 auto *
Mask =
B.CreateShl(ConstantInt::get(WaveTy, 1), FF1);
564 auto *InverseMask =
B.CreateXor(Mask, ConstantInt::getAllOnesValue(WaveTy));
565 auto *NewActiveBits =
B.CreateAnd(ActiveBits, InverseMask);
566 ActiveBits->addIncoming(NewActiveBits, ComputeLoop);
569 auto *IsEnd =
B.CreateICmpEQ(NewActiveBits, ConstantInt::get(WaveTy, 0));
570 B.CreateCondBr(IsEnd, ComputeEnd, ComputeLoop);
572 B.SetInsertPoint(ComputeEnd);
574 return {OldValue, NewAccumulator};
580 const unsigned BitWidth = Ty->getPrimitiveSizeInBits();
615 "Atomic Op yet to be ported to use Wave Reduction intrinsics.");
618 return Intrinsic::amdgcn_wave_reduce_add;
621 return Intrinsic::amdgcn_wave_reduce_fadd;
623 return Intrinsic::amdgcn_wave_reduce_and;
625 return Intrinsic::amdgcn_wave_reduce_or;
627 return Intrinsic::amdgcn_wave_reduce_xor;
629 return Intrinsic::amdgcn_wave_reduce_umax;
631 return Intrinsic::amdgcn_wave_reduce_max;
633 return Intrinsic::amdgcn_wave_reduce_fmax;
635 return Intrinsic::amdgcn_wave_reduce_umin;
637 return Intrinsic::amdgcn_wave_reduce_min;
639 return Intrinsic::amdgcn_wave_reduce_fmin;
648void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &
I,
659 if (IsLDS && ValDivergent && ScanImpl == ScanOptions::DPP) {
660 if (MDNode *MD =
I.getMetadata(
"amdgpu.expected.active.lanes")) {
662 constexpr unsigned ActiveLanesThreshold = 5;
663 if (CI->getValue().ule(ActiveLanesThreshold))
672 B.setIsFPConstrained(
I.getFunction()->hasFnAttribute(Attribute::StrictFP));
689 Value *
const Cond =
B.CreateIntrinsic(Intrinsic::amdgcn_ps_live, {});
697 B.SetInsertPoint(&
I);
700 Type *
const Ty =
I.getType();
701 Type *Int32Ty =
B.getInt32Ty();
703 [[maybe_unused]]
const unsigned TyBitWidth =
DL.getTypeSizeInBits(Ty);
707 Value *
V =
I.getOperand(ValIdx);
712 CallInst *
const Ballot =
B.CreateIntrinsicWithoutFolding(
713 Intrinsic::amdgcn_ballot, WaveTy,
B.getTrue());
722 B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {Ballot,
B.getInt32(0)});
724 Value *
const ExtractLo =
B.CreateTrunc(Ballot, Int32Ty);
725 Value *
const ExtractHi =
B.CreateTrunc(
B.CreateLShr(Ballot, 32), Int32Ty);
726 Mbcnt =
B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo,
727 {ExtractLo,
B.getInt32(0)});
728 Mbcnt =
B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {ExtractHi, Mbcnt});
732 LLVMContext &
C =
F->getContext();
733 const bool NeedResult = !
I.use_empty();
734 const bool UseWaveReductionIntrinsic = !ValDivergent || !NeedResult;
746 Value *ExclScan =
nullptr;
747 Value *NewV =
nullptr;
751 if (UseWaveReductionIntrinsic) {
753 unsigned Strategy = ScanImpl == ScanOptions::DPP ? 2 : 1;
755 NewV =
B.CreateIntrinsic(WaveRedIntrinsic, Ty, {
V,
B.getInt32(Strategy)});
759 assert(ValDivergent && NeedResult);
760 if (ScanImpl == ScanOptions::DPP) {
764 B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, Ty, {
V, Identity});
765 NewV = buildScan(
B, ScanOp, NewV, Identity);
766 ExclScan = buildShiftRight(
B, NewV, Identity);
771 NewV =
B.CreateIntrinsic(Ty, Intrinsic::amdgcn_readlane,
772 {NewV, LastLaneIdx});
774 NewV =
B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, NewV);
775 }
else if (ScanImpl == ScanOptions::Iterative) {
779 std::tie(ExclScan, NewV) = buildScanIteratively(
B, ScanOp, Identity,
V,
I,
780 ComputeLoop, ComputeEnd);
789 Value *
const Cond =
B.CreateICmpEQ(Mbcnt,
B.getInt32(0));
811 if (!UseWaveReductionIntrinsic && ScanImpl == ScanOptions::Iterative) {
817 B.SetInsertPoint(ComputeEnd);
819 B.Insert(Terminator);
823 B.SetInsertPoint(OriginalBB);
824 B.CreateBr(ComputeLoop);
828 {{DominatorTree::Insert, OriginalBB, ComputeLoop},
829 {DominatorTree::Insert, ComputeLoop, ComputeEnd}});
834 DomTreeUpdates.push_back({DominatorTree::Insert, ComputeEnd, Succ});
835 DomTreeUpdates.push_back({DominatorTree::Delete, OriginalBB, Succ});
840 Predecessor = ComputeEnd;
842 Predecessor = OriginalBB;
845 B.SetInsertPoint(SingleLaneTerminator);
855 B.SetInsertPoint(&
I);
859 PHINode *
const PHI =
B.CreatePHI(Ty, 2);
861 PHI->addIncoming(NewI, SingleLaneTerminator->
getParent());
868 ReadlaneVal =
B.CreateZExt(
PHI,
B.getInt32Ty());
870 Value *BroadcastI =
B.CreateIntrinsic(
871 ReadlaneVal->
getType(), Intrinsic::amdgcn_readfirstlane, ReadlaneVal);
873 BroadcastI =
B.CreateTrunc(BroadcastI, Ty);
879 Value *LaneOffset =
nullptr;
881 if (ScanImpl == ScanOptions::DPP) {
883 B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, ExclScan);
884 }
else if (ScanImpl == ScanOptions::Iterative) {
885 LaneOffset = ExclScan;
890 Mbcnt = isAtomicFloatingPointTy ?
B.CreateUIToFP(Mbcnt, Ty)
891 :
B.CreateIntCast(Mbcnt, Ty,
false);
907 LaneOffset =
B.CreateSelect(
Cond, Identity,
V);
910 LaneOffset =
buildMul(
B,
V,
B.CreateAnd(Mbcnt, 1));
914 LaneOffset =
B.CreateFMul(
V, Mbcnt);
920 if (isAtomicFloatingPointTy) {
938 PHINode *
const PHI =
B.CreatePHI(Ty, 2);
940 PHI->addIncoming(Result,
I.getParent());
941 I.replaceAllUsesWith(
PHI);
944 I.replaceAllUsesWith(Result);
953 "AMDGPU atomic optimizations",
false,
false)
960 return new AMDGPUAtomicOptimizer(ScanStrategy);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static Constant * getIdentityValueForAtomicOp(Type *const Ty, AtomicRMWInst::BinOp Op)
static bool isLegalCrossLaneType(Type *Ty)
static Value * buildMul(IRBuilder<> &B, Value *LHS, Value *RHS)
static Value * buildNonAtomicBinOp(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *LHS, Value *RHS)
static Intrinsic::ID getWaveReductionIntrinsic(AtomicRMWInst::BinOp Op)
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool runOnFunction(Function &F, bool PostInlining)
AMD GCN specific subclass of TargetSubtarget.
Machine Check Debug Module
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
SmallVector< MachineOperand, 4 > Cond
static void visit(BasicBlock &Start, std::function< bool(BasicBlock *)> op)
Target-Independent Code Generator Pass Configuration Options pass.
bool isSingleLaneExecution(const Function &Kernel) const
Return true if only a single workitem can be active in a wave.
unsigned getWavefrontSize() const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
an instruction that atomically reads a memory location, combines it with another value,...
static bool isFPOperation(BinOp Op)
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ ICMP_ULT
unsigned less than
This is the shared class of boolean and integer constants.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
Analysis pass which computes a DominatorTree.
Legacy analysis pass which computes a DominatorTree.
DominatorTree & getDomTree()
FunctionPass class - This class is used to implement most global optimizations.
const SITargetLowering * getTargetLowering() const override
void applyUpdates(ArrayRef< UpdateT > Updates)
Submit updates to all available trees.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Base class for instruction visitors.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserve()
Mark an analysis as preserved.
AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Primary interface to the complete machine description for the target machine.
const STC & getSubtarget(const Function &F) const
This method returns a pointer to the specified type of TargetSubtargetInfo.
Target-Independent Code Generator Pass Configuration Options.
TMC & getTM() const
Get the right type of TargetMachine for this target.
The instances of the Type class are immutable: once they are created, they are never changed.
@ FloatTyID
32-bit floating point type
@ IntegerTyID
Arbitrary bit width integers.
@ DoubleTyID
64-bit floating point type
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
void setOperand(unsigned i, Value *Val)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ LOCAL_ADDRESS
Address space for local memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
const BufferIntrinsicInfo * lookupBufferIntrinsic(unsigned Intr)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ BasicBlock
Various leaf nodes.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
GenericUniformityInfo< SSAContext > UniformityInfo
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
DWARFExpression::Operation Op
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
char & AMDGPUAtomicOptimizerID
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
FunctionPass * createAMDGPUAtomicOptimizerPass(AMDGPU::ScanOptions ScanStrategy)
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
std::optional< unsigned > getAuxArg() const
The auxiliary (cache policy) argument.
std::optional< unsigned > getVDataArg() const
The stored value for stores and atomics.