LLVM  9.0.0svn
AMDGPUAtomicOptimizer.cpp
Go to the documentation of this file.
1 //===-- AMDGPUAtomicOptimizer.cpp -----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass optimizes atomic operations by using a single lane of a wavefront
11 /// to perform the atomic operation, thus reducing contention on that memory
12 /// location.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "AMDGPU.h"
17 #include "AMDGPUSubtarget.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/InstVisitor.h"
23 
24 #define DEBUG_TYPE "amdgpu-atomic-optimizer"
25 
26 using namespace llvm;
27 
28 namespace {
29 
30 enum DPP_CTRL {
31  DPP_ROW_SR1 = 0x111,
32  DPP_ROW_SR2 = 0x112,
33  DPP_ROW_SR3 = 0x113,
34  DPP_ROW_SR4 = 0x114,
35  DPP_ROW_SR8 = 0x118,
36  DPP_WF_SR1 = 0x138,
37  DPP_ROW_BCAST15 = 0x142,
38  DPP_ROW_BCAST31 = 0x143
39 };
40 
41 struct ReplacementInfo {
42  Instruction *I;
44  unsigned ValIdx;
45  bool ValDivergent;
46 };
47 
48 class AMDGPUAtomicOptimizer : public FunctionPass,
49  public InstVisitor<AMDGPUAtomicOptimizer> {
50 private:
52  const LegacyDivergenceAnalysis *DA;
53  const DataLayout *DL;
54  DominatorTree *DT;
55  bool HasDPP;
56  bool IsPixelShader;
57 
58  void optimizeAtomic(Instruction &I, Instruction::BinaryOps Op,
59  unsigned ValIdx, bool ValDivergent) const;
60 
61 public:
62  static char ID;
63 
64  AMDGPUAtomicOptimizer() : FunctionPass(ID) {}
65 
66  bool runOnFunction(Function &F) override;
67 
68  void getAnalysisUsage(AnalysisUsage &AU) const override {
72  }
73 
74  void visitAtomicRMWInst(AtomicRMWInst &I);
75  void visitIntrinsicInst(IntrinsicInst &I);
76 };
77 
78 } // namespace
79 
81 
83 
85  if (skipFunction(F)) {
86  return false;
87  }
88 
89  DA = &getAnalysis<LegacyDivergenceAnalysis>();
90  DL = &F.getParent()->getDataLayout();
91  DominatorTreeWrapperPass *const DTW =
92  getAnalysisIfAvailable<DominatorTreeWrapperPass>();
93  DT = DTW ? &DTW->getDomTree() : nullptr;
94  const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
95  const TargetMachine &TM = TPC.getTM<TargetMachine>();
96  const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
97  HasDPP = ST.hasDPP();
98  IsPixelShader = F.getCallingConv() == CallingConv::AMDGPU_PS;
99 
100  visit(F);
101 
102  const bool Changed = !ToReplace.empty();
103 
104  for (ReplacementInfo &Info : ToReplace) {
105  optimizeAtomic(*Info.I, Info.Op, Info.ValIdx, Info.ValDivergent);
106  }
107 
108  ToReplace.clear();
109 
110  return Changed;
111 }
112 
113 void AMDGPUAtomicOptimizer::visitAtomicRMWInst(AtomicRMWInst &I) {
114  // Early exit for unhandled address space atomic instructions.
115  switch (I.getPointerAddressSpace()) {
116  default:
117  return;
120  break;
121  }
122 
124 
125  switch (I.getOperation()) {
126  default:
127  return;
128  case AtomicRMWInst::Add:
129  Op = Instruction::Add;
130  break;
131  case AtomicRMWInst::Sub:
132  Op = Instruction::Sub;
133  break;
134  }
135 
136  const unsigned PtrIdx = 0;
137  const unsigned ValIdx = 1;
138 
139  // If the pointer operand is divergent, then each lane is doing an atomic
140  // operation on a different address, and we cannot optimize that.
141  if (DA->isDivergent(I.getOperand(PtrIdx))) {
142  return;
143  }
144 
145  const bool ValDivergent = DA->isDivergent(I.getOperand(ValIdx));
146 
147  // If the value operand is divergent, each lane is contributing a different
148  // value to the atomic calculation. We can only optimize divergent values if
149  // we have DPP available on our subtarget, and the atomic operation is 32
150  // bits.
151  if (ValDivergent && (!HasDPP || (DL->getTypeSizeInBits(I.getType()) != 32))) {
152  return;
153  }
154 
155  // If we get here, we can optimize the atomic using a single wavefront-wide
156  // atomic operation to do the calculation for the entire wavefront, so
157  // remember the instruction so we can come back to it.
158  const ReplacementInfo Info = {&I, Op, ValIdx, ValDivergent};
159 
160  ToReplace.push_back(Info);
161 }
162 
163 void AMDGPUAtomicOptimizer::visitIntrinsicInst(IntrinsicInst &I) {
165 
166  switch (I.getIntrinsicID()) {
167  default:
168  return;
169  case Intrinsic::amdgcn_buffer_atomic_add:
170  case Intrinsic::amdgcn_struct_buffer_atomic_add:
171  case Intrinsic::amdgcn_raw_buffer_atomic_add:
172  Op = Instruction::Add;
173  break;
174  case Intrinsic::amdgcn_buffer_atomic_sub:
175  case Intrinsic::amdgcn_struct_buffer_atomic_sub:
176  case Intrinsic::amdgcn_raw_buffer_atomic_sub:
177  Op = Instruction::Sub;
178  break;
179  }
180 
181  const unsigned ValIdx = 0;
182 
183  const bool ValDivergent = DA->isDivergent(I.getOperand(ValIdx));
184 
185  // If the value operand is divergent, each lane is contributing a different
186  // value to the atomic calculation. We can only optimize divergent values if
187  // we have DPP available on our subtarget, and the atomic operation is 32
188  // bits.
189  if (ValDivergent && (!HasDPP || (DL->getTypeSizeInBits(I.getType()) != 32))) {
190  return;
191  }
192 
193  // If any of the other arguments to the intrinsic are divergent, we can't
194  // optimize the operation.
195  for (unsigned Idx = 1; Idx < I.getNumOperands(); Idx++) {
196  if (DA->isDivergent(I.getOperand(Idx))) {
197  return;
198  }
199  }
200 
201  // If we get here, we can optimize the atomic using a single wavefront-wide
202  // atomic operation to do the calculation for the entire wavefront, so
203  // remember the instruction so we can come back to it.
204  const ReplacementInfo Info = {&I, Op, ValIdx, ValDivergent};
205 
206  ToReplace.push_back(Info);
207 }
208 
209 void AMDGPUAtomicOptimizer::optimizeAtomic(Instruction &I,
211  unsigned ValIdx,
212  bool ValDivergent) const {
213  // Start building just before the instruction.
214  IRBuilder<> B(&I);
215 
216  // If we are in a pixel shader, because of how we have to mask out helper
217  // lane invocations, we need to record the entry and exit BB's.
218  BasicBlock *PixelEntryBB = nullptr;
219  BasicBlock *PixelExitBB = nullptr;
220 
221  // If we're optimizing an atomic within a pixel shader, we need to wrap the
222  // entire atomic operation in a helper-lane check. We do not want any helper
223  // lanes that are around only for the purposes of derivatives to take part
224  // in any cross-lane communication, and we use a branch on whether the lane is
225  // live to do this.
226  if (IsPixelShader) {
227  // Record I's original position as the entry block.
228  PixelEntryBB = I.getParent();
229 
230  Value *const Cond = B.CreateIntrinsic(Intrinsic::amdgcn_ps_live, {}, {});
231  Instruction *const NonHelperTerminator =
232  SplitBlockAndInsertIfThen(Cond, &I, false, nullptr, DT, nullptr);
233 
234  // Record I's new position as the exit block.
235  PixelExitBB = I.getParent();
236 
237  I.moveBefore(NonHelperTerminator);
238  B.SetInsertPoint(&I);
239  }
240 
241  Type *const Ty = I.getType();
242  const unsigned TyBitWidth = DL->getTypeSizeInBits(Ty);
243  Type *const VecTy = VectorType::get(B.getInt32Ty(), 2);
244 
245  // This is the value in the atomic operation we need to combine in order to
246  // reduce the number of atomic operations.
247  Value *const V = I.getOperand(ValIdx);
248 
249  // We need to know how many lanes are active within the wavefront, and we do
250  // this by doing a ballot of active lanes.
251  CallInst *const Ballot =
252  B.CreateIntrinsic(Intrinsic::amdgcn_icmp, {B.getInt32Ty()},
253  {B.getInt32(1), B.getInt32(0), B.getInt32(33)});
254 
255  // We need to know how many lanes are active within the wavefront that are
256  // below us. If we counted each lane linearly starting from 0, a lane is
257  // below us only if its associated index was less than ours. We do this by
258  // using the mbcnt intrinsic.
259  Value *const BitCast = B.CreateBitCast(Ballot, VecTy);
260  Value *const ExtractLo = B.CreateExtractElement(BitCast, B.getInt32(0));
261  Value *const ExtractHi = B.CreateExtractElement(BitCast, B.getInt32(1));
262  CallInst *const PartialMbcnt = B.CreateIntrinsic(
263  Intrinsic::amdgcn_mbcnt_lo, {}, {ExtractLo, B.getInt32(0)});
264  CallInst *const Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {},
265  {ExtractHi, PartialMbcnt});
266 
267  Value *const MbcntCast = B.CreateIntCast(Mbcnt, Ty, false);
268 
269  Value *LaneOffset = nullptr;
270  Value *NewV = nullptr;
271 
272  // If we have a divergent value in each lane, we need to combine the value
273  // using DPP.
274  if (ValDivergent) {
275  Value *const Identity = B.getIntN(TyBitWidth, 0);
276 
277  // First we need to set all inactive invocations to 0, so that they can
278  // correctly contribute to the final result.
279  CallInst *const SetInactive =
280  B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, Ty, {V, Identity});
281 
282  CallInst *const FirstDPP =
283  B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, Ty,
284  {Identity, SetInactive, B.getInt32(DPP_WF_SR1),
285  B.getInt32(0xf), B.getInt32(0xf), B.getFalse()});
286  NewV = FirstDPP;
287 
288  const unsigned Iters = 7;
289  const unsigned DPPCtrl[Iters] = {
290  DPP_ROW_SR1, DPP_ROW_SR2, DPP_ROW_SR3, DPP_ROW_SR4,
291  DPP_ROW_SR8, DPP_ROW_BCAST15, DPP_ROW_BCAST31};
292  const unsigned RowMask[Iters] = {0xf, 0xf, 0xf, 0xf, 0xf, 0xa, 0xc};
293  const unsigned BankMask[Iters] = {0xf, 0xf, 0xf, 0xe, 0xc, 0xf, 0xf};
294 
295  // This loop performs an exclusive scan across the wavefront, with all lanes
296  // active (by using the WWM intrinsic).
297  for (unsigned Idx = 0; Idx < Iters; Idx++) {
298  Value *const UpdateValue = Idx < 3 ? FirstDPP : NewV;
299  CallInst *const DPP = B.CreateIntrinsic(
300  Intrinsic::amdgcn_update_dpp, Ty,
301  {Identity, UpdateValue, B.getInt32(DPPCtrl[Idx]),
302  B.getInt32(RowMask[Idx]), B.getInt32(BankMask[Idx]), B.getFalse()});
303 
304  NewV = B.CreateBinOp(Op, NewV, DPP);
305  }
306 
307  LaneOffset = B.CreateIntrinsic(Intrinsic::amdgcn_wwm, Ty, NewV);
308  NewV = B.CreateBinOp(Op, SetInactive, NewV);
309 
310  // Read the value from the last lane, which has accumlated the values of
311  // each active lane in the wavefront. This will be our new value with which
312  // we will provide to the atomic operation.
313  if (TyBitWidth == 64) {
314  Value *const ExtractLo = B.CreateTrunc(NewV, B.getInt32Ty());
315  Value *const ExtractHi =
316  B.CreateTrunc(B.CreateLShr(NewV, B.getInt64(32)), B.getInt32Ty());
317  CallInst *const ReadLaneLo = B.CreateIntrinsic(
318  Intrinsic::amdgcn_readlane, {}, {ExtractLo, B.getInt32(63)});
319  CallInst *const ReadLaneHi = B.CreateIntrinsic(
320  Intrinsic::amdgcn_readlane, {}, {ExtractHi, B.getInt32(63)});
321  Value *const PartialInsert = B.CreateInsertElement(
322  UndefValue::get(VecTy), ReadLaneLo, B.getInt32(0));
323  Value *const Insert =
324  B.CreateInsertElement(PartialInsert, ReadLaneHi, B.getInt32(1));
325  NewV = B.CreateBitCast(Insert, Ty);
326  } else if (TyBitWidth == 32) {
327  CallInst *const ReadLane = B.CreateIntrinsic(Intrinsic::amdgcn_readlane,
328  {}, {NewV, B.getInt32(63)});
329  NewV = ReadLane;
330  } else {
331  llvm_unreachable("Unhandled atomic bit width");
332  }
333 
334  // Finally mark the readlanes in the WWM section.
335  NewV = B.CreateIntrinsic(Intrinsic::amdgcn_wwm, Ty, NewV);
336  } else {
337  // Get the total number of active lanes we have by using popcount.
338  Instruction *const Ctpop = B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot);
339  Value *const CtpopCast = B.CreateIntCast(Ctpop, Ty, false);
340 
341  // Calculate the new value we will be contributing to the atomic operation
342  // for the entire wavefront.
343  NewV = B.CreateMul(V, CtpopCast);
344  LaneOffset = B.CreateMul(V, MbcntCast);
345  }
346 
347  // We only want a single lane to enter our new control flow, and we do this
348  // by checking if there are any active lanes below us. Only one lane will
349  // have 0 active lanes below us, so that will be the only one to progress.
350  Value *const Cond = B.CreateICmpEQ(MbcntCast, B.getIntN(TyBitWidth, 0));
351 
352  // Store I's original basic block before we split the block.
353  BasicBlock *const EntryBB = I.getParent();
354 
355  // We need to introduce some new control flow to force a single lane to be
356  // active. We do this by splitting I's basic block at I, and introducing the
357  // new block such that:
358  // entry --> single_lane -\
359  // \------------------> exit
360  Instruction *const SingleLaneTerminator =
361  SplitBlockAndInsertIfThen(Cond, &I, false, nullptr, DT, nullptr);
362 
363  // Move the IR builder into single_lane next.
364  B.SetInsertPoint(SingleLaneTerminator);
365 
366  // Clone the original atomic operation into single lane, replacing the
367  // original value with our newly created one.
368  Instruction *const NewI = I.clone();
369  B.Insert(NewI);
370  NewI->setOperand(ValIdx, NewV);
371 
372  // Move the IR builder into exit next, and start inserting just before the
373  // original instruction.
374  B.SetInsertPoint(&I);
375 
376  // Create a PHI node to get our new atomic result into the exit block.
377  PHINode *const PHI = B.CreatePHI(Ty, 2);
378  PHI->addIncoming(UndefValue::get(Ty), EntryBB);
379  PHI->addIncoming(NewI, SingleLaneTerminator->getParent());
380 
381  // We need to broadcast the value who was the lowest active lane (the first
382  // lane) to all other lanes in the wavefront. We use an intrinsic for this,
383  // but have to handle 64-bit broadcasts with two calls to this intrinsic.
384  Value *BroadcastI = nullptr;
385 
386  if (TyBitWidth == 64) {
387  Value *const ExtractLo = B.CreateTrunc(PHI, B.getInt32Ty());
388  Value *const ExtractHi =
389  B.CreateTrunc(B.CreateLShr(PHI, B.getInt64(32)), B.getInt32Ty());
390  CallInst *const ReadFirstLaneLo =
391  B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractLo);
392  CallInst *const ReadFirstLaneHi =
393  B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractHi);
394  Value *const PartialInsert = B.CreateInsertElement(
395  UndefValue::get(VecTy), ReadFirstLaneLo, B.getInt32(0));
396  Value *const Insert =
397  B.CreateInsertElement(PartialInsert, ReadFirstLaneHi, B.getInt32(1));
398  BroadcastI = B.CreateBitCast(Insert, Ty);
399  } else if (TyBitWidth == 32) {
400 
401  BroadcastI = B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, PHI);
402  } else {
403  llvm_unreachable("Unhandled atomic bit width");
404  }
405 
406  // Now that we have the result of our single atomic operation, we need to
407  // get our individual lane's slice into the result. We use the lane offset we
408  // previously calculated combined with the atomic result value we got from the
409  // first lane, to get our lane's index into the atomic result.
410  Value *const Result = B.CreateBinOp(Op, BroadcastI, LaneOffset);
411 
412  if (IsPixelShader) {
413  // Need a final PHI to reconverge to above the helper lane branch mask.
414  B.SetInsertPoint(PixelExitBB->getFirstNonPHI());
415 
416  PHINode *const PHI = B.CreatePHI(Ty, 2);
417  PHI->addIncoming(UndefValue::get(Ty), PixelEntryBB);
418  PHI->addIncoming(Result, I.getParent());
419  I.replaceAllUsesWith(PHI);
420  } else {
421  // Replace the original atomic instruction with the new one.
422  I.replaceAllUsesWith(Result);
423  }
424 
425  // And delete the original.
426  I.eraseFromParent();
427 }
428 
429 INITIALIZE_PASS_BEGIN(AMDGPUAtomicOptimizer, DEBUG_TYPE,
430  "AMDGPU atomic optimizations", false, false)
433 INITIALIZE_PASS_END(AMDGPUAtomicOptimizer, DEBUG_TYPE,
434  "AMDGPU atomic optimizations", false, false)
435 
437  return new AMDGPUAtomicOptimizer();
438 }
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:67
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1333
Base class for instruction visitors.
Definition: InstVisitor.h:80
AMDGPU specific subclass of TargetSubtarget.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
This class represents a function call, abstracting a target machine&#39;s calling convention.
TMC & getTM() const
Get the right type of TargetMachine for this target.
F(f)
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:691
Calling convention used for Mesa/AMDPAL pixel shaders.
Definition: CallingConv.h:194
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Definition: Instructions.h:819
char & AMDGPUAtomicOptimizerID
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:346
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
#define DEBUG_TYPE
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
BinOp getOperation() const
Definition: Instructions.h:750
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:742
DominatorTree & getDomTree()
Definition: Dominators.h:269
Target-Independent Code Generator Pass Configuration Options.
Instruction * clone() const
Create a copy of &#39;this&#39; instruction that is identical in all ways except the following: ...
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1767
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:429
CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Definition: IRBuilder.cpp:733
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:144
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Definition: IRBuilder.h:126
Value * getOperand(unsigned i) const
Definition: User.h:169
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Definition: IRBuilder.h:317
static bool runOnFunction(Function &F, bool PostInlining)
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:189
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Address space for global memory (RAT0, VTX0).
Definition: AMDGPU.h:251
Represent the analysis usage information of a pass.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1874
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:311
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2077
static UndefValue * get(Type *T)
Static factory methods - Return an &#39;undef&#39; object of the specified type.
Definition: Constants.cpp:1424
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1083
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1690
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:50
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2004
unsigned getNumOperands() const
Definition: User.h:191
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:212
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:1836
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type *> Types, ArrayRef< Value *> Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with args, mangled using Types.
Definition: IRBuilder.cpp:750
Value * CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2090
FunctionPass * createAMDGPUAtomicOptimizerPass()
AMDGPU atomic optimizations
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:306
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition: IRBuilder.h:291
#define I(x, y, z)
Definition: MD5.cpp:58
INITIALIZE_PASS_BEGIN(AMDGPUAtomicOptimizer, DEBUG_TYPE, "AMDGPU atomic optimizations", false, false) INITIALIZE_PASS_END(AMDGPUAtomicOptimizer
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Definition: IRBuilder.h:793
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:565
LLVM Value Representation.
Definition: Value.h:72
static VectorType * get(Type *ElementType, unsigned NumElements)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:605
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
Definition: Instruction.cpp:86
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1159
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:259
Address space for local memory.
Definition: AMDGPU.h:255
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:43
const BasicBlock * getParent() const
Definition: Instruction.h:66