LLVM 19.0.0git
AMDGPUAtomicOptimizer.cpp
Go to the documentation of this file.
1//===-- AMDGPUAtomicOptimizer.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass optimizes atomic operations by using a single lane of a wavefront
11/// to perform the atomic operation, thus reducing contention on that memory
12/// location.
13/// Atomic optimizer uses following strategies to compute scan and reduced
14/// values
15/// 1. DPP -
16/// This is the most efficient implementation for scan. DPP uses Whole Wave
17/// Mode (WWM)
18/// 2. Iterative -
19// An alternative implementation iterates over all active lanes
20/// of Wavefront using llvm.cttz and performs scan using readlane & writelane
21/// intrinsics
22//===----------------------------------------------------------------------===//
23
24#include "AMDGPU.h"
25#include "GCNSubtarget.h"
29#include "llvm/IR/IRBuilder.h"
30#include "llvm/IR/InstVisitor.h"
31#include "llvm/IR/IntrinsicsAMDGPU.h"
35
36#define DEBUG_TYPE "amdgpu-atomic-optimizer"
37
38using namespace llvm;
39using namespace llvm::AMDGPU;
40
41namespace {
42
43struct ReplacementInfo {
46 unsigned ValIdx;
47 bool ValDivergent;
48};
49
50class AMDGPUAtomicOptimizer : public FunctionPass {
51public:
52 static char ID;
53 ScanOptions ScanImpl;
54 AMDGPUAtomicOptimizer(ScanOptions ScanImpl)
55 : FunctionPass(ID), ScanImpl(ScanImpl) {}
56
57 bool runOnFunction(Function &F) override;
58
59 void getAnalysisUsage(AnalysisUsage &AU) const override {
63 }
64};
65
66class AMDGPUAtomicOptimizerImpl
67 : public InstVisitor<AMDGPUAtomicOptimizerImpl> {
68private:
70 const UniformityInfo *UA;
71 const DataLayout *DL;
72 DomTreeUpdater &DTU;
73 const GCNSubtarget *ST;
74 bool IsPixelShader;
75 ScanOptions ScanImpl;
76
77 Value *buildReduction(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
78 Value *const Identity) const;
80 Value *const Identity) const;
81 Value *buildShiftRight(IRBuilder<> &B, Value *V, Value *const Identity) const;
82
83 std::pair<Value *, Value *>
84 buildScanIteratively(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
85 Value *const Identity, Value *V, Instruction &I,
86 BasicBlock *ComputeLoop, BasicBlock *ComputeEnd) const;
87
88 void optimizeAtomic(Instruction &I, AtomicRMWInst::BinOp Op, unsigned ValIdx,
89 bool ValDivergent) const;
90
91public:
92 AMDGPUAtomicOptimizerImpl() = delete;
93
94 AMDGPUAtomicOptimizerImpl(const UniformityInfo *UA, const DataLayout *DL,
95 DomTreeUpdater &DTU, const GCNSubtarget *ST,
96 bool IsPixelShader, ScanOptions ScanImpl)
97 : UA(UA), DL(DL), DTU(DTU), ST(ST), IsPixelShader(IsPixelShader),
98 ScanImpl(ScanImpl) {}
99
100 bool run(Function &F);
101
104};
105
106} // namespace
107
108char AMDGPUAtomicOptimizer::ID = 0;
109
110char &llvm::AMDGPUAtomicOptimizerID = AMDGPUAtomicOptimizer::ID;
111
112bool AMDGPUAtomicOptimizer::runOnFunction(Function &F) {
113 if (skipFunction(F)) {
114 return false;
115 }
116
117 const UniformityInfo *UA =
118 &getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
119 const DataLayout *DL = &F.getParent()->getDataLayout();
120
121 DominatorTreeWrapperPass *const DTW =
122 getAnalysisIfAvailable<DominatorTreeWrapperPass>();
123 DomTreeUpdater DTU(DTW ? &DTW->getDomTree() : nullptr,
124 DomTreeUpdater::UpdateStrategy::Lazy);
125
126 const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
127 const TargetMachine &TM = TPC.getTM<TargetMachine>();
128 const GCNSubtarget *ST = &TM.getSubtarget<GCNSubtarget>(F);
129
130 bool IsPixelShader = F.getCallingConv() == CallingConv::AMDGPU_PS;
131
132 return AMDGPUAtomicOptimizerImpl(UA, DL, DTU, ST, IsPixelShader, ScanImpl)
133 .run(F);
134}
135
138
139 const auto *UA = &AM.getResult<UniformityInfoAnalysis>(F);
140 const DataLayout *DL = &F.getParent()->getDataLayout();
141
144 const GCNSubtarget *ST = &TM.getSubtarget<GCNSubtarget>(F);
145
146 bool IsPixelShader = F.getCallingConv() == CallingConv::AMDGPU_PS;
147
148 bool IsChanged =
149 AMDGPUAtomicOptimizerImpl(UA, DL, DTU, ST, IsPixelShader, ScanImpl)
150 .run(F);
151
152 if (!IsChanged) {
153 return PreservedAnalyses::all();
154 }
155
158 return PA;
159}
160
161bool AMDGPUAtomicOptimizerImpl::run(Function &F) {
162
163 // Scan option None disables the Pass
164 if (ScanImpl == ScanOptions::None) {
165 return false;
166 }
167
168 visit(F);
169
170 const bool Changed = !ToReplace.empty();
171
172 for (ReplacementInfo &Info : ToReplace) {
173 optimizeAtomic(*Info.I, Info.Op, Info.ValIdx, Info.ValDivergent);
174 }
175
176 ToReplace.clear();
177
178 return Changed;
179}
180
181void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
182 // Early exit for unhandled address space atomic instructions.
183 switch (I.getPointerAddressSpace()) {
184 default:
185 return;
188 break;
189 }
190
191 AtomicRMWInst::BinOp Op = I.getOperation();
192
193 switch (Op) {
194 default:
195 return;
209 break;
210 }
211
212 // Only 32-bit floating point atomic ops are supported.
213 if (AtomicRMWInst::isFPOperation(Op) && !I.getType()->isFloatTy()) {
214 return;
215 }
216
217 const unsigned PtrIdx = 0;
218 const unsigned ValIdx = 1;
219
220 // If the pointer operand is divergent, then each lane is doing an atomic
221 // operation on a different address, and we cannot optimize that.
222 if (UA->isDivergentUse(I.getOperandUse(PtrIdx))) {
223 return;
224 }
225
226 const bool ValDivergent = UA->isDivergentUse(I.getOperandUse(ValIdx));
227
228 // If the value operand is divergent, each lane is contributing a different
229 // value to the atomic calculation. We can only optimize divergent values if
230 // we have DPP available on our subtarget, and the atomic operation is 32
231 // bits.
232 if (ValDivergent &&
233 (!ST->hasDPP() || DL->getTypeSizeInBits(I.getType()) != 32)) {
234 return;
235 }
236
237 // If we get here, we can optimize the atomic using a single wavefront-wide
238 // atomic operation to do the calculation for the entire wavefront, so
239 // remember the instruction so we can come back to it.
240 const ReplacementInfo Info = {&I, Op, ValIdx, ValDivergent};
241
242 ToReplace.push_back(Info);
243}
244
245void AMDGPUAtomicOptimizerImpl::visitIntrinsicInst(IntrinsicInst &I) {
247
248 switch (I.getIntrinsicID()) {
249 default:
250 return;
251 case Intrinsic::amdgcn_buffer_atomic_add:
252 case Intrinsic::amdgcn_struct_buffer_atomic_add:
253 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
254 case Intrinsic::amdgcn_raw_buffer_atomic_add:
255 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
257 break;
258 case Intrinsic::amdgcn_buffer_atomic_sub:
259 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
260 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
261 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
262 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
264 break;
265 case Intrinsic::amdgcn_buffer_atomic_and:
266 case Intrinsic::amdgcn_struct_buffer_atomic_and:
267 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
268 case Intrinsic::amdgcn_raw_buffer_atomic_and:
269 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
271 break;
272 case Intrinsic::amdgcn_buffer_atomic_or:
273 case Intrinsic::amdgcn_struct_buffer_atomic_or:
274 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
275 case Intrinsic::amdgcn_raw_buffer_atomic_or:
276 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
278 break;
279 case Intrinsic::amdgcn_buffer_atomic_xor:
280 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
281 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
282 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
283 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
285 break;
286 case Intrinsic::amdgcn_buffer_atomic_smin:
287 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
288 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
289 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
290 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
292 break;
293 case Intrinsic::amdgcn_buffer_atomic_umin:
294 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
295 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
296 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
297 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
299 break;
300 case Intrinsic::amdgcn_buffer_atomic_smax:
301 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
302 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
303 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
304 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
306 break;
307 case Intrinsic::amdgcn_buffer_atomic_umax:
308 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
309 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
310 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
311 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
313 break;
314 }
315
316 const unsigned ValIdx = 0;
317
318 const bool ValDivergent = UA->isDivergentUse(I.getOperandUse(ValIdx));
319
320 // If the value operand is divergent, each lane is contributing a different
321 // value to the atomic calculation. We can only optimize divergent values if
322 // we have DPP available on our subtarget, and the atomic operation is 32
323 // bits.
324 if (ValDivergent &&
325 (!ST->hasDPP() || DL->getTypeSizeInBits(I.getType()) != 32)) {
326 return;
327 }
328
329 // If any of the other arguments to the intrinsic are divergent, we can't
330 // optimize the operation.
331 for (unsigned Idx = 1; Idx < I.getNumOperands(); Idx++) {
332 if (UA->isDivergentUse(I.getOperandUse(Idx))) {
333 return;
334 }
335 }
336
337 // If we get here, we can optimize the atomic using a single wavefront-wide
338 // atomic operation to do the calculation for the entire wavefront, so
339 // remember the instruction so we can come back to it.
340 const ReplacementInfo Info = {&I, Op, ValIdx, ValDivergent};
341
342 ToReplace.push_back(Info);
343}
344
345// Use the builder to create the non-atomic counterpart of the specified
346// atomicrmw binary op.
348 Value *LHS, Value *RHS) {
350
351 switch (Op) {
352 default:
353 llvm_unreachable("Unhandled atomic op");
355 return B.CreateBinOp(Instruction::Add, LHS, RHS);
357 return B.CreateFAdd(LHS, RHS);
359 return B.CreateBinOp(Instruction::Sub, LHS, RHS);
361 return B.CreateFSub(LHS, RHS);
363 return B.CreateBinOp(Instruction::And, LHS, RHS);
365 return B.CreateBinOp(Instruction::Or, LHS, RHS);
367 return B.CreateBinOp(Instruction::Xor, LHS, RHS);
368
370 Pred = CmpInst::ICMP_SGT;
371 break;
373 Pred = CmpInst::ICMP_SLT;
374 break;
376 Pred = CmpInst::ICMP_UGT;
377 break;
379 Pred = CmpInst::ICMP_ULT;
380 break;
382 return B.CreateMaxNum(LHS, RHS);
384 return B.CreateMinNum(LHS, RHS);
385 }
386 Value *Cond = B.CreateICmp(Pred, LHS, RHS);
387 return B.CreateSelect(Cond, LHS, RHS);
388}
389
390// Use the builder to create a reduction of V across the wavefront, with all
391// lanes active, returning the same result in all lanes.
392Value *AMDGPUAtomicOptimizerImpl::buildReduction(IRBuilder<> &B,
394 Value *V,
395 Value *const Identity) const {
396 Type *AtomicTy = V->getType();
397 Type *IntNTy = B.getIntNTy(AtomicTy->getPrimitiveSizeInBits());
398 Module *M = B.GetInsertBlock()->getModule();
399 Function *UpdateDPP =
400 Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, AtomicTy);
401
402 // Reduce within each row of 16 lanes.
403 for (unsigned Idx = 0; Idx < 4; Idx++) {
405 B, Op, V,
406 B.CreateCall(UpdateDPP,
407 {Identity, V, B.getInt32(DPP::ROW_XMASK0 | 1 << Idx),
408 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()}));
409 }
410
411 // Reduce within each pair of rows (i.e. 32 lanes).
412 assert(ST->hasPermLaneX16());
413 V = B.CreateBitCast(V, IntNTy);
414 Value *Permlanex16Call = B.CreateIntrinsic(
415 Intrinsic::amdgcn_permlanex16, {},
416 {V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()});
417 V = buildNonAtomicBinOp(B, Op, B.CreateBitCast(V, AtomicTy),
418 B.CreateBitCast(Permlanex16Call, AtomicTy));
419 if (ST->isWave32()) {
420 return V;
421 }
422
423 if (ST->hasPermLane64()) {
424 // Reduce across the upper and lower 32 lanes.
425 V = B.CreateBitCast(V, IntNTy);
426 Value *Permlane64Call =
427 B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {}, V);
428 return buildNonAtomicBinOp(B, Op, B.CreateBitCast(V, AtomicTy),
429 B.CreateBitCast(Permlane64Call, AtomicTy));
430 }
431
432 // Pick an arbitrary lane from 0..31 and an arbitrary lane from 32..63 and
433 // combine them with a scalar operation.
434 Function *ReadLane =
435 Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, {});
436 V = B.CreateBitCast(V, IntNTy);
437 Value *Lane0 = B.CreateCall(ReadLane, {V, B.getInt32(0)});
438 Value *Lane32 = B.CreateCall(ReadLane, {V, B.getInt32(32)});
439 return buildNonAtomicBinOp(B, Op, B.CreateBitCast(Lane0, AtomicTy),
440 B.CreateBitCast(Lane32, AtomicTy));
441}
442
443// Use the builder to create an inclusive scan of V across the wavefront, with
444// all lanes active.
445Value *AMDGPUAtomicOptimizerImpl::buildScan(IRBuilder<> &B,
447 Value *Identity) const {
448 Type *AtomicTy = V->getType();
449 Type *IntNTy = B.getIntNTy(AtomicTy->getPrimitiveSizeInBits());
450
451 Module *M = B.GetInsertBlock()->getModule();
452 Function *UpdateDPP =
453 Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, AtomicTy);
454
455 for (unsigned Idx = 0; Idx < 4; Idx++) {
457 B, Op, V,
458 B.CreateCall(UpdateDPP,
459 {Identity, V, B.getInt32(DPP::ROW_SHR0 | 1 << Idx),
460 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()}));
461 }
462 if (ST->hasDPPBroadcasts()) {
463 // GFX9 has DPP row broadcast operations.
465 B, Op, V,
466 B.CreateCall(UpdateDPP,
467 {Identity, V, B.getInt32(DPP::BCAST15), B.getInt32(0xa),
468 B.getInt32(0xf), B.getFalse()}));
470 B, Op, V,
471 B.CreateCall(UpdateDPP,
472 {Identity, V, B.getInt32(DPP::BCAST31), B.getInt32(0xc),
473 B.getInt32(0xf), B.getFalse()}));
474 } else {
475 // On GFX10 all DPP operations are confined to a single row. To get cross-
476 // row operations we have to use permlane or readlane.
477
478 // Combine lane 15 into lanes 16..31 (and, for wave 64, lane 47 into lanes
479 // 48..63).
480 assert(ST->hasPermLaneX16());
481 V = B.CreateBitCast(V, IntNTy);
482 Value *PermX = B.CreateIntrinsic(
483 Intrinsic::amdgcn_permlanex16, {},
484 {V, V, B.getInt32(-1), B.getInt32(-1), B.getFalse(), B.getFalse()});
485
486 Value *UpdateDPPCall =
487 B.CreateCall(UpdateDPP, {Identity, B.CreateBitCast(PermX, AtomicTy),
488 B.getInt32(DPP::QUAD_PERM_ID), B.getInt32(0xa),
489 B.getInt32(0xf), B.getFalse()});
490 V = buildNonAtomicBinOp(B, Op, B.CreateBitCast(V, AtomicTy), UpdateDPPCall);
491
492 if (!ST->isWave32()) {
493 // Combine lane 31 into lanes 32..63.
494 V = B.CreateBitCast(V, IntNTy);
495 Value *const Lane31 = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {},
496 {V, B.getInt32(31)});
497
498 Value *UpdateDPPCall = B.CreateCall(
499 UpdateDPP, {Identity, Lane31, B.getInt32(DPP::QUAD_PERM_ID),
500 B.getInt32(0xc), B.getInt32(0xf), B.getFalse()});
501
502 V = buildNonAtomicBinOp(B, Op, B.CreateBitCast(V, AtomicTy),
503 UpdateDPPCall);
504 }
505 }
506 return V;
507}
508
509// Use the builder to create a shift right of V across the wavefront, with all
510// lanes active, to turn an inclusive scan into an exclusive scan.
511Value *AMDGPUAtomicOptimizerImpl::buildShiftRight(IRBuilder<> &B, Value *V,
512 Value *Identity) const {
513 Type *AtomicTy = V->getType();
514 Type *IntNTy = B.getIntNTy(AtomicTy->getPrimitiveSizeInBits());
515
516 Module *M = B.GetInsertBlock()->getModule();
517 Function *UpdateDPP =
518 Intrinsic::getDeclaration(M, Intrinsic::amdgcn_update_dpp, AtomicTy);
519 if (ST->hasDPPWavefrontShifts()) {
520 // GFX9 has DPP wavefront shift operations.
521 V = B.CreateCall(UpdateDPP,
522 {Identity, V, B.getInt32(DPP::WAVE_SHR1), B.getInt32(0xf),
523 B.getInt32(0xf), B.getFalse()});
524 } else {
525 Function *ReadLane =
526 Intrinsic::getDeclaration(M, Intrinsic::amdgcn_readlane, {});
527 Function *WriteLane =
528 Intrinsic::getDeclaration(M, Intrinsic::amdgcn_writelane, {});
529
530 // On GFX10 all DPP operations are confined to a single row. To get cross-
531 // row operations we have to use permlane or readlane.
532 Value *Old = V;
533 V = B.CreateCall(UpdateDPP,
534 {Identity, V, B.getInt32(DPP::ROW_SHR0 + 1),
535 B.getInt32(0xf), B.getInt32(0xf), B.getFalse()});
536
537 // Copy the old lane 15 to the new lane 16.
538 V = B.CreateCall(
539 WriteLane,
540 {B.CreateCall(ReadLane, {B.CreateBitCast(Old, IntNTy), B.getInt32(15)}),
541 B.getInt32(16), B.CreateBitCast(V, IntNTy)});
542 V = B.CreateBitCast(V, AtomicTy);
543 if (!ST->isWave32()) {
544 // Copy the old lane 31 to the new lane 32.
545 V = B.CreateBitCast(V, IntNTy);
546 V = B.CreateCall(WriteLane,
547 {B.CreateCall(ReadLane, {B.CreateBitCast(Old, IntNTy),
548 B.getInt32(31)}),
549 B.getInt32(32), V});
550
551 // Copy the old lane 47 to the new lane 48.
552 V = B.CreateCall(
553 WriteLane,
554 {B.CreateCall(ReadLane, {Old, B.getInt32(47)}), B.getInt32(48), V});
555 V = B.CreateBitCast(V, AtomicTy);
556 }
557 }
558
559 return V;
560}
561
562// Use the builder to create an exclusive scan and compute the final reduced
563// value using an iterative approach. This provides an alternative
564// implementation to DPP which uses WMM for scan computations. This API iterate
565// over active lanes to read, compute and update the value using
566// readlane and writelane intrinsics.
567std::pair<Value *, Value *> AMDGPUAtomicOptimizerImpl::buildScanIteratively(
568 IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *const Identity, Value *V,
569 Instruction &I, BasicBlock *ComputeLoop, BasicBlock *ComputeEnd) const {
570 auto *Ty = I.getType();
571 auto *WaveTy = B.getIntNTy(ST->getWavefrontSize());
572 auto *EntryBB = I.getParent();
573 auto NeedResult = !I.use_empty();
574
575 auto *Ballot =
576 B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue());
577
578 // Start inserting instructions for ComputeLoop block
579 B.SetInsertPoint(ComputeLoop);
580 // Phi nodes for Accumulator, Scan results destination, and Active Lanes
581 auto *Accumulator = B.CreatePHI(Ty, 2, "Accumulator");
582 Accumulator->addIncoming(Identity, EntryBB);
583 PHINode *OldValuePhi = nullptr;
584 if (NeedResult) {
585 OldValuePhi = B.CreatePHI(Ty, 2, "OldValuePhi");
586 OldValuePhi->addIncoming(PoisonValue::get(Ty), EntryBB);
587 }
588 auto *ActiveBits = B.CreatePHI(WaveTy, 2, "ActiveBits");
589 ActiveBits->addIncoming(Ballot, EntryBB);
590
591 // Use llvm.cttz instrinsic to find the lowest remaining active lane.
592 auto *FF1 =
593 B.CreateIntrinsic(Intrinsic::cttz, WaveTy, {ActiveBits, B.getTrue()});
594
595 Type *IntNTy = B.getIntNTy(Ty->getPrimitiveSizeInBits());
596 auto *LaneIdxInt = B.CreateTrunc(FF1, IntNTy);
597
598 // Get the value required for atomic operation
599 V = B.CreateBitCast(V, IntNTy);
600 Value *LaneValue =
601 B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {}, {V, LaneIdxInt});
602 LaneValue = B.CreateBitCast(LaneValue, Ty);
603
604 // Perform writelane if intermediate scan results are required later in the
605 // kernel computations
606 Value *OldValue = nullptr;
607 if (NeedResult) {
608 OldValue =
609 B.CreateIntrinsic(Intrinsic::amdgcn_writelane, {},
610 {B.CreateBitCast(Accumulator, IntNTy), LaneIdxInt,
611 B.CreateBitCast(OldValuePhi, IntNTy)});
612 OldValue = B.CreateBitCast(OldValue, Ty);
613 OldValuePhi->addIncoming(OldValue, ComputeLoop);
614 }
615
616 // Accumulate the results
617 auto *NewAccumulator = buildNonAtomicBinOp(B, Op, Accumulator, LaneValue);
618 Accumulator->addIncoming(NewAccumulator, ComputeLoop);
619
620 // Set bit to zero of current active lane so that for next iteration llvm.cttz
621 // return the next active lane
622 auto *Mask = B.CreateShl(ConstantInt::get(WaveTy, 1), FF1);
623
624 auto *InverseMask = B.CreateXor(Mask, ConstantInt::get(WaveTy, -1));
625 auto *NewActiveBits = B.CreateAnd(ActiveBits, InverseMask);
626 ActiveBits->addIncoming(NewActiveBits, ComputeLoop);
627
628 // Branch out of the loop when all lanes are processed.
629 auto *IsEnd = B.CreateICmpEQ(NewActiveBits, ConstantInt::get(WaveTy, 0));
630 B.CreateCondBr(IsEnd, ComputeEnd, ComputeLoop);
631
632 B.SetInsertPoint(ComputeEnd);
633
634 return {OldValue, NewAccumulator};
635}
636
639 LLVMContext &C = Ty->getContext();
640 const unsigned BitWidth = Ty->getPrimitiveSizeInBits();
641 switch (Op) {
642 default:
643 llvm_unreachable("Unhandled atomic op");
660 return ConstantFP::get(C, APFloat::getZero(Ty->getFltSemantics(), false));
662 return ConstantFP::get(C, APFloat::getInf(Ty->getFltSemantics(), false));
664 return ConstantFP::get(C, APFloat::getInf(Ty->getFltSemantics(), true));
665 }
666}
667
668static Value *buildMul(IRBuilder<> &B, Value *LHS, Value *RHS) {
669 const ConstantInt *CI = dyn_cast<ConstantInt>(LHS);
670 return (CI && CI->isOne()) ? RHS : B.CreateMul(LHS, RHS);
671}
672
673void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
675 unsigned ValIdx,
676 bool ValDivergent) const {
677 // Start building just before the instruction.
678 IRBuilder<> B(&I);
679
681 B.setIsFPConstrained(I.getFunction()->hasFnAttribute(Attribute::StrictFP));
682 }
683
684 // If we are in a pixel shader, because of how we have to mask out helper
685 // lane invocations, we need to record the entry and exit BB's.
686 BasicBlock *PixelEntryBB = nullptr;
687 BasicBlock *PixelExitBB = nullptr;
688
689 // If we're optimizing an atomic within a pixel shader, we need to wrap the
690 // entire atomic operation in a helper-lane check. We do not want any helper
691 // lanes that are around only for the purposes of derivatives to take part
692 // in any cross-lane communication, and we use a branch on whether the lane is
693 // live to do this.
694 if (IsPixelShader) {
695 // Record I's original position as the entry block.
696 PixelEntryBB = I.getParent();
697
698 Value *const Cond = B.CreateIntrinsic(Intrinsic::amdgcn_ps_live, {}, {});
699 Instruction *const NonHelperTerminator =
700 SplitBlockAndInsertIfThen(Cond, &I, false, nullptr, &DTU, nullptr);
701
702 // Record I's new position as the exit block.
703 PixelExitBB = I.getParent();
704
705 I.moveBefore(NonHelperTerminator);
706 B.SetInsertPoint(&I);
707 }
708
709 Type *const Ty = I.getType();
710 Type *Int32Ty = B.getInt32Ty();
711 Type *IntNTy = B.getIntNTy(Ty->getPrimitiveSizeInBits());
712 bool isAtomicFloatingPointTy = Ty->isFloatingPointTy();
713 const unsigned TyBitWidth = DL->getTypeSizeInBits(Ty);
714 auto *const VecTy = FixedVectorType::get(Int32Ty, 2);
715
716 // This is the value in the atomic operation we need to combine in order to
717 // reduce the number of atomic operations.
718 Value *V = I.getOperand(ValIdx);
719
720 // We need to know how many lanes are active within the wavefront, and we do
721 // this by doing a ballot of active lanes.
722 Type *const WaveTy = B.getIntNTy(ST->getWavefrontSize());
723 CallInst *const Ballot =
724 B.CreateIntrinsic(Intrinsic::amdgcn_ballot, WaveTy, B.getTrue());
725
726 // We need to know how many lanes are active within the wavefront that are
727 // below us. If we counted each lane linearly starting from 0, a lane is
728 // below us only if its associated index was less than ours. We do this by
729 // using the mbcnt intrinsic.
730 Value *Mbcnt;
731 if (ST->isWave32()) {
732 Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {},
733 {Ballot, B.getInt32(0)});
734 } else {
735 Value *const ExtractLo = B.CreateTrunc(Ballot, Int32Ty);
736 Value *const ExtractHi = B.CreateTrunc(B.CreateLShr(Ballot, 32), Int32Ty);
737 Mbcnt = B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {},
738 {ExtractLo, B.getInt32(0)});
739 Mbcnt =
740 B.CreateIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {}, {ExtractHi, Mbcnt});
741 }
742
743 Function *F = I.getFunction();
744 LLVMContext &C = F->getContext();
745
746 // For atomic sub, perform scan with add operation and allow one lane to
747 // subtract the reduced value later.
748 AtomicRMWInst::BinOp ScanOp = Op;
749 if (Op == AtomicRMWInst::Sub) {
750 ScanOp = AtomicRMWInst::Add;
751 } else if (Op == AtomicRMWInst::FSub) {
752 ScanOp = AtomicRMWInst::FAdd;
753 }
754 Value *Identity = getIdentityValueForAtomicOp(Ty, ScanOp);
755
756 Value *ExclScan = nullptr;
757 Value *NewV = nullptr;
758
759 const bool NeedResult = !I.use_empty();
760
761 BasicBlock *ComputeLoop = nullptr;
762 BasicBlock *ComputeEnd = nullptr;
763 // If we have a divergent value in each lane, we need to combine the value
764 // using DPP.
765 if (ValDivergent) {
766 if (ScanImpl == ScanOptions::DPP) {
767 // First we need to set all inactive invocations to the identity value, so
768 // that they can correctly contribute to the final result.
769 V = B.CreateBitCast(V, IntNTy);
770 Identity = B.CreateBitCast(Identity, IntNTy);
771 NewV = B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, IntNTy,
772 {V, Identity});
773 NewV = B.CreateBitCast(NewV, Ty);
774 V = B.CreateBitCast(V, Ty);
775 Identity = B.CreateBitCast(Identity, Ty);
776 if (!NeedResult && ST->hasPermLaneX16()) {
777 // On GFX10 the permlanex16 instruction helps us build a reduction
778 // without too many readlanes and writelanes, which are generally bad
779 // for performance.
780 NewV = buildReduction(B, ScanOp, NewV, Identity);
781 } else {
782 NewV = buildScan(B, ScanOp, NewV, Identity);
783 if (NeedResult)
784 ExclScan = buildShiftRight(B, NewV, Identity);
785 // Read the value from the last lane, which has accumulated the values
786 // of each active lane in the wavefront. This will be our new value
787 // which we will provide to the atomic operation.
788 Value *const LastLaneIdx = B.getInt32(ST->getWavefrontSize() - 1);
789 assert(TyBitWidth == 32);
790 NewV = B.CreateBitCast(NewV, IntNTy);
791 NewV = B.CreateIntrinsic(Intrinsic::amdgcn_readlane, {},
792 {NewV, LastLaneIdx});
793 NewV = B.CreateBitCast(NewV, Ty);
794 }
795 // Finally mark the readlanes in the WWM section.
796 NewV = B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, NewV);
797 } else if (ScanImpl == ScanOptions::Iterative) {
798 // Alternative implementation for scan
799 ComputeLoop = BasicBlock::Create(C, "ComputeLoop", F);
800 ComputeEnd = BasicBlock::Create(C, "ComputeEnd", F);
801 std::tie(ExclScan, NewV) = buildScanIteratively(B, ScanOp, Identity, V, I,
802 ComputeLoop, ComputeEnd);
803 } else {
804 llvm_unreachable("Atomic Optimzer is disabled for None strategy");
805 }
806 } else {
807 switch (Op) {
808 default:
809 llvm_unreachable("Unhandled atomic op");
810
812 case AtomicRMWInst::Sub: {
813 // The new value we will be contributing to the atomic operation is the
814 // old value times the number of active lanes.
815 Value *const Ctpop = B.CreateIntCast(
816 B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false);
817 NewV = buildMul(B, V, Ctpop);
818 break;
819 }
821 case AtomicRMWInst::FSub: {
822 Value *const Ctpop = B.CreateIntCast(
823 B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Int32Ty, false);
824 Value *const CtpopFP = B.CreateUIToFP(Ctpop, Ty);
825 NewV = B.CreateFMul(V, CtpopFP);
826 break;
827 }
836 // These operations with a uniform value are idempotent: doing the atomic
837 // operation multiple times has the same effect as doing it once.
838 NewV = V;
839 break;
840
842 // The new value we will be contributing to the atomic operation is the
843 // old value times the parity of the number of active lanes.
844 Value *const Ctpop = B.CreateIntCast(
845 B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false);
846 NewV = buildMul(B, V, B.CreateAnd(Ctpop, 1));
847 break;
848 }
849 }
850
851 // We only want a single lane to enter our new control flow, and we do this
852 // by checking if there are any active lanes below us. Only one lane will
853 // have 0 active lanes below us, so that will be the only one to progress.
854 Value *const Cond = B.CreateICmpEQ(Mbcnt, B.getInt32(0));
855
856 // Store I's original basic block before we split the block.
857 BasicBlock *const EntryBB = I.getParent();
858
859 // We need to introduce some new control flow to force a single lane to be
860 // active. We do this by splitting I's basic block at I, and introducing the
861 // new block such that:
862 // entry --> single_lane -\
863 // \------------------> exit
864 Instruction *const SingleLaneTerminator =
865 SplitBlockAndInsertIfThen(Cond, &I, false, nullptr, &DTU, nullptr);
866
867 // At this point, we have split the I's block to allow one lane in wavefront
868 // to update the precomputed reduced value. Also, completed the codegen for
869 // new control flow i.e. iterative loop which perform reduction and scan using
870 // ComputeLoop and ComputeEnd.
871 // For the new control flow, we need to move branch instruction i.e.
872 // terminator created during SplitBlockAndInsertIfThen from I's block to
873 // ComputeEnd block. We also need to set up predecessor to next block when
874 // single lane done updating the final reduced value.
875 BasicBlock *Predecessor = nullptr;
876 if (ValDivergent && ScanImpl == ScanOptions::Iterative) {
877 // Move terminator from I's block to ComputeEnd block.
879 B.SetInsertPoint(ComputeEnd);
880 Terminator->removeFromParent();
881 B.Insert(Terminator);
882
883 // Branch to ComputeLoop Block unconditionally from the I's block for
884 // iterative approach.
885 B.SetInsertPoint(EntryBB);
886 B.CreateBr(ComputeLoop);
887
888 // Update the dominator tree for new control flow.
889 DTU.applyUpdates(
890 {{DominatorTree::Insert, EntryBB, ComputeLoop},
891 {DominatorTree::Insert, ComputeLoop, ComputeEnd},
892 {DominatorTree::Delete, EntryBB, SingleLaneTerminator->getParent()}});
893
894 Predecessor = ComputeEnd;
895 } else {
896 Predecessor = EntryBB;
897 }
898 // Move the IR builder into single_lane next.
899 B.SetInsertPoint(SingleLaneTerminator);
900
901 // Clone the original atomic operation into single lane, replacing the
902 // original value with our newly created one.
903 Instruction *const NewI = I.clone();
904 B.Insert(NewI);
905 NewI->setOperand(ValIdx, NewV);
906
907 // Move the IR builder into exit next, and start inserting just before the
908 // original instruction.
909 B.SetInsertPoint(&I);
910
911 if (NeedResult) {
912 // Create a PHI node to get our new atomic result into the exit block.
913 PHINode *const PHI = B.CreatePHI(Ty, 2);
914 PHI->addIncoming(PoisonValue::get(Ty), Predecessor);
915 PHI->addIncoming(NewI, SingleLaneTerminator->getParent());
916
917 // We need to broadcast the value who was the lowest active lane (the first
918 // lane) to all other lanes in the wavefront. We use an intrinsic for this,
919 // but have to handle 64-bit broadcasts with two calls to this intrinsic.
920 Value *BroadcastI = nullptr;
921
922 if (TyBitWidth == 64) {
923 Value *const ExtractLo = B.CreateTrunc(PHI, Int32Ty);
924 Value *const ExtractHi = B.CreateTrunc(B.CreateLShr(PHI, 32), Int32Ty);
925 CallInst *const ReadFirstLaneLo =
926 B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractLo);
927 CallInst *const ReadFirstLaneHi =
928 B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractHi);
929 Value *const PartialInsert = B.CreateInsertElement(
930 PoisonValue::get(VecTy), ReadFirstLaneLo, B.getInt32(0));
931 Value *const Insert =
932 B.CreateInsertElement(PartialInsert, ReadFirstLaneHi, B.getInt32(1));
933 BroadcastI = B.CreateBitCast(Insert, Ty);
934 } else if (TyBitWidth == 32) {
935 Value *CastedPhi = B.CreateBitCast(PHI, IntNTy);
936 BroadcastI =
937 B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, CastedPhi);
938 BroadcastI = B.CreateBitCast(BroadcastI, Ty);
939
940 } else {
941 llvm_unreachable("Unhandled atomic bit width");
942 }
943
944 // Now that we have the result of our single atomic operation, we need to
945 // get our individual lane's slice into the result. We use the lane offset
946 // we previously calculated combined with the atomic result value we got
947 // from the first lane, to get our lane's index into the atomic result.
948 Value *LaneOffset = nullptr;
949 if (ValDivergent) {
950 if (ScanImpl == ScanOptions::DPP) {
951 LaneOffset =
952 B.CreateIntrinsic(Intrinsic::amdgcn_strict_wwm, Ty, ExclScan);
953 } else if (ScanImpl == ScanOptions::Iterative) {
954 LaneOffset = ExclScan;
955 } else {
956 llvm_unreachable("Atomic Optimzer is disabled for None strategy");
957 }
958 } else {
959 Mbcnt = isAtomicFloatingPointTy ? B.CreateUIToFP(Mbcnt, Ty)
960 : B.CreateIntCast(Mbcnt, Ty, false);
961 switch (Op) {
962 default:
963 llvm_unreachable("Unhandled atomic op");
966 LaneOffset = buildMul(B, V, Mbcnt);
967 break;
976 LaneOffset = B.CreateSelect(Cond, Identity, V);
977 break;
979 LaneOffset = buildMul(B, V, B.CreateAnd(Mbcnt, 1));
980 break;
982 case AtomicRMWInst::FSub: {
983 LaneOffset = B.CreateFMul(V, Mbcnt);
984 break;
985 }
986 }
987 }
988 Value *const Result = buildNonAtomicBinOp(B, Op, BroadcastI, LaneOffset);
989
990 if (IsPixelShader) {
991 // Need a final PHI to reconverge to above the helper lane branch mask.
992 B.SetInsertPoint(PixelExitBB, PixelExitBB->getFirstNonPHIIt());
993
994 PHINode *const PHI = B.CreatePHI(Ty, 2);
995 PHI->addIncoming(PoisonValue::get(Ty), PixelEntryBB);
996 PHI->addIncoming(Result, I.getParent());
997 I.replaceAllUsesWith(PHI);
998 } else {
999 // Replace the original atomic instruction with the new one.
1000 I.replaceAllUsesWith(Result);
1001 }
1002 }
1003
1004 // And delete the original.
1005 I.eraseFromParent();
1006}
1007
1008INITIALIZE_PASS_BEGIN(AMDGPUAtomicOptimizer, DEBUG_TYPE,
1009 "AMDGPU atomic optimizations", false, false)
1012INITIALIZE_PASS_END(AMDGPUAtomicOptimizer, DEBUG_TYPE,
1013 "AMDGPU atomic optimizations", false, false)
1014
1016 return new AMDGPUAtomicOptimizer(ScanStrategy);
1017}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static Constant * getIdentityValueForAtomicOp(Type *const Ty, AtomicRMWInst::BinOp Op)
static Value * buildMul(IRBuilder<> &B, Value *LHS, Value *RHS)
static Value * buildNonAtomicBinOp(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *LHS, Value *RHS)
Rewrite undef for PHI
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
AMD GCN specific subclass of TargetSubtarget.
#define DEBUG_TYPE
Generic memory optimizations
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
IntegerType * Int32Ty
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Target-Independent Code Generator Pass Configuration Options pass.
LLVM IR instance of the generic uniformity analysis.
Value * RHS
Value * LHS
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:966
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition: APFloat.h:957
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:184
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition: APInt.h:194
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:197
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:348
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:500
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:727
static bool isFPOperation(BinOp Op)
Definition: Instructions.h:825
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition: Instructions.h:739
@ Add
*p = old + v
Definition: Instructions.h:743
@ FAdd
*p = old + v
Definition: Instructions.h:764
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:757
@ Or
*p = old | v
Definition: Instructions.h:751
@ Sub
*p = old - v
Definition: Instructions.h:745
@ And
*p = old & v
Definition: Instructions.h:747
@ Xor
*p = old ^ v
Definition: Instructions.h:753
@ FSub
*p = old - v
Definition: Instructions.h:767
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:755
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:761
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:775
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:759
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:771
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
Definition: BasicBlock.cpp:412
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:207
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:229
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:780
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:809
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:803
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:807
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:805
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:927
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:203
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:275
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:313
DominatorTree & getDomTree()
Definition: Dominators.h:321
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:692
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
virtual bool runOnFunction(Function &F)=0
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2649
Base class for instruction visitors.
Definition: InstVisitor.h:78
RetTy visitIntrinsicInst(IntrinsicInst &I)
Definition: InstVisitor.h:219
RetTy visitAtomicRMWInst(AtomicRMWInst &I)
Definition: InstVisitor.h:172
const BasicBlock * getParent() const
Definition: Instruction.h:150
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: Pass.cpp:98
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1743
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
void preserve()
Mark an analysis as preserved.
Definition: Analysis.h:129
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
const STC & getSubtarget(const Function &F) const
This method returns a pointer to the specified type of TargetSubtargetInfo.
Target-Independent Code Generator Pass Configuration Options.
TMC & getTM() const
Get the right type of TargetMachine for this target.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
const fltSemantics & getFltSemantics() const
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition: Type.h:185
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Analysis pass which computes UniformityInfo.
Legacy analysis pass which computes a CycleInfo.
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
LLVM Value Representation.
Definition: Value.h:74
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ LOCAL_ADDRESS
Address space for local memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
Definition: CallingConv.h:194
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1447
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
ScanOptions
Definition: AMDGPU.h:98
FunctionPass * createAMDGPUAtomicOptimizerPass(ScanOptions ScanStrategy)
DWARFExpression::Operation Op
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
char & AMDGPUAtomicOptimizerID
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)