LLVM 19.0.0git
GCNHazardRecognizer.h
Go to the documentation of this file.
1//===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines hazard recognizers for scheduling on GCN processors.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
14#define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
15
16#include "llvm/ADT/BitVector.h"
17#include "llvm/ADT/STLExtras.h"
20#include <list>
21
22namespace llvm {
23
24class MachineFunction;
25class MachineInstr;
26class MachineOperand;
27class MachineRegisterInfo;
28class SIInstrInfo;
29class SIRegisterInfo;
30class GCNSubtarget;
31
33public:
35
36private:
37 // Distinguish if we are called from scheduler or hazard recognizer
38 bool IsHazardRecognizerMode;
39
40 // This variable stores the instruction that has been emitted this cycle. It
41 // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
42 // called.
43 MachineInstr *CurrCycleInstr;
44 std::list<MachineInstr*> EmittedInstrs;
45 const MachineFunction &MF;
46 const GCNSubtarget &ST;
47 const SIInstrInfo &TII;
48 const SIRegisterInfo &TRI;
49 TargetSchedModel TSchedModel;
50 bool RunLdsBranchVmemWARHazardFixup;
51
52 /// RegUnits of uses in the current soft memory clause.
53 BitVector ClauseUses;
54
55 /// RegUnits of defs in the current soft memory clause.
56 BitVector ClauseDefs;
57
58 void resetClause() {
59 ClauseUses.reset();
60 ClauseDefs.reset();
61 }
62
63 void addClauseInst(const MachineInstr &MI);
64
65 /// \returns the number of wait states before another MFMA instruction can be
66 /// issued after \p MI.
67 unsigned getMFMAPipelineWaitStates(const MachineInstr &MI) const;
68
69 // Advance over a MachineInstr bundle. Look for hazards in the bundled
70 // instructions.
71 void processBundle();
72
73 // Run on an individual instruction in hazard recognizer mode. This can be
74 // used on a newly inserted instruction before returning from PreEmitNoops.
75 void runOnInstruction(MachineInstr *MI);
76
77 int getWaitStatesSince(IsHazardFn IsHazard, int Limit);
78 int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit);
79 int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit);
80
81 int checkSoftClauseHazards(MachineInstr *SMEM);
82 int checkSMRDHazards(MachineInstr *SMRD);
83 int checkVMEMHazards(MachineInstr* VMEM);
84 int checkDPPHazards(MachineInstr *DPP);
85 int checkDivFMasHazards(MachineInstr *DivFMas);
86 int checkGetRegHazards(MachineInstr *GetRegInstr);
87 int checkSetRegHazards(MachineInstr *SetRegInstr);
88 int createsVALUHazard(const MachineInstr &MI);
89 int checkVALUHazards(MachineInstr *VALU);
90 int checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI);
91 int checkRWLaneHazards(MachineInstr *RWLane);
92 int checkRFEHazards(MachineInstr *RFE);
93 int checkInlineAsmHazards(MachineInstr *IA);
94 int checkReadM0Hazards(MachineInstr *SMovRel);
95 int checkNSAtoVMEMHazard(MachineInstr *MI);
96 int checkFPAtomicToDenormModeHazard(MachineInstr *MI);
97 void fixHazards(MachineInstr *MI);
98 bool fixVcmpxPermlaneHazards(MachineInstr *MI);
99 bool fixVMEMtoScalarWriteHazards(MachineInstr *MI);
100 bool fixSMEMtoVectorWriteHazards(MachineInstr *MI);
101 bool fixVcmpxExecWARHazard(MachineInstr *MI);
102 bool fixLdsBranchVmemWARHazard(MachineInstr *MI);
103 bool fixLdsDirectVALUHazard(MachineInstr *MI);
104 bool fixLdsDirectVMEMHazard(MachineInstr *MI);
105 bool fixVALUPartialForwardingHazard(MachineInstr *MI);
106 bool fixVALUTransUseHazard(MachineInstr *MI);
107 bool fixWMMAHazards(MachineInstr *MI);
108 bool fixShift64HighRegBug(MachineInstr *MI);
109 bool fixVALUMaskWriteHazard(MachineInstr *MI);
110
111 int checkMAIHazards(MachineInstr *MI);
112 int checkMAIHazards908(MachineInstr *MI);
113 int checkMAIHazards90A(MachineInstr *MI);
114 /// Pad the latency between neighboring MFMA instructions with s_nops. The
115 /// percentage of wait states to fill with s_nops is specified by the command
116 /// line option '-amdgpu-mfma-padding-ratio'.
117 ///
118 /// For example, with '-amdgpu-mfma-padding-ratio=100':
119 ///
120 /// 2 pass MFMA instructions have a latency of 2 wait states. Therefore, a
121 /// 'S_NOP 1' will be added between sequential MFMA instructions.
122 ///
123 /// V_MFMA_F32_4X4X1F32
124 /// V_MFMA_F32_4X4X1F32
125 ///-->
126 /// V_MFMA_F32_4X4X1F32
127 /// S_NOP 1
128 /// V_MFMA_F32_4X4X1F32
129 int checkMFMAPadding(MachineInstr *MI);
130 int checkMAIVALUHazards(MachineInstr *MI);
131 int checkMAILdStHazards(MachineInstr *MI);
132
133public:
135 // We can only issue one instruction per cycle.
136 bool atIssueLimit() const override { return true; }
137 void EmitInstruction(SUnit *SU) override;
138 void EmitInstruction(MachineInstr *MI) override;
139 HazardType getHazardType(SUnit *SU, int Stalls) override;
140 void EmitNoop() override;
141 unsigned PreEmitNoops(MachineInstr *) override;
143 void AdvanceCycle() override;
144 void RecedeCycle() override;
145 bool ShouldPreferAnother(SUnit *SU) override;
146 void Reset() override;
147};
148
149} // end namespace llvm
150
151#endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
unsigned const MachineRegisterInfo * MRI
This file implements the BitVector class.
IRTranslator LLVM IR MI
unsigned Reg
This file contains some templates that are useful if you are working with the STL at all.
BitVector & reset()
Definition: BitVector.h:392
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
void Reset() override
Reset - This callback is invoked when a new block of instructions is about to be schedule.
unsigned PreEmitNoops(MachineInstr *) override
This overload will be used when the hazard recognizer is being used by a non-scheduling pass,...
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
function_ref< bool(const MachineInstr &)> IsHazardFn
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
unsigned PreEmitNoopsCommon(MachineInstr *)
bool ShouldPreferAnother(SUnit *SU) override
ShouldPreferAnother - This callback may be invoked if getHazardType returns NoHazard.
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
bool atIssueLimit() const override
atIssueLimit - Return true if no more instructions may be issued in this cycle.
Representation of each machine instruction.
Definition: MachineInstr.h:69
MachineOperand class - Representation of each machine instruction operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242
HazardRecognizer - This determines whether or not an instruction can be issued this cycle,...
Provide an instruction scheduling machine model to CodeGen passes.
An efficient, type-erasing, non-owning reference to a callable.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18