File: | build/source/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp |
Warning: | line 85, column 12 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- GCNVOPDUtils.cpp - GCN VOPD Utils ------------------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | /// \file This file contains the AMDGPU DAG scheduling | |||
10 | /// mutation to pair VOPD instructions back to back. It also contains | |||
11 | // subroutines useful in the creation of VOPD instructions | |||
12 | // | |||
13 | //===----------------------------------------------------------------------===// | |||
14 | ||||
15 | #include "GCNVOPDUtils.h" | |||
16 | #include "AMDGPUSubtarget.h" | |||
17 | #include "GCNSubtarget.h" | |||
18 | #include "MCTargetDesc/AMDGPUMCTargetDesc.h" | |||
19 | #include "SIInstrInfo.h" | |||
20 | #include "Utils/AMDGPUBaseInfo.h" | |||
21 | #include "llvm/ADT/STLExtras.h" | |||
22 | #include "llvm/ADT/SmallVector.h" | |||
23 | #include "llvm/CodeGen/MachineBasicBlock.h" | |||
24 | #include "llvm/CodeGen/MachineInstr.h" | |||
25 | #include "llvm/CodeGen/MachineOperand.h" | |||
26 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
27 | #include "llvm/CodeGen/MacroFusion.h" | |||
28 | #include "llvm/CodeGen/ScheduleDAG.h" | |||
29 | #include "llvm/CodeGen/ScheduleDAGMutation.h" | |||
30 | #include "llvm/CodeGen/TargetInstrInfo.h" | |||
31 | #include "llvm/MC/MCInst.h" | |||
32 | ||||
33 | using namespace llvm; | |||
34 | ||||
35 | #define DEBUG_TYPE"gcn-vopd-utils" "gcn-vopd-utils" | |||
36 | ||||
37 | bool llvm::checkVOPDRegConstraints(const SIInstrInfo &TII, | |||
38 | const MachineInstr &FirstMI, | |||
39 | const MachineInstr &SecondMI) { | |||
40 | namespace VOPD = AMDGPU::VOPD; | |||
41 | ||||
42 | const MachineFunction *MF = FirstMI.getMF(); | |||
43 | const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); | |||
44 | const SIRegisterInfo *TRI = dyn_cast<SIRegisterInfo>(ST.getRegisterInfo()); | |||
| ||||
45 | const MachineRegisterInfo &MRI = MF->getRegInfo(); | |||
46 | // Literals also count against scalar bus limit | |||
47 | SmallVector<const MachineOperand *> UniqueLiterals; | |||
48 | auto addLiteral = [&](const MachineOperand &Op) { | |||
49 | for (auto &Literal : UniqueLiterals) { | |||
50 | if (Literal->isIdenticalTo(Op)) | |||
51 | return; | |||
52 | } | |||
53 | UniqueLiterals.push_back(&Op); | |||
54 | }; | |||
55 | SmallVector<Register> UniqueScalarRegs; | |||
56 | assert([&]() -> bool {(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
57 | for (auto MII = MachineBasicBlock::const_iterator(&FirstMI);(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
58 | MII != FirstMI.getParent()->instr_end(); ++MII) {(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
59 | if (&*MII == &SecondMI)(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
60 | return true;(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
61 | }(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
62 | return false;(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
63 | }() && "Expected FirstMI to precede SecondMI")(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)); | |||
64 | // Cannot pair dependent instructions | |||
65 | for (const auto &Use : SecondMI.uses()) | |||
66 | if (Use.isReg() && FirstMI.modifiesRegister(Use.getReg())) | |||
67 | return false; | |||
68 | ||||
69 | auto getVRegIdx = [&](unsigned OpcodeIdx, unsigned OperandIdx) { | |||
70 | const MachineInstr &MI = (OpcodeIdx == VOPD::X) ? FirstMI : SecondMI; | |||
71 | const MachineOperand &Operand = MI.getOperand(OperandIdx); | |||
72 | if (Operand.isReg() && TRI->isVectorRegister(MRI, Operand.getReg())) | |||
73 | return Operand.getReg(); | |||
74 | return Register(); | |||
75 | }; | |||
76 | ||||
77 | auto InstInfo = | |||
78 | AMDGPU::getVOPDInstInfo(FirstMI.getDesc(), SecondMI.getDesc()); | |||
79 | ||||
80 | for (auto CompIdx : VOPD::COMPONENTS) { | |||
81 | const MachineInstr &MI = (CompIdx
| |||
82 | ||||
83 | const MachineOperand &Src0 = MI.getOperand(VOPD::Component::SRC0); | |||
84 | if (Src0.isReg()) { | |||
85 | if (!TRI->isVectorRegister(MRI, Src0.getReg())) { | |||
| ||||
86 | if (!is_contained(UniqueScalarRegs, Src0.getReg())) | |||
87 | UniqueScalarRegs.push_back(Src0.getReg()); | |||
88 | } | |||
89 | } else { | |||
90 | if (!TII.isInlineConstant(MI, VOPD::Component::SRC0)) | |||
91 | addLiteral(Src0); | |||
92 | } | |||
93 | ||||
94 | if (InstInfo[CompIdx].hasMandatoryLiteral()) { | |||
95 | auto CompOprIdx = InstInfo[CompIdx].getMandatoryLiteralCompOperandIndex(); | |||
96 | addLiteral(MI.getOperand(CompOprIdx)); | |||
97 | } | |||
98 | if (MI.getDesc().hasImplicitUseOfPhysReg(AMDGPU::VCC)) | |||
99 | UniqueScalarRegs.push_back(AMDGPU::VCC_LO); | |||
100 | } | |||
101 | ||||
102 | if (UniqueLiterals.size() > 1) | |||
103 | return false; | |||
104 | if ((UniqueLiterals.size() + UniqueScalarRegs.size()) > 2) | |||
105 | return false; | |||
106 | if (InstInfo.hasInvalidOperand(getVRegIdx)) | |||
107 | return false; | |||
108 | ||||
109 | LLVM_DEBUG(dbgs() << "VOPD Reg Constraints Passed\n\tX: " << FirstMIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("gcn-vopd-utils")) { dbgs() << "VOPD Reg Constraints Passed\n\tX: " << FirstMI << "\n\tY: " << SecondMI << "\n"; } } while (false) | |||
110 | << "\n\tY: " << SecondMI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("gcn-vopd-utils")) { dbgs() << "VOPD Reg Constraints Passed\n\tX: " << FirstMI << "\n\tY: " << SecondMI << "\n"; } } while (false); | |||
111 | return true; | |||
112 | } | |||
113 | ||||
114 | /// Check if the instr pair, FirstMI and SecondMI, should be scheduled | |||
115 | /// together. Given SecondMI, when FirstMI is unspecified, then check if | |||
116 | /// SecondMI may be part of a fused pair at all. | |||
117 | static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII, | |||
118 | const TargetSubtargetInfo &TSI, | |||
119 | const MachineInstr *FirstMI, | |||
120 | const MachineInstr &SecondMI) { | |||
121 | const SIInstrInfo &STII = static_cast<const SIInstrInfo &>(TII); | |||
122 | unsigned Opc2 = SecondMI.getOpcode(); | |||
123 | auto SecondCanBeVOPD = AMDGPU::getCanBeVOPD(Opc2); | |||
124 | ||||
125 | // One instruction case | |||
126 | if (!FirstMI) | |||
127 | return SecondCanBeVOPD.Y; | |||
128 | ||||
129 | unsigned Opc = FirstMI->getOpcode(); | |||
130 | auto FirstCanBeVOPD = AMDGPU::getCanBeVOPD(Opc); | |||
131 | ||||
132 | if (!((FirstCanBeVOPD.X && SecondCanBeVOPD.Y) || | |||
133 | (FirstCanBeVOPD.Y && SecondCanBeVOPD.X))) | |||
134 | return false; | |||
135 | ||||
136 | return checkVOPDRegConstraints(STII, *FirstMI, SecondMI); | |||
137 | } | |||
138 | ||||
139 | namespace { | |||
140 | /// Adapts design from MacroFusion | |||
141 | /// Puts valid candidate instructions back-to-back so they can easily | |||
142 | /// be turned into VOPD instructions | |||
143 | /// Greedily pairs instruction candidates. O(n^2) algorithm. | |||
144 | struct VOPDPairingMutation : ScheduleDAGMutation { | |||
145 | ShouldSchedulePredTy shouldScheduleAdjacent; // NOLINT: function pointer | |||
146 | ||||
147 | VOPDPairingMutation( | |||
148 | ShouldSchedulePredTy shouldScheduleAdjacent) // NOLINT: function pointer | |||
149 | : shouldScheduleAdjacent(shouldScheduleAdjacent) {} | |||
150 | ||||
151 | void apply(ScheduleDAGInstrs *DAG) override { | |||
152 | const TargetInstrInfo &TII = *DAG->TII; | |||
153 | const GCNSubtarget &ST = DAG->MF.getSubtarget<GCNSubtarget>(); | |||
154 | if (!AMDGPU::hasVOPD(ST) || !ST.isWave32()) { | |||
155 | LLVM_DEBUG(dbgs() << "Target does not support VOPDPairingMutation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("gcn-vopd-utils")) { dbgs() << "Target does not support VOPDPairingMutation\n" ; } } while (false); | |||
156 | return; | |||
157 | } | |||
158 | ||||
159 | std::vector<SUnit>::iterator ISUI, JSUI; | |||
160 | for (ISUI = DAG->SUnits.begin(); ISUI != DAG->SUnits.end(); ++ISUI) { | |||
161 | const MachineInstr *IMI = ISUI->getInstr(); | |||
162 | if (!shouldScheduleAdjacent(TII, ST, nullptr, *IMI)) | |||
163 | continue; | |||
164 | if (!hasLessThanNumFused(*ISUI, 2)) | |||
165 | continue; | |||
166 | ||||
167 | for (JSUI = ISUI + 1; JSUI != DAG->SUnits.end(); ++JSUI) { | |||
168 | if (JSUI->isBoundaryNode()) | |||
169 | continue; | |||
170 | const MachineInstr *JMI = JSUI->getInstr(); | |||
171 | if (!hasLessThanNumFused(*JSUI, 2) || | |||
172 | !shouldScheduleAdjacent(TII, ST, IMI, *JMI)) | |||
173 | continue; | |||
174 | if (fuseInstructionPair(*DAG, *ISUI, *JSUI)) | |||
175 | break; | |||
176 | } | |||
177 | } | |||
178 | LLVM_DEBUG(dbgs() << "Completed VOPDPairingMutation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("gcn-vopd-utils")) { dbgs() << "Completed VOPDPairingMutation\n" ; } } while (false); | |||
179 | } | |||
180 | }; | |||
181 | } // namespace | |||
182 | ||||
183 | std::unique_ptr<ScheduleDAGMutation> llvm::createVOPDPairingMutation() { | |||
184 | return std::make_unique<VOPDPairingMutation>(shouldScheduleVOPDAdjacent); | |||
185 | } |