| File: | build/source/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp |
| Warning: | line 85, column 12 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- GCNVOPDUtils.cpp - GCN VOPD Utils ------------------------===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | /// \file This file contains the AMDGPU DAG scheduling | |||
| 10 | /// mutation to pair VOPD instructions back to back. It also contains | |||
| 11 | // subroutines useful in the creation of VOPD instructions | |||
| 12 | // | |||
| 13 | //===----------------------------------------------------------------------===// | |||
| 14 | ||||
| 15 | #include "GCNVOPDUtils.h" | |||
| 16 | #include "AMDGPUSubtarget.h" | |||
| 17 | #include "GCNSubtarget.h" | |||
| 18 | #include "MCTargetDesc/AMDGPUMCTargetDesc.h" | |||
| 19 | #include "SIInstrInfo.h" | |||
| 20 | #include "Utils/AMDGPUBaseInfo.h" | |||
| 21 | #include "llvm/ADT/STLExtras.h" | |||
| 22 | #include "llvm/ADT/SmallVector.h" | |||
| 23 | #include "llvm/CodeGen/MachineBasicBlock.h" | |||
| 24 | #include "llvm/CodeGen/MachineInstr.h" | |||
| 25 | #include "llvm/CodeGen/MachineOperand.h" | |||
| 26 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
| 27 | #include "llvm/CodeGen/MacroFusion.h" | |||
| 28 | #include "llvm/CodeGen/ScheduleDAG.h" | |||
| 29 | #include "llvm/CodeGen/ScheduleDAGMutation.h" | |||
| 30 | #include "llvm/CodeGen/TargetInstrInfo.h" | |||
| 31 | #include "llvm/MC/MCInst.h" | |||
| 32 | ||||
| 33 | using namespace llvm; | |||
| 34 | ||||
| 35 | #define DEBUG_TYPE"gcn-vopd-utils" "gcn-vopd-utils" | |||
| 36 | ||||
| 37 | bool llvm::checkVOPDRegConstraints(const SIInstrInfo &TII, | |||
| 38 | const MachineInstr &FirstMI, | |||
| 39 | const MachineInstr &SecondMI) { | |||
| 40 | namespace VOPD = AMDGPU::VOPD; | |||
| 41 | ||||
| 42 | const MachineFunction *MF = FirstMI.getMF(); | |||
| 43 | const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>(); | |||
| 44 | const SIRegisterInfo *TRI = dyn_cast<SIRegisterInfo>(ST.getRegisterInfo()); | |||
| ||||
| 45 | const MachineRegisterInfo &MRI = MF->getRegInfo(); | |||
| 46 | // Literals also count against scalar bus limit | |||
| 47 | SmallVector<const MachineOperand *> UniqueLiterals; | |||
| 48 | auto addLiteral = [&](const MachineOperand &Op) { | |||
| 49 | for (auto &Literal : UniqueLiterals) { | |||
| 50 | if (Literal->isIdenticalTo(Op)) | |||
| 51 | return; | |||
| 52 | } | |||
| 53 | UniqueLiterals.push_back(&Op); | |||
| 54 | }; | |||
| 55 | SmallVector<Register> UniqueScalarRegs; | |||
| 56 | assert([&]() -> bool {(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
| 57 | for (auto MII = MachineBasicBlock::const_iterator(&FirstMI);(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
| 58 | MII != FirstMI.getParent()->instr_end(); ++MII) {(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
| 59 | if (&*MII == &SecondMI)(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
| 60 | return true;(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
| 61 | }(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
| 62 | return false;(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)) | |||
| 63 | }() && "Expected FirstMI to precede SecondMI")(static_cast <bool> ([&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI .getParent()->instr_end(); ++MII) { if (&*MII == & SecondMI) return true; } return false; }() && "Expected FirstMI to precede SecondMI" ) ? void (0) : __assert_fail ("[&]() -> bool { for (auto MII = MachineBasicBlock::const_iterator(&FirstMI); MII != FirstMI.getParent()->instr_end(); ++MII) { if (&*MII == &SecondMI) return true; } return false; }() && \"Expected FirstMI to precede SecondMI\"" , "llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp", 63, __extension__ __PRETTY_FUNCTION__)); | |||
| 64 | // Cannot pair dependent instructions | |||
| 65 | for (const auto &Use : SecondMI.uses()) | |||
| 66 | if (Use.isReg() && FirstMI.modifiesRegister(Use.getReg())) | |||
| 67 | return false; | |||
| 68 | ||||
| 69 | auto getVRegIdx = [&](unsigned OpcodeIdx, unsigned OperandIdx) { | |||
| 70 | const MachineInstr &MI = (OpcodeIdx == VOPD::X) ? FirstMI : SecondMI; | |||
| 71 | const MachineOperand &Operand = MI.getOperand(OperandIdx); | |||
| 72 | if (Operand.isReg() && TRI->isVectorRegister(MRI, Operand.getReg())) | |||
| 73 | return Operand.getReg(); | |||
| 74 | return Register(); | |||
| 75 | }; | |||
| 76 | ||||
| 77 | auto InstInfo = | |||
| 78 | AMDGPU::getVOPDInstInfo(FirstMI.getDesc(), SecondMI.getDesc()); | |||
| 79 | ||||
| 80 | for (auto CompIdx : VOPD::COMPONENTS) { | |||
| 81 | const MachineInstr &MI = (CompIdx
| |||
| 82 | ||||
| 83 | const MachineOperand &Src0 = MI.getOperand(VOPD::Component::SRC0); | |||
| 84 | if (Src0.isReg()) { | |||
| 85 | if (!TRI->isVectorRegister(MRI, Src0.getReg())) { | |||
| ||||
| 86 | if (!is_contained(UniqueScalarRegs, Src0.getReg())) | |||
| 87 | UniqueScalarRegs.push_back(Src0.getReg()); | |||
| 88 | } | |||
| 89 | } else { | |||
| 90 | if (!TII.isInlineConstant(MI, VOPD::Component::SRC0)) | |||
| 91 | addLiteral(Src0); | |||
| 92 | } | |||
| 93 | ||||
| 94 | if (InstInfo[CompIdx].hasMandatoryLiteral()) { | |||
| 95 | auto CompOprIdx = InstInfo[CompIdx].getMandatoryLiteralCompOperandIndex(); | |||
| 96 | addLiteral(MI.getOperand(CompOprIdx)); | |||
| 97 | } | |||
| 98 | if (MI.getDesc().hasImplicitUseOfPhysReg(AMDGPU::VCC)) | |||
| 99 | UniqueScalarRegs.push_back(AMDGPU::VCC_LO); | |||
| 100 | } | |||
| 101 | ||||
| 102 | if (UniqueLiterals.size() > 1) | |||
| 103 | return false; | |||
| 104 | if ((UniqueLiterals.size() + UniqueScalarRegs.size()) > 2) | |||
| 105 | return false; | |||
| 106 | if (InstInfo.hasInvalidOperand(getVRegIdx)) | |||
| 107 | return false; | |||
| 108 | ||||
| 109 | LLVM_DEBUG(dbgs() << "VOPD Reg Constraints Passed\n\tX: " << FirstMIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("gcn-vopd-utils")) { dbgs() << "VOPD Reg Constraints Passed\n\tX: " << FirstMI << "\n\tY: " << SecondMI << "\n"; } } while (false) | |||
| 110 | << "\n\tY: " << SecondMI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("gcn-vopd-utils")) { dbgs() << "VOPD Reg Constraints Passed\n\tX: " << FirstMI << "\n\tY: " << SecondMI << "\n"; } } while (false); | |||
| 111 | return true; | |||
| 112 | } | |||
| 113 | ||||
| 114 | /// Check if the instr pair, FirstMI and SecondMI, should be scheduled | |||
| 115 | /// together. Given SecondMI, when FirstMI is unspecified, then check if | |||
| 116 | /// SecondMI may be part of a fused pair at all. | |||
| 117 | static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII, | |||
| 118 | const TargetSubtargetInfo &TSI, | |||
| 119 | const MachineInstr *FirstMI, | |||
| 120 | const MachineInstr &SecondMI) { | |||
| 121 | const SIInstrInfo &STII = static_cast<const SIInstrInfo &>(TII); | |||
| 122 | unsigned Opc2 = SecondMI.getOpcode(); | |||
| 123 | auto SecondCanBeVOPD = AMDGPU::getCanBeVOPD(Opc2); | |||
| 124 | ||||
| 125 | // One instruction case | |||
| 126 | if (!FirstMI) | |||
| 127 | return SecondCanBeVOPD.Y; | |||
| 128 | ||||
| 129 | unsigned Opc = FirstMI->getOpcode(); | |||
| 130 | auto FirstCanBeVOPD = AMDGPU::getCanBeVOPD(Opc); | |||
| 131 | ||||
| 132 | if (!((FirstCanBeVOPD.X && SecondCanBeVOPD.Y) || | |||
| 133 | (FirstCanBeVOPD.Y && SecondCanBeVOPD.X))) | |||
| 134 | return false; | |||
| 135 | ||||
| 136 | return checkVOPDRegConstraints(STII, *FirstMI, SecondMI); | |||
| 137 | } | |||
| 138 | ||||
| 139 | namespace { | |||
| 140 | /// Adapts design from MacroFusion | |||
| 141 | /// Puts valid candidate instructions back-to-back so they can easily | |||
| 142 | /// be turned into VOPD instructions | |||
| 143 | /// Greedily pairs instruction candidates. O(n^2) algorithm. | |||
| 144 | struct VOPDPairingMutation : ScheduleDAGMutation { | |||
| 145 | ShouldSchedulePredTy shouldScheduleAdjacent; // NOLINT: function pointer | |||
| 146 | ||||
| 147 | VOPDPairingMutation( | |||
| 148 | ShouldSchedulePredTy shouldScheduleAdjacent) // NOLINT: function pointer | |||
| 149 | : shouldScheduleAdjacent(shouldScheduleAdjacent) {} | |||
| 150 | ||||
| 151 | void apply(ScheduleDAGInstrs *DAG) override { | |||
| 152 | const TargetInstrInfo &TII = *DAG->TII; | |||
| 153 | const GCNSubtarget &ST = DAG->MF.getSubtarget<GCNSubtarget>(); | |||
| 154 | if (!AMDGPU::hasVOPD(ST) || !ST.isWave32()) { | |||
| 155 | LLVM_DEBUG(dbgs() << "Target does not support VOPDPairingMutation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("gcn-vopd-utils")) { dbgs() << "Target does not support VOPDPairingMutation\n" ; } } while (false); | |||
| 156 | return; | |||
| 157 | } | |||
| 158 | ||||
| 159 | std::vector<SUnit>::iterator ISUI, JSUI; | |||
| 160 | for (ISUI = DAG->SUnits.begin(); ISUI != DAG->SUnits.end(); ++ISUI) { | |||
| 161 | const MachineInstr *IMI = ISUI->getInstr(); | |||
| 162 | if (!shouldScheduleAdjacent(TII, ST, nullptr, *IMI)) | |||
| 163 | continue; | |||
| 164 | if (!hasLessThanNumFused(*ISUI, 2)) | |||
| 165 | continue; | |||
| 166 | ||||
| 167 | for (JSUI = ISUI + 1; JSUI != DAG->SUnits.end(); ++JSUI) { | |||
| 168 | if (JSUI->isBoundaryNode()) | |||
| 169 | continue; | |||
| 170 | const MachineInstr *JMI = JSUI->getInstr(); | |||
| 171 | if (!hasLessThanNumFused(*JSUI, 2) || | |||
| 172 | !shouldScheduleAdjacent(TII, ST, IMI, *JMI)) | |||
| 173 | continue; | |||
| 174 | if (fuseInstructionPair(*DAG, *ISUI, *JSUI)) | |||
| 175 | break; | |||
| 176 | } | |||
| 177 | } | |||
| 178 | LLVM_DEBUG(dbgs() << "Completed VOPDPairingMutation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("gcn-vopd-utils")) { dbgs() << "Completed VOPDPairingMutation\n" ; } } while (false); | |||
| 179 | } | |||
| 180 | }; | |||
| 181 | } // namespace | |||
| 182 | ||||
| 183 | std::unique_ptr<ScheduleDAGMutation> llvm::createVOPDPairingMutation() { | |||
| 184 | return std::make_unique<VOPDPairingMutation>(shouldScheduleVOPDAdjacent); | |||
| 185 | } |