Line data Source code
1 : //===-- AArch64A53Fix835769.cpp -------------------------------------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : // This pass changes code to work around Cortex-A53 erratum 835769.
10 : // It works around it by inserting a nop instruction in code sequences that
11 : // in some circumstances may trigger the erratum.
12 : // It inserts a nop instruction between a sequence of the following 2 classes
13 : // of instructions:
14 : // instr 1: mem-instr (including loads, stores and prefetches).
15 : // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
16 : //===----------------------------------------------------------------------===//
17 :
18 : #include "AArch64.h"
19 : #include "llvm/ADT/Statistic.h"
20 : #include "llvm/CodeGen/MachineFunction.h"
21 : #include "llvm/CodeGen/MachineFunctionPass.h"
22 : #include "llvm/CodeGen/MachineInstr.h"
23 : #include "llvm/CodeGen/MachineInstrBuilder.h"
24 : #include "llvm/CodeGen/MachineRegisterInfo.h"
25 : #include "llvm/CodeGen/TargetInstrInfo.h"
26 : #include "llvm/Support/Debug.h"
27 : #include "llvm/Support/raw_ostream.h"
28 :
29 : using namespace llvm;
30 :
31 : #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
32 :
33 : STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
34 :
35 : //===----------------------------------------------------------------------===//
36 : // Helper functions
37 :
38 : // Is the instruction a match for the instruction that comes first in the
39 : // sequence of instructions that can trigger the erratum?
40 6 : static bool isFirstInstructionInSequence(MachineInstr *MI) {
41 : // Must return true if this instruction is a load, a store or a prefetch.
42 6 : switch (MI->getOpcode()) {
43 : case AArch64::PRFMl:
44 : case AArch64::PRFMroW:
45 : case AArch64::PRFMroX:
46 : case AArch64::PRFMui:
47 : case AArch64::PRFUMi:
48 : return true;
49 6 : default:
50 6 : return MI->mayLoadOrStore();
51 : }
52 : }
53 :
54 : // Is the instruction a match for the instruction that comes second in the
55 : // sequence that can trigger the erratum?
56 3 : static bool isSecondInstructionInSequence(MachineInstr *MI) {
57 : // Must return true for non-SIMD integer multiply-accumulates, writing
58 : // to a 64-bit register.
59 6 : switch (MI->getOpcode()) {
60 : // Erratum cannot be triggered when the destination register is 32 bits,
61 : // therefore only include the following.
62 3 : case AArch64::MSUBXrrr:
63 : case AArch64::MADDXrrr:
64 : case AArch64::SMADDLrrr:
65 : case AArch64::SMSUBLrrr:
66 : case AArch64::UMADDLrrr:
67 : case AArch64::UMSUBLrrr:
68 : // Erratum can only be triggered by multiply-adds, not by regular
69 : // non-accumulating multiplies, i.e. when Ra=XZR='11111'
70 3 : return MI->getOperand(3).getReg() != AArch64::XZR;
71 : default:
72 : return false;
73 : }
74 : }
75 :
76 :
77 : //===----------------------------------------------------------------------===//
78 :
79 : namespace {
80 : class AArch64A53Fix835769 : public MachineFunctionPass {
81 : const TargetInstrInfo *TII;
82 :
83 : public:
84 : static char ID;
85 3 : explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {
86 3 : initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry());
87 3 : }
88 :
89 : bool runOnMachineFunction(MachineFunction &F) override;
90 :
91 3 : MachineFunctionProperties getRequiredProperties() const override {
92 3 : return MachineFunctionProperties().set(
93 3 : MachineFunctionProperties::Property::NoVRegs);
94 : }
95 :
96 3 : StringRef getPassName() const override {
97 3 : return "Workaround A53 erratum 835769 pass";
98 : }
99 :
100 3 : void getAnalysisUsage(AnalysisUsage &AU) const override {
101 3 : AU.setPreservesCFG();
102 3 : MachineFunctionPass::getAnalysisUsage(AU);
103 3 : }
104 :
105 : private:
106 : bool runOnBasicBlock(MachineBasicBlock &MBB);
107 : };
108 : char AArch64A53Fix835769::ID = 0;
109 :
110 : } // end anonymous namespace
111 :
112 199035 : INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",
113 : "AArch64 fix for A53 erratum 835769", false, false)
114 :
115 : //===----------------------------------------------------------------------===//
116 :
117 : bool
118 3 : AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
119 : LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
120 : bool Changed = false;
121 3 : TII = F.getSubtarget().getInstrInfo();
122 :
123 6 : for (auto &MBB : F) {
124 3 : Changed |= runOnBasicBlock(MBB);
125 : }
126 3 : return Changed;
127 : }
128 :
129 : // Return the block that was fallen through to get to MBB, if any,
130 : // otherwise nullptr.
131 3 : static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
132 : const TargetInstrInfo *TII) {
133 : // Get the previous machine basic block in the function.
134 : MachineFunction::iterator MBBI(MBB);
135 :
136 : // Can't go off top of function.
137 6 : if (MBBI == MBB->getParent()->begin())
138 : return nullptr;
139 :
140 0 : MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
141 : SmallVector<MachineOperand, 2> Cond;
142 :
143 : MachineBasicBlock *PrevBB = &*std::prev(MBBI);
144 0 : for (MachineBasicBlock *S : MBB->predecessors())
145 0 : if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&
146 0 : !FBB)
147 : return S;
148 :
149 : return nullptr;
150 : }
151 :
152 : // Iterate through fallen through blocks trying to find a previous non-pseudo if
153 : // there is one, otherwise return nullptr. Only look for instructions in
154 : // previous blocks, not the current block, since we only use this to look at
155 : // previous blocks.
156 3 : static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
157 : const TargetInstrInfo *TII) {
158 : MachineBasicBlock *FMBB = &MBB;
159 :
160 : // If there is no non-pseudo in the current block, loop back around and try
161 : // the previous block (if there is one).
162 3 : while ((FMBB = getBBFallenThrough(FMBB, TII))) {
163 0 : for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend()))
164 0 : if (!I.isPseudo())
165 : return &I;
166 : }
167 :
168 : // There was no previous non-pseudo in the fallen through blocks
169 : return nullptr;
170 : }
171 :
172 3 : static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
173 : const TargetInstrInfo *TII) {
174 : // If we are the first instruction of the block, put the NOP at the end of
175 : // the previous fallthrough block
176 3 : if (MI == &MBB.front()) {
177 0 : MachineInstr *I = getLastNonPseudo(MBB, TII);
178 : assert(I && "Expected instruction");
179 : DebugLoc DL = I->getDebugLoc();
180 0 : BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
181 : }
182 : else {
183 : DebugLoc DL = MI->getDebugLoc();
184 3 : BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
185 : }
186 :
187 : ++NumNopsAdded;
188 3 : }
189 :
190 : bool
191 0 : AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
192 : bool Changed = false;
193 : LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB
194 : << " - scanning instructions...\n");
195 :
196 : // First, scan the basic block, looking for a sequence of 2 instructions
197 : // that match the conditions under which the erratum may trigger.
198 :
199 : // List of terminating instructions in matching sequences
200 : std::vector<MachineInstr*> Sequences;
201 : unsigned Idx = 0;
202 : MachineInstr *PrevInstr = nullptr;
203 :
204 : // Try and find the last non-pseudo instruction in any fallen through blocks,
205 : // if there isn't one, then we use nullptr to represent that.
206 0 : PrevInstr = getLastNonPseudo(MBB, TII);
207 :
208 0 : for (auto &MI : MBB) {
209 0 : MachineInstr *CurrInstr = &MI;
210 : LLVM_DEBUG(dbgs() << " Examining: " << MI);
211 0 : if (PrevInstr) {
212 : LLVM_DEBUG(dbgs() << " PrevInstr: " << *PrevInstr
213 : << " CurrInstr: " << *CurrInstr
214 : << " isFirstInstructionInSequence(PrevInstr): "
215 : << isFirstInstructionInSequence(PrevInstr) << "\n"
216 : << " isSecondInstructionInSequence(CurrInstr): "
217 : << isSecondInstructionInSequence(CurrInstr) << "\n");
218 0 : if (isFirstInstructionInSequence(PrevInstr) &&
219 0 : isSecondInstructionInSequence(CurrInstr)) {
220 : LLVM_DEBUG(dbgs() << " ** pattern found at Idx " << Idx << "!\n");
221 0 : Sequences.push_back(CurrInstr);
222 : }
223 : }
224 0 : if (!CurrInstr->isPseudo())
225 0 : PrevInstr = CurrInstr;
226 : ++Idx;
227 : }
228 :
229 : LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size()
230 : << " occurrences of pattern found.\n");
231 :
232 : // Then update the basic block, inserting nops between the detected sequences.
233 0 : for (auto &MI : Sequences) {
234 : Changed = true;
235 0 : insertNopBeforeInstruction(MBB, MI, TII);
236 : }
237 :
238 0 : return Changed;
239 : }
240 :
241 : // Factory function used by AArch64TargetMachine to add the pass to
242 : // the passmanager.
243 3 : FunctionPass *llvm::createAArch64A53Fix835769() {
244 3 : return new AArch64A53Fix835769();
245 : }
|
