LLVM  6.0.0svn
AArch64FalkorHWPFFix.cpp
Go to the documentation of this file.
1 //===- AArch64FalkorHWPFFix.cpp - Avoid HW prefetcher pitfalls on Falkor --===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 /// \file For Falkor, we want to avoid HW prefetcher instruction tag collisions
10 /// that may inhibit the HW prefetching. This is done in two steps. Before
11 /// ISel, we mark strided loads (i.e. those that will likely benefit from
12 /// prefetching) with metadata. Then, after opcodes have been finalized, we
13 /// insert MOVs and re-write loads to prevent unintnentional tag collisions.
14 // ===---------------------------------------------------------------------===//
15 
16 #include "AArch64.h"
17 #include "AArch64InstrInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetMachine.h"
20 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/None.h"
23 #include "llvm/ADT/Optional.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/Statistic.h"
26 #include "llvm/Analysis/LoopInfo.h"
40 #include "llvm/IR/DebugLoc.h"
41 #include "llvm/IR/Dominators.h"
42 #include "llvm/IR/Function.h"
43 #include "llvm/IR/Instruction.h"
44 #include "llvm/IR/Instructions.h"
45 #include "llvm/IR/Metadata.h"
46 #include "llvm/Pass.h"
47 #include "llvm/Support/Casting.h"
48 #include "llvm/Support/Debug.h"
50 #include <cassert>
51 #include <iterator>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "falkor-hwpf-fix"
57 
58 STATISTIC(NumStridedLoadsMarked, "Number of strided loads marked");
59 STATISTIC(NumCollisionsAvoided,
60  "Number of HW prefetch tag collisions avoided");
61 STATISTIC(NumCollisionsNotAvoided,
62  "Number of HW prefetch tag collisions not avoided due to lack of regsiters");
63 
64 namespace {
65 
66 class FalkorMarkStridedAccesses {
67 public:
68  FalkorMarkStridedAccesses(LoopInfo &LI, ScalarEvolution &SE)
69  : LI(LI), SE(SE) {}
70 
71  bool run();
72 
73 private:
74  bool runOnLoop(Loop &L);
75 
76  LoopInfo &LI;
77  ScalarEvolution &SE;
78 };
79 
80 class FalkorMarkStridedAccessesLegacy : public FunctionPass {
81 public:
82  static char ID; // Pass ID, replacement for typeid
83 
84  FalkorMarkStridedAccessesLegacy() : FunctionPass(ID) {
87  }
88 
89  void getAnalysisUsage(AnalysisUsage &AU) const override {
96  }
97 
98  bool runOnFunction(Function &F) override;
99 };
100 
101 } // end anonymous namespace
102 
104 
105 INITIALIZE_PASS_BEGIN(FalkorMarkStridedAccessesLegacy, DEBUG_TYPE,
106  "Falkor HW Prefetch Fix", false, false)
110 INITIALIZE_PASS_END(FalkorMarkStridedAccessesLegacy, DEBUG_TYPE,
111  "Falkor HW Prefetch Fix", false, false)
112 
114  return new FalkorMarkStridedAccessesLegacy();
115 }
116 
118  TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
119  const AArch64Subtarget *ST =
120  TPC.getTM<AArch64TargetMachine>().getSubtargetImpl(F);
121  if (ST->getProcFamily() != AArch64Subtarget::Falkor)
122  return false;
123 
124  if (skipFunction(F))
125  return false;
126 
127  LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
128  ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
129 
130  FalkorMarkStridedAccesses LDP(LI, SE);
131  return LDP.run();
132 }
133 
134 bool FalkorMarkStridedAccesses::run() {
135  bool MadeChange = false;
136 
137  for (Loop *L : LI)
138  for (auto LIt = df_begin(L), LE = df_end(L); LIt != LE; ++LIt)
139  MadeChange |= runOnLoop(**LIt);
140 
141  return MadeChange;
142 }
143 
144 bool FalkorMarkStridedAccesses::runOnLoop(Loop &L) {
145  // Only mark strided loads in the inner-most loop
146  if (!L.empty())
147  return false;
148 
149  bool MadeChange = false;
150 
151  for (BasicBlock *BB : L.blocks()) {
152  for (Instruction &I : *BB) {
153  LoadInst *LoadI = dyn_cast<LoadInst>(&I);
154  if (!LoadI)
155  continue;
156 
157  Value *PtrValue = LoadI->getPointerOperand();
158  if (L.isLoopInvariant(PtrValue))
159  continue;
160 
161  const SCEV *LSCEV = SE.getSCEV(PtrValue);
162  const SCEVAddRecExpr *LSCEVAddRec = dyn_cast<SCEVAddRecExpr>(LSCEV);
163  if (!LSCEVAddRec || !LSCEVAddRec->isAffine())
164  continue;
165 
167  MDNode::get(LoadI->getContext(), {}));
168  ++NumStridedLoadsMarked;
169  DEBUG(dbgs() << "Load: " << I << " marked as strided\n");
170  MadeChange = true;
171  }
172  }
173 
174  return MadeChange;
175 }
176 
177 namespace {
178 
179 class FalkorHWPFFix : public MachineFunctionPass {
180 public:
181  static char ID;
182 
183  FalkorHWPFFix() : MachineFunctionPass(ID) {
185  }
186 
187  bool runOnMachineFunction(MachineFunction &Fn) override;
188 
189  void getAnalysisUsage(AnalysisUsage &AU) const override {
192  }
193 
194  MachineFunctionProperties getRequiredProperties() const override {
197  }
198 
199 private:
200  void runOnLoop(MachineLoop &L, MachineFunction &Fn);
201 
202  const AArch64InstrInfo *TII;
203  const TargetRegisterInfo *TRI;
205  bool Modified;
206 };
207 
208 /// Bits from load opcodes used to compute HW prefetcher instruction tags.
209 struct LoadInfo {
210  LoadInfo() = default;
211 
212  unsigned DestReg = 0;
213  unsigned BaseReg = 0;
214  int BaseRegIdx = -1;
215  const MachineOperand *OffsetOpnd = nullptr;
216  bool IsPrePost = false;
217 };
218 
219 } // end anonymous namespace
220 
221 char FalkorHWPFFix::ID = 0;
222 
223 INITIALIZE_PASS_BEGIN(FalkorHWPFFix, "falkor-hwpf-fix-late",
224  "Falkor HW Prefetch Fix Late Phase", false, false)
226 INITIALIZE_PASS_END(FalkorHWPFFix, "falkor-hwpf-fix-late",
227  "Falkor HW Prefetch Fix Late Phase", false, false)
228 
229 static unsigned makeTag(unsigned Dest, unsigned Base, unsigned Offset) {
230  return (Dest & 0xf) | ((Base & 0xf) << 4) | ((Offset & 0x3f) << 8);
231 }
232 
234  int DestRegIdx;
235  int BaseRegIdx;
236  int OffsetIdx;
237  bool IsPrePost;
238 
239  switch (MI.getOpcode()) {
240  default:
241  return None;
242 
243  case AArch64::LD1i64:
244  case AArch64::LD2i64:
245  DestRegIdx = 0;
246  BaseRegIdx = 3;
247  OffsetIdx = -1;
248  IsPrePost = false;
249  break;
250 
251  case AArch64::LD1i8:
252  case AArch64::LD1i16:
253  case AArch64::LD1i32:
254  case AArch64::LD2i8:
255  case AArch64::LD2i16:
256  case AArch64::LD2i32:
257  case AArch64::LD3i8:
258  case AArch64::LD3i16:
259  case AArch64::LD3i32:
260  case AArch64::LD3i64:
261  case AArch64::LD4i8:
262  case AArch64::LD4i16:
263  case AArch64::LD4i32:
264  case AArch64::LD4i64:
265  DestRegIdx = -1;
266  BaseRegIdx = 3;
267  OffsetIdx = -1;
268  IsPrePost = false;
269  break;
270 
271  case AArch64::LD1Onev1d:
272  case AArch64::LD1Onev2s:
273  case AArch64::LD1Onev4h:
274  case AArch64::LD1Onev8b:
275  case AArch64::LD1Onev2d:
276  case AArch64::LD1Onev4s:
277  case AArch64::LD1Onev8h:
278  case AArch64::LD1Onev16b:
279  case AArch64::LD1Rv1d:
280  case AArch64::LD1Rv2s:
281  case AArch64::LD1Rv4h:
282  case AArch64::LD1Rv8b:
283  case AArch64::LD1Rv2d:
284  case AArch64::LD1Rv4s:
285  case AArch64::LD1Rv8h:
286  case AArch64::LD1Rv16b:
287  DestRegIdx = 0;
288  BaseRegIdx = 1;
289  OffsetIdx = -1;
290  IsPrePost = false;
291  break;
292 
293  case AArch64::LD1Twov1d:
294  case AArch64::LD1Twov2s:
295  case AArch64::LD1Twov4h:
296  case AArch64::LD1Twov8b:
297  case AArch64::LD1Twov2d:
298  case AArch64::LD1Twov4s:
299  case AArch64::LD1Twov8h:
300  case AArch64::LD1Twov16b:
301  case AArch64::LD1Threev1d:
302  case AArch64::LD1Threev2s:
303  case AArch64::LD1Threev4h:
304  case AArch64::LD1Threev8b:
305  case AArch64::LD1Threev2d:
306  case AArch64::LD1Threev4s:
307  case AArch64::LD1Threev8h:
308  case AArch64::LD1Threev16b:
309  case AArch64::LD1Fourv1d:
310  case AArch64::LD1Fourv2s:
311  case AArch64::LD1Fourv4h:
312  case AArch64::LD1Fourv8b:
313  case AArch64::LD1Fourv2d:
314  case AArch64::LD1Fourv4s:
315  case AArch64::LD1Fourv8h:
316  case AArch64::LD1Fourv16b:
317  case AArch64::LD2Twov2s:
318  case AArch64::LD2Twov4s:
319  case AArch64::LD2Twov8b:
320  case AArch64::LD2Twov2d:
321  case AArch64::LD2Twov4h:
322  case AArch64::LD2Twov8h:
323  case AArch64::LD2Twov16b:
324  case AArch64::LD2Rv1d:
325  case AArch64::LD2Rv2s:
326  case AArch64::LD2Rv4s:
327  case AArch64::LD2Rv8b:
328  case AArch64::LD2Rv2d:
329  case AArch64::LD2Rv4h:
330  case AArch64::LD2Rv8h:
331  case AArch64::LD2Rv16b:
332  case AArch64::LD3Threev2s:
333  case AArch64::LD3Threev4h:
334  case AArch64::LD3Threev8b:
335  case AArch64::LD3Threev2d:
336  case AArch64::LD3Threev4s:
337  case AArch64::LD3Threev8h:
338  case AArch64::LD3Threev16b:
339  case AArch64::LD3Rv1d:
340  case AArch64::LD3Rv2s:
341  case AArch64::LD3Rv4h:
342  case AArch64::LD3Rv8b:
343  case AArch64::LD3Rv2d:
344  case AArch64::LD3Rv4s:
345  case AArch64::LD3Rv8h:
346  case AArch64::LD3Rv16b:
347  case AArch64::LD4Fourv2s:
348  case AArch64::LD4Fourv4h:
349  case AArch64::LD4Fourv8b:
350  case AArch64::LD4Fourv2d:
351  case AArch64::LD4Fourv4s:
352  case AArch64::LD4Fourv8h:
353  case AArch64::LD4Fourv16b:
354  case AArch64::LD4Rv1d:
355  case AArch64::LD4Rv2s:
356  case AArch64::LD4Rv4h:
357  case AArch64::LD4Rv8b:
358  case AArch64::LD4Rv2d:
359  case AArch64::LD4Rv4s:
360  case AArch64::LD4Rv8h:
361  case AArch64::LD4Rv16b:
362  DestRegIdx = -1;
363  BaseRegIdx = 1;
364  OffsetIdx = -1;
365  IsPrePost = false;
366  break;
367 
368  case AArch64::LD1i64_POST:
369  case AArch64::LD2i64_POST:
370  DestRegIdx = 1;
371  BaseRegIdx = 4;
372  OffsetIdx = 5;
373  IsPrePost = true;
374  break;
375 
376  case AArch64::LD1i8_POST:
377  case AArch64::LD1i16_POST:
378  case AArch64::LD1i32_POST:
379  case AArch64::LD2i8_POST:
380  case AArch64::LD2i16_POST:
381  case AArch64::LD2i32_POST:
382  case AArch64::LD3i8_POST:
383  case AArch64::LD3i16_POST:
384  case AArch64::LD3i32_POST:
385  case AArch64::LD3i64_POST:
386  case AArch64::LD4i8_POST:
387  case AArch64::LD4i16_POST:
388  case AArch64::LD4i32_POST:
389  case AArch64::LD4i64_POST:
390  DestRegIdx = -1;
391  BaseRegIdx = 4;
392  OffsetIdx = 5;
393  IsPrePost = true;
394  break;
395 
396  case AArch64::LD1Onev1d_POST:
397  case AArch64::LD1Onev2s_POST:
398  case AArch64::LD1Onev4h_POST:
399  case AArch64::LD1Onev8b_POST:
400  case AArch64::LD1Onev2d_POST:
401  case AArch64::LD1Onev4s_POST:
402  case AArch64::LD1Onev8h_POST:
403  case AArch64::LD1Onev16b_POST:
404  case AArch64::LD1Rv1d_POST:
405  case AArch64::LD1Rv2s_POST:
406  case AArch64::LD1Rv4h_POST:
407  case AArch64::LD1Rv8b_POST:
408  case AArch64::LD1Rv2d_POST:
409  case AArch64::LD1Rv4s_POST:
410  case AArch64::LD1Rv8h_POST:
411  case AArch64::LD1Rv16b_POST:
412  DestRegIdx = 1;
413  BaseRegIdx = 2;
414  OffsetIdx = 3;
415  IsPrePost = true;
416  break;
417 
418  case AArch64::LD1Twov1d_POST:
419  case AArch64::LD1Twov2s_POST:
420  case AArch64::LD1Twov4h_POST:
421  case AArch64::LD1Twov8b_POST:
422  case AArch64::LD1Twov2d_POST:
423  case AArch64::LD1Twov4s_POST:
424  case AArch64::LD1Twov8h_POST:
425  case AArch64::LD1Twov16b_POST:
426  case AArch64::LD1Threev1d_POST:
427  case AArch64::LD1Threev2s_POST:
428  case AArch64::LD1Threev4h_POST:
429  case AArch64::LD1Threev8b_POST:
430  case AArch64::LD1Threev2d_POST:
431  case AArch64::LD1Threev4s_POST:
432  case AArch64::LD1Threev8h_POST:
433  case AArch64::LD1Threev16b_POST:
434  case AArch64::LD1Fourv1d_POST:
435  case AArch64::LD1Fourv2s_POST:
436  case AArch64::LD1Fourv4h_POST:
437  case AArch64::LD1Fourv8b_POST:
438  case AArch64::LD1Fourv2d_POST:
439  case AArch64::LD1Fourv4s_POST:
440  case AArch64::LD1Fourv8h_POST:
441  case AArch64::LD1Fourv16b_POST:
442  case AArch64::LD2Twov2s_POST:
443  case AArch64::LD2Twov4s_POST:
444  case AArch64::LD2Twov8b_POST:
445  case AArch64::LD2Twov2d_POST:
446  case AArch64::LD2Twov4h_POST:
447  case AArch64::LD2Twov8h_POST:
448  case AArch64::LD2Twov16b_POST:
449  case AArch64::LD2Rv1d_POST:
450  case AArch64::LD2Rv2s_POST:
451  case AArch64::LD2Rv4s_POST:
452  case AArch64::LD2Rv8b_POST:
453  case AArch64::LD2Rv2d_POST:
454  case AArch64::LD2Rv4h_POST:
455  case AArch64::LD2Rv8h_POST:
456  case AArch64::LD2Rv16b_POST:
457  case AArch64::LD3Threev2s_POST:
458  case AArch64::LD3Threev4h_POST:
459  case AArch64::LD3Threev8b_POST:
460  case AArch64::LD3Threev2d_POST:
461  case AArch64::LD3Threev4s_POST:
462  case AArch64::LD3Threev8h_POST:
463  case AArch64::LD3Threev16b_POST:
464  case AArch64::LD3Rv1d_POST:
465  case AArch64::LD3Rv2s_POST:
466  case AArch64::LD3Rv4h_POST:
467  case AArch64::LD3Rv8b_POST:
468  case AArch64::LD3Rv2d_POST:
469  case AArch64::LD3Rv4s_POST:
470  case AArch64::LD3Rv8h_POST:
471  case AArch64::LD3Rv16b_POST:
472  case AArch64::LD4Fourv2s_POST:
473  case AArch64::LD4Fourv4h_POST:
474  case AArch64::LD4Fourv8b_POST:
475  case AArch64::LD4Fourv2d_POST:
476  case AArch64::LD4Fourv4s_POST:
477  case AArch64::LD4Fourv8h_POST:
478  case AArch64::LD4Fourv16b_POST:
479  case AArch64::LD4Rv1d_POST:
480  case AArch64::LD4Rv2s_POST:
481  case AArch64::LD4Rv4h_POST:
482  case AArch64::LD4Rv8b_POST:
483  case AArch64::LD4Rv2d_POST:
484  case AArch64::LD4Rv4s_POST:
485  case AArch64::LD4Rv8h_POST:
486  case AArch64::LD4Rv16b_POST:
487  DestRegIdx = -1;
488  BaseRegIdx = 2;
489  OffsetIdx = 3;
490  IsPrePost = true;
491  break;
492 
493  case AArch64::LDRBBroW:
494  case AArch64::LDRBBroX:
495  case AArch64::LDRBBui:
496  case AArch64::LDRBroW:
497  case AArch64::LDRBroX:
498  case AArch64::LDRBui:
499  case AArch64::LDRDl:
500  case AArch64::LDRDroW:
501  case AArch64::LDRDroX:
502  case AArch64::LDRDui:
503  case AArch64::LDRHHroW:
504  case AArch64::LDRHHroX:
505  case AArch64::LDRHHui:
506  case AArch64::LDRHroW:
507  case AArch64::LDRHroX:
508  case AArch64::LDRHui:
509  case AArch64::LDRQl:
510  case AArch64::LDRQroW:
511  case AArch64::LDRQroX:
512  case AArch64::LDRQui:
513  case AArch64::LDRSBWroW:
514  case AArch64::LDRSBWroX:
515  case AArch64::LDRSBWui:
516  case AArch64::LDRSBXroW:
517  case AArch64::LDRSBXroX:
518  case AArch64::LDRSBXui:
519  case AArch64::LDRSHWroW:
520  case AArch64::LDRSHWroX:
521  case AArch64::LDRSHWui:
522  case AArch64::LDRSHXroW:
523  case AArch64::LDRSHXroX:
524  case AArch64::LDRSHXui:
525  case AArch64::LDRSWl:
526  case AArch64::LDRSWroW:
527  case AArch64::LDRSWroX:
528  case AArch64::LDRSWui:
529  case AArch64::LDRSl:
530  case AArch64::LDRSroW:
531  case AArch64::LDRSroX:
532  case AArch64::LDRSui:
533  case AArch64::LDRWl:
534  case AArch64::LDRWroW:
535  case AArch64::LDRWroX:
536  case AArch64::LDRWui:
537  case AArch64::LDRXl:
538  case AArch64::LDRXroW:
539  case AArch64::LDRXroX:
540  case AArch64::LDRXui:
541  case AArch64::LDURBBi:
542  case AArch64::LDURBi:
543  case AArch64::LDURDi:
544  case AArch64::LDURHHi:
545  case AArch64::LDURHi:
546  case AArch64::LDURQi:
547  case AArch64::LDURSBWi:
548  case AArch64::LDURSBXi:
549  case AArch64::LDURSHWi:
550  case AArch64::LDURSHXi:
551  case AArch64::LDURSWi:
552  case AArch64::LDURSi:
553  case AArch64::LDURWi:
554  case AArch64::LDURXi:
555  DestRegIdx = 0;
556  BaseRegIdx = 1;
557  OffsetIdx = 2;
558  IsPrePost = false;
559  break;
560 
561  case AArch64::LDRBBpost:
562  case AArch64::LDRBBpre:
563  case AArch64::LDRBpost:
564  case AArch64::LDRBpre:
565  case AArch64::LDRDpost:
566  case AArch64::LDRDpre:
567  case AArch64::LDRHHpost:
568  case AArch64::LDRHHpre:
569  case AArch64::LDRHpost:
570  case AArch64::LDRHpre:
571  case AArch64::LDRQpost:
572  case AArch64::LDRQpre:
573  case AArch64::LDRSBWpost:
574  case AArch64::LDRSBWpre:
575  case AArch64::LDRSBXpost:
576  case AArch64::LDRSBXpre:
577  case AArch64::LDRSHWpost:
578  case AArch64::LDRSHWpre:
579  case AArch64::LDRSHXpost:
580  case AArch64::LDRSHXpre:
581  case AArch64::LDRSWpost:
582  case AArch64::LDRSWpre:
583  case AArch64::LDRSpost:
584  case AArch64::LDRSpre:
585  case AArch64::LDRWpost:
586  case AArch64::LDRWpre:
587  case AArch64::LDRXpost:
588  case AArch64::LDRXpre:
589  DestRegIdx = 1;
590  BaseRegIdx = 2;
591  OffsetIdx = 3;
592  IsPrePost = true;
593  break;
594 
595  case AArch64::LDNPDi:
596  case AArch64::LDNPQi:
597  case AArch64::LDNPSi:
598  case AArch64::LDPQi:
599  case AArch64::LDPDi:
600  case AArch64::LDPSi:
601  DestRegIdx = -1;
602  BaseRegIdx = 2;
603  OffsetIdx = 3;
604  IsPrePost = false;
605  break;
606 
607  case AArch64::LDPSWi:
608  case AArch64::LDPWi:
609  case AArch64::LDPXi:
610  DestRegIdx = 0;
611  BaseRegIdx = 2;
612  OffsetIdx = 3;
613  IsPrePost = false;
614  break;
615 
616  case AArch64::LDPQpost:
617  case AArch64::LDPQpre:
618  case AArch64::LDPDpost:
619  case AArch64::LDPDpre:
620  case AArch64::LDPSpost:
621  case AArch64::LDPSpre:
622  DestRegIdx = -1;
623  BaseRegIdx = 3;
624  OffsetIdx = 4;
625  IsPrePost = true;
626  break;
627 
628  case AArch64::LDPSWpost:
629  case AArch64::LDPSWpre:
630  case AArch64::LDPWpost:
631  case AArch64::LDPWpre:
632  case AArch64::LDPXpost:
633  case AArch64::LDPXpre:
634  DestRegIdx = 1;
635  BaseRegIdx = 3;
636  OffsetIdx = 4;
637  IsPrePost = true;
638  break;
639  }
640 
641  // Loads from the stack pointer don't get prefetched.
642  unsigned BaseReg = MI.getOperand(BaseRegIdx).getReg();
643  if (BaseReg == AArch64::SP || BaseReg == AArch64::WSP)
644  return None;
645 
646  LoadInfo LI;
647  LI.DestReg = DestRegIdx == -1 ? 0 : MI.getOperand(DestRegIdx).getReg();
648  LI.BaseReg = BaseReg;
649  LI.BaseRegIdx = BaseRegIdx;
650  LI.OffsetOpnd = OffsetIdx == -1 ? nullptr : &MI.getOperand(OffsetIdx);
651  LI.IsPrePost = IsPrePost;
652  return LI;
653 }
654 
656  const MachineInstr &MI, const LoadInfo &LI) {
657  unsigned Dest = LI.DestReg ? TRI->getEncodingValue(LI.DestReg) : 0;
658  unsigned Base = TRI->getEncodingValue(LI.BaseReg);
659  unsigned Off;
660  if (LI.OffsetOpnd == nullptr)
661  Off = 0;
662  else if (LI.OffsetOpnd->isGlobal() || LI.OffsetOpnd->isSymbol() ||
663  LI.OffsetOpnd->isCPI())
664  return None;
665  else if (LI.OffsetOpnd->isReg())
666  Off = (1 << 5) | TRI->getEncodingValue(LI.OffsetOpnd->getReg());
667  else
668  Off = LI.OffsetOpnd->getImm() >> 2;
669 
670  return makeTag(Dest, Base, Off);
671 }
672 
673 void FalkorHWPFFix::runOnLoop(MachineLoop &L, MachineFunction &Fn) {
674  // Build the initial tag map for the whole loop.
675  TagMap.clear();
676  for (MachineBasicBlock *MBB : L.getBlocks())
677  for (MachineInstr &MI : *MBB) {
679  if (!LInfo)
680  continue;
681  Optional<unsigned> Tag = getTag(TRI, MI, *LInfo);
682  if (!Tag)
683  continue;
684  TagMap[*Tag].push_back(&MI);
685  }
686 
687  bool AnyCollisions = false;
688  for (auto &P : TagMap) {
689  auto Size = P.second.size();
690  if (Size > 1) {
691  for (auto *MI : P.second) {
692  if (TII->isStridedAccess(*MI)) {
693  AnyCollisions = true;
694  break;
695  }
696  }
697  }
698  if (AnyCollisions)
699  break;
700  }
701  // Nothing to fix.
702  if (!AnyCollisions)
703  return;
704 
706 
707  // Go through all the basic blocks in the current loop and fix any streaming
708  // loads to avoid collisions with any other loads.
709  LiveRegUnits LR(*TRI);
710  for (MachineBasicBlock *MBB : L.getBlocks()) {
711  LR.clear();
712  LR.addLiveOuts(*MBB);
713  for (auto I = MBB->rbegin(); I != MBB->rend(); LR.stepBackward(*I), ++I) {
714  MachineInstr &MI = *I;
715  if (!TII->isStridedAccess(MI))
716  continue;
717 
718  Optional<LoadInfo> OptLdI = getLoadInfo(MI);
719  if (!OptLdI)
720  continue;
721  LoadInfo LdI = *OptLdI;
722  Optional<unsigned> OptOldTag = getTag(TRI, MI, LdI);
723  if (!OptOldTag)
724  continue;
725  auto &OldCollisions = TagMap[*OptOldTag];
726  if (OldCollisions.size() <= 1)
727  continue;
728 
729  bool Fixed = false;
730  DEBUG(dbgs() << "Attempting to fix tag collision: " << MI);
731 
732  for (unsigned ScratchReg : AArch64::GPR64RegClass) {
733  if (!LR.available(ScratchReg) || MRI.isReserved(ScratchReg))
734  continue;
735 
736  LoadInfo NewLdI(LdI);
737  NewLdI.BaseReg = ScratchReg;
738  unsigned NewTag = *getTag(TRI, MI, NewLdI);
739  // Scratch reg tag would collide too, so don't use it.
740  if (TagMap.count(NewTag))
741  continue;
742 
743  DEBUG(dbgs() << "Changing base reg to: " << PrintReg(ScratchReg, TRI)
744  << '\n');
745 
746  // Rewrite:
747  // Xd = LOAD Xb, off
748  // to:
749  // Xc = MOV Xb
750  // Xd = LOAD Xc, off
751  DebugLoc DL = MI.getDebugLoc();
752  BuildMI(*MBB, &MI, DL, TII->get(AArch64::ORRXrs), ScratchReg)
753  .addReg(AArch64::XZR)
754  .addReg(LdI.BaseReg)
755  .addImm(0);
756  MachineOperand &BaseOpnd = MI.getOperand(LdI.BaseRegIdx);
757  BaseOpnd.setReg(ScratchReg);
758 
759  // If the load does a pre/post increment, then insert a MOV after as
760  // well to update the real base register.
761  if (LdI.IsPrePost) {
762  DEBUG(dbgs() << "Doing post MOV of incremented reg: "
763  << PrintReg(ScratchReg, TRI) << '\n');
764  MI.getOperand(0).setReg(
765  ScratchReg); // Change tied operand pre/post update dest.
766  BuildMI(*MBB, std::next(MachineBasicBlock::iterator(MI)), DL,
767  TII->get(AArch64::ORRXrs), LdI.BaseReg)
768  .addReg(AArch64::XZR)
769  .addReg(ScratchReg)
770  .addImm(0);
771  }
772 
773  for (int I = 0, E = OldCollisions.size(); I != E; ++I)
774  if (OldCollisions[I] == &MI) {
775  std::swap(OldCollisions[I], OldCollisions[E - 1]);
776  OldCollisions.pop_back();
777  break;
778  }
779 
780  // Update TagMap to reflect instruction changes to reduce the number
781  // of later MOVs to be inserted. This needs to be done after
782  // OldCollisions is updated since it may be relocated by this
783  // insertion.
784  TagMap[NewTag].push_back(&MI);
785  ++NumCollisionsAvoided;
786  Fixed = true;
787  Modified = true;
788  break;
789  }
790  if (!Fixed)
791  ++NumCollisionsNotAvoided;
792  }
793  }
794 }
795 
796 bool FalkorHWPFFix::runOnMachineFunction(MachineFunction &Fn) {
797  auto &ST = static_cast<const AArch64Subtarget &>(Fn.getSubtarget());
798  if (ST.getProcFamily() != AArch64Subtarget::Falkor)
799  return false;
800 
801  if (skipFunction(*Fn.getFunction()))
802  return false;
803 
804  TII = static_cast<const AArch64InstrInfo *>(ST.getInstrInfo());
805  TRI = ST.getRegisterInfo();
806 
807  assert(TRI->trackLivenessAfterRegAlloc(Fn) &&
808  "Register liveness not available!");
809 
810  MachineLoopInfo &LI = getAnalysis<MachineLoopInfo>();
811 
812  Modified = false;
813 
814  for (MachineLoop *I : LI)
815  for (auto L = df_begin(I), LE = df_end(I); L != LE; ++L)
816  // Only process inner-loops
817  if (L->empty())
818  runOnLoop(**L, Fn);
819 
820  return Modified;
821 }
822 
823 FunctionPass *llvm::createFalkorHWPFFixPass() { return new FalkorHWPFFix(); }
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
FunctionPass * createFalkorMarkStridedAccessesPass()
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:268
unsigned getReg() const
getReg - Returns the register number.
The main scalar evolution driver.
This file contains the declarations for metadata subclasses.
static Optional< LoadInfo > getLoadInfo(const MachineInstr &MI)
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:728
STATISTIC(NumFunctions, "Total number of functions")
A debug info location.
Definition: DebugLoc.h:34
TMC & getTM() const
Get the right type of TargetMachine for this target.
F(f)
An instruction for reading from memory.
Definition: Instructions.h:164
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:51
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
falkor hwpf fix Falkor HW Prefetch Fix Late static false unsigned makeTag(unsigned Dest, unsigned Base, unsigned Offset)
void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:290
Target-Independent Code Generator Pass Configuration Options.
falkor hwpf fix Falkor HW Prefetch Fix Late Phase
This node represents a polynomial recurrence on the trip count of the specified loop.
Printable PrintReg(unsigned Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubRegIdx=0)
Prints virtual and physical registers with or without a TRI instance.
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata *> MDs)
Definition: Metadata.h:1164
static bool runOnFunction(Function &F, bool PostInlining)
#define P(N)
unsigned const MachineRegisterInfo * MRI
LLVM Basic Block Representation.
Definition: BasicBlock.h:59
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
df_iterator< T > df_end(const T &G)
#define DEBUG_TYPE
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Represent the analysis usage information of a pass.
void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:285
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values...
Value * getPointerOperand()
Definition: Instructions.h:270
A set of register units.
Falkor HW Prefetch Fix
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1214
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition: LoopInfo.cpp:56
falkor hwpf fix late
void initializeFalkorMarkStridedAccessesLegacyPass(PassRegistry &)
INITIALIZE_PASS_BEGIN(FalkorMarkStridedAccessesLegacy, DEBUG_TYPE, "Falkor HW Prefetch Fix", false, false) INITIALIZE_PASS_END(FalkorMarkStridedAccessesLegacy
MachineOperand class - Representation of each machine instruction operand.
void initializeFalkorHWPFFixPass(PassRegistry &)
static Optional< unsigned > getTag(const TargetRegisterInfo *TRI, const MachineInstr &MI, const LoadInfo &LI)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:923
df_iterator< T > df_begin(const T &G)
loop data Loop Data Prefetch
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
MachineFunctionProperties & set(Property P)
Representation of each machine instruction.
Definition: MachineInstr.h:59
#define FALKOR_STRIDED_ACCESS_MD
uint16_t getEncodingValue(unsigned RegNo) const
Returns the encoding for RegNo.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
This class represents an analyzed expression in the program.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:439
bool available(unsigned Reg) const
Returns true if no part of physical register Reg is live.
Definition: LiveRegUnits.h:87
ArrayRef< BlockT * > getBlocks() const
Get a list of the basic blocks which make up this loop.
Definition: LoopInfo.h:149
void clear()
Clears the set.
Definition: LiveRegUnits.h:51
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:58
FunctionPass * createFalkorHWPFFixPass()
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
A set of register units used to track register liveness.
Definition: LiveRegUnits.h:30
bool empty() const
Definition: LoopInfo.h:146
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
Definition: Value.h:73
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
#define DEBUG(X)
Definition: Debug.h:118
The legacy pass manager&#39;s analysis pass to compute loop information.
Definition: LoopInfo.h:958
IRTranslator LLVM IR MI
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:267
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:295
iterator_range< block_iterator > blocks() const
Definition: LoopInfo.h:156
Properties which a MachineFunction may have at a given point in time.
bool isReserved(unsigned PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.