LLVM  10.0.0svn
X86AvoidStoreForwardingBlocks.cpp
Go to the documentation of this file.
1 //===- X86AvoidStoreForwardingBlockis.cpp - Avoid HW Store Forward Block --===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // If a load follows a store and reloads data that the store has written to
10 // memory, Intel microarchitectures can in many cases forward the data directly
11 // from the store to the load, This "store forwarding" saves cycles by enabling
12 // the load to directly obtain the data instead of accessing the data from
13 // cache or memory.
14 // A "store forward block" occurs in cases that a store cannot be forwarded to
15 // the load. The most typical case of store forward block on Intel Core
16 // microarchitecture that a small store cannot be forwarded to a large load.
17 // The estimated penalty for a store forward block is ~13 cycles.
18 //
19 // This pass tries to recognize and handle cases where "store forward block"
20 // is created by the compiler when lowering memcpy calls to a sequence
21 // of a load and a store.
22 //
23 // The pass currently only handles cases where memcpy is lowered to
24 // XMM/YMM registers, it tries to break the memcpy into smaller copies.
25 // breaking the memcpy should be possible since there is no atomicity
26 // guarantee for loads and stores to XMM/YMM.
27 //
28 // It could be better for performance to solve the problem by loading
29 // to XMM/YMM then inserting the partial store before storing back from XMM/YMM
30 // to memory, but this will result in a more conservative optimization since it
31 // requires we prove that all memory accesses between the blocking store and the
32 // load must alias/don't alias before we can move the store, whereas the
33 // transformation done here is correct regardless to other memory accesses.
34 //===----------------------------------------------------------------------===//
35 
36 #include "X86InstrInfo.h"
37 #include "X86Subtarget.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/Function.h"
48 #include "llvm/MC/MCInstrDesc.h"
49 
50 using namespace llvm;
51 
52 #define DEBUG_TYPE "x86-avoid-SFB"
53 
55  "x86-disable-avoid-SFB", cl::Hidden,
56  cl::desc("X86: Disable Store Forwarding Blocks fixup."), cl::init(false));
57 
59  "x86-sfb-inspection-limit",
60  cl::desc("X86: Number of instructions backward to "
61  "inspect for store forwarding blocks."),
62  cl::init(20), cl::Hidden);
63 
64 namespace {
65 
66 using DisplacementSizeMap = std::map<int64_t, unsigned>;
67 
68 class X86AvoidSFBPass : public MachineFunctionPass {
69 public:
70  static char ID;
71  X86AvoidSFBPass() : MachineFunctionPass(ID) { }
72 
73  StringRef getPassName() const override {
74  return "X86 Avoid Store Forwarding Blocks";
75  }
76 
77  bool runOnMachineFunction(MachineFunction &MF) override;
78 
79  void getAnalysisUsage(AnalysisUsage &AU) const override {
82  }
83 
84 private:
86  const X86InstrInfo *TII;
87  const X86RegisterInfo *TRI;
89  BlockedLoadsStoresPairs;
91  AliasAnalysis *AA;
92 
93  /// Returns couples of Load then Store to memory which look
94  /// like a memcpy.
95  void findPotentiallylBlockedCopies(MachineFunction &MF);
96  /// Break the memcpy's load and store into smaller copies
97  /// such that each memory load that was blocked by a smaller store
98  /// would now be copied separately.
99  void breakBlockedCopies(MachineInstr *LoadInst, MachineInstr *StoreInst,
100  const DisplacementSizeMap &BlockingStoresDispSizeMap);
101  /// Break a copy of size Size to smaller copies.
102  void buildCopies(int Size, MachineInstr *LoadInst, int64_t LdDispImm,
103  MachineInstr *StoreInst, int64_t StDispImm,
104  int64_t LMMOffset, int64_t SMMOffset);
105 
106  void buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode, int64_t LoadDisp,
107  MachineInstr *StoreInst, unsigned NStoreOpcode,
108  int64_t StoreDisp, unsigned Size, int64_t LMMOffset,
109  int64_t SMMOffset);
110 
111  bool alias(const MachineMemOperand &Op1, const MachineMemOperand &Op2) const;
112 
113  unsigned getRegSizeInBytes(MachineInstr *Inst);
114 };
115 
116 } // end anonymous namespace
117 
118 char X86AvoidSFBPass::ID = 0;
119 
120 INITIALIZE_PASS_BEGIN(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking",
121  false, false)
124  false)
125 
127  return new X86AvoidSFBPass();
128 }
129 
130 static bool isXMMLoadOpcode(unsigned Opcode) {
131  return Opcode == X86::MOVUPSrm || Opcode == X86::MOVAPSrm ||
132  Opcode == X86::VMOVUPSrm || Opcode == X86::VMOVAPSrm ||
133  Opcode == X86::VMOVUPDrm || Opcode == X86::VMOVAPDrm ||
134  Opcode == X86::VMOVDQUrm || Opcode == X86::VMOVDQArm ||
135  Opcode == X86::VMOVUPSZ128rm || Opcode == X86::VMOVAPSZ128rm ||
136  Opcode == X86::VMOVUPDZ128rm || Opcode == X86::VMOVAPDZ128rm ||
137  Opcode == X86::VMOVDQU64Z128rm || Opcode == X86::VMOVDQA64Z128rm ||
138  Opcode == X86::VMOVDQU32Z128rm || Opcode == X86::VMOVDQA32Z128rm;
139 }
140 static bool isYMMLoadOpcode(unsigned Opcode) {
141  return Opcode == X86::VMOVUPSYrm || Opcode == X86::VMOVAPSYrm ||
142  Opcode == X86::VMOVUPDYrm || Opcode == X86::VMOVAPDYrm ||
143  Opcode == X86::VMOVDQUYrm || Opcode == X86::VMOVDQAYrm ||
144  Opcode == X86::VMOVUPSZ256rm || Opcode == X86::VMOVAPSZ256rm ||
145  Opcode == X86::VMOVUPDZ256rm || Opcode == X86::VMOVAPDZ256rm ||
146  Opcode == X86::VMOVDQU64Z256rm || Opcode == X86::VMOVDQA64Z256rm ||
147  Opcode == X86::VMOVDQU32Z256rm || Opcode == X86::VMOVDQA32Z256rm;
148 }
149 
150 static bool isPotentialBlockedMemCpyLd(unsigned Opcode) {
151  return isXMMLoadOpcode(Opcode) || isYMMLoadOpcode(Opcode);
152 }
153 
154 static bool isPotentialBlockedMemCpyPair(int LdOpcode, int StOpcode) {
155  switch (LdOpcode) {
156  case X86::MOVUPSrm:
157  case X86::MOVAPSrm:
158  return StOpcode == X86::MOVUPSmr || StOpcode == X86::MOVAPSmr;
159  case X86::VMOVUPSrm:
160  case X86::VMOVAPSrm:
161  return StOpcode == X86::VMOVUPSmr || StOpcode == X86::VMOVAPSmr;
162  case X86::VMOVUPDrm:
163  case X86::VMOVAPDrm:
164  return StOpcode == X86::VMOVUPDmr || StOpcode == X86::VMOVAPDmr;
165  case X86::VMOVDQUrm:
166  case X86::VMOVDQArm:
167  return StOpcode == X86::VMOVDQUmr || StOpcode == X86::VMOVDQAmr;
168  case X86::VMOVUPSZ128rm:
169  case X86::VMOVAPSZ128rm:
170  return StOpcode == X86::VMOVUPSZ128mr || StOpcode == X86::VMOVAPSZ128mr;
171  case X86::VMOVUPDZ128rm:
172  case X86::VMOVAPDZ128rm:
173  return StOpcode == X86::VMOVUPDZ128mr || StOpcode == X86::VMOVAPDZ128mr;
174  case X86::VMOVUPSYrm:
175  case X86::VMOVAPSYrm:
176  return StOpcode == X86::VMOVUPSYmr || StOpcode == X86::VMOVAPSYmr;
177  case X86::VMOVUPDYrm:
178  case X86::VMOVAPDYrm:
179  return StOpcode == X86::VMOVUPDYmr || StOpcode == X86::VMOVAPDYmr;
180  case X86::VMOVDQUYrm:
181  case X86::VMOVDQAYrm:
182  return StOpcode == X86::VMOVDQUYmr || StOpcode == X86::VMOVDQAYmr;
183  case X86::VMOVUPSZ256rm:
184  case X86::VMOVAPSZ256rm:
185  return StOpcode == X86::VMOVUPSZ256mr || StOpcode == X86::VMOVAPSZ256mr;
186  case X86::VMOVUPDZ256rm:
187  case X86::VMOVAPDZ256rm:
188  return StOpcode == X86::VMOVUPDZ256mr || StOpcode == X86::VMOVAPDZ256mr;
189  case X86::VMOVDQU64Z128rm:
190  case X86::VMOVDQA64Z128rm:
191  return StOpcode == X86::VMOVDQU64Z128mr || StOpcode == X86::VMOVDQA64Z128mr;
192  case X86::VMOVDQU32Z128rm:
193  case X86::VMOVDQA32Z128rm:
194  return StOpcode == X86::VMOVDQU32Z128mr || StOpcode == X86::VMOVDQA32Z128mr;
195  case X86::VMOVDQU64Z256rm:
196  case X86::VMOVDQA64Z256rm:
197  return StOpcode == X86::VMOVDQU64Z256mr || StOpcode == X86::VMOVDQA64Z256mr;
198  case X86::VMOVDQU32Z256rm:
199  case X86::VMOVDQA32Z256rm:
200  return StOpcode == X86::VMOVDQU32Z256mr || StOpcode == X86::VMOVDQA32Z256mr;
201  default:
202  return false;
203  }
204 }
205 
206 static bool isPotentialBlockingStoreInst(int Opcode, int LoadOpcode) {
207  bool PBlock = false;
208  PBlock |= Opcode == X86::MOV64mr || Opcode == X86::MOV64mi32 ||
209  Opcode == X86::MOV32mr || Opcode == X86::MOV32mi ||
210  Opcode == X86::MOV16mr || Opcode == X86::MOV16mi ||
211  Opcode == X86::MOV8mr || Opcode == X86::MOV8mi;
212  if (isYMMLoadOpcode(LoadOpcode))
213  PBlock |= Opcode == X86::VMOVUPSmr || Opcode == X86::VMOVAPSmr ||
214  Opcode == X86::VMOVUPDmr || Opcode == X86::VMOVAPDmr ||
215  Opcode == X86::VMOVDQUmr || Opcode == X86::VMOVDQAmr ||
216  Opcode == X86::VMOVUPSZ128mr || Opcode == X86::VMOVAPSZ128mr ||
217  Opcode == X86::VMOVUPDZ128mr || Opcode == X86::VMOVAPDZ128mr ||
218  Opcode == X86::VMOVDQU64Z128mr ||
219  Opcode == X86::VMOVDQA64Z128mr ||
220  Opcode == X86::VMOVDQU32Z128mr || Opcode == X86::VMOVDQA32Z128mr;
221  return PBlock;
222 }
223 
224 static const int MOV128SZ = 16;
225 static const int MOV64SZ = 8;
226 static const int MOV32SZ = 4;
227 static const int MOV16SZ = 2;
228 static const int MOV8SZ = 1;
229 
230 static unsigned getYMMtoXMMLoadOpcode(unsigned LoadOpcode) {
231  switch (LoadOpcode) {
232  case X86::VMOVUPSYrm:
233  case X86::VMOVAPSYrm:
234  return X86::VMOVUPSrm;
235  case X86::VMOVUPDYrm:
236  case X86::VMOVAPDYrm:
237  return X86::VMOVUPDrm;
238  case X86::VMOVDQUYrm:
239  case X86::VMOVDQAYrm:
240  return X86::VMOVDQUrm;
241  case X86::VMOVUPSZ256rm:
242  case X86::VMOVAPSZ256rm:
243  return X86::VMOVUPSZ128rm;
244  case X86::VMOVUPDZ256rm:
245  case X86::VMOVAPDZ256rm:
246  return X86::VMOVUPDZ128rm;
247  case X86::VMOVDQU64Z256rm:
248  case X86::VMOVDQA64Z256rm:
249  return X86::VMOVDQU64Z128rm;
250  case X86::VMOVDQU32Z256rm:
251  case X86::VMOVDQA32Z256rm:
252  return X86::VMOVDQU32Z128rm;
253  default:
254  llvm_unreachable("Unexpected Load Instruction Opcode");
255  }
256  return 0;
257 }
258 
259 static unsigned getYMMtoXMMStoreOpcode(unsigned StoreOpcode) {
260  switch (StoreOpcode) {
261  case X86::VMOVUPSYmr:
262  case X86::VMOVAPSYmr:
263  return X86::VMOVUPSmr;
264  case X86::VMOVUPDYmr:
265  case X86::VMOVAPDYmr:
266  return X86::VMOVUPDmr;
267  case X86::VMOVDQUYmr:
268  case X86::VMOVDQAYmr:
269  return X86::VMOVDQUmr;
270  case X86::VMOVUPSZ256mr:
271  case X86::VMOVAPSZ256mr:
272  return X86::VMOVUPSZ128mr;
273  case X86::VMOVUPDZ256mr:
274  case X86::VMOVAPDZ256mr:
275  return X86::VMOVUPDZ128mr;
276  case X86::VMOVDQU64Z256mr:
277  case X86::VMOVDQA64Z256mr:
278  return X86::VMOVDQU64Z128mr;
279  case X86::VMOVDQU32Z256mr:
280  case X86::VMOVDQA32Z256mr:
281  return X86::VMOVDQU32Z128mr;
282  default:
283  llvm_unreachable("Unexpected Load Instruction Opcode");
284  }
285  return 0;
286 }
287 
289  const MCInstrDesc &Descl = MI->getDesc();
290  int AddrOffset = X86II::getMemoryOperandNo(Descl.TSFlags);
291  assert(AddrOffset != -1 && "Expected Memory Operand");
292  AddrOffset += X86II::getOperandBias(Descl);
293  return AddrOffset;
294 }
295 
297  int AddrOffset = getAddrOffset(MI);
298  return MI->getOperand(AddrOffset + X86::AddrBaseReg);
299 }
300 
302  int AddrOffset = getAddrOffset(MI);
303  return MI->getOperand(AddrOffset + X86::AddrDisp);
304 }
305 
306 // Relevant addressing modes contain only base register and immediate
307 // displacement or frameindex and immediate displacement.
308 // TODO: Consider expanding to other addressing modes in the future
310  int AddrOffset = getAddrOffset(MI);
312  MachineOperand &Disp = getDispOperand(MI);
313  MachineOperand &Scale = MI->getOperand(AddrOffset + X86::AddrScaleAmt);
314  MachineOperand &Index = MI->getOperand(AddrOffset + X86::AddrIndexReg);
315  MachineOperand &Segment = MI->getOperand(AddrOffset + X86::AddrSegmentReg);
316 
317  if (!((Base.isReg() && Base.getReg() != X86::NoRegister) || Base.isFI()))
318  return false;
319  if (!Disp.isImm())
320  return false;
321  if (Scale.getImm() != 1)
322  return false;
323  if (!(Index.isReg() && Index.getReg() == X86::NoRegister))
324  return false;
325  if (!(Segment.isReg() && Segment.getReg() == X86::NoRegister))
326  return false;
327  return true;
328 }
329 
330 // Collect potentially blocking stores.
331 // Limit the number of instructions backwards we want to inspect
332 // since the effect of store block won't be visible if the store
333 // and load instructions have enough instructions in between to
334 // keep the core busy.
337  SmallVector<MachineInstr *, 2> PotentialBlockers;
338  unsigned BlockCount = 0;
339  const unsigned InspectionLimit = X86AvoidSFBInspectionLimit;
340  for (auto PBInst = std::next(MachineBasicBlock::reverse_iterator(LoadInst)),
341  E = LoadInst->getParent()->rend();
342  PBInst != E; ++PBInst) {
343  if (PBInst->isMetaInstruction())
344  continue;
345  BlockCount++;
346  if (BlockCount >= InspectionLimit)
347  break;
348  MachineInstr &MI = *PBInst;
349  if (MI.getDesc().isCall())
350  return PotentialBlockers;
351  PotentialBlockers.push_back(&MI);
352  }
353  // If we didn't get to the instructions limit try predecessing blocks.
354  // Ideally we should traverse the predecessor blocks in depth with some
355  // coloring algorithm, but for now let's just look at the first order
356  // predecessors.
357  if (BlockCount < InspectionLimit) {
358  MachineBasicBlock *MBB = LoadInst->getParent();
359  int LimitLeft = InspectionLimit - BlockCount;
361  PE = MBB->pred_end();
362  PB != PE; ++PB) {
363  MachineBasicBlock *PMBB = *PB;
364  int PredCount = 0;
365  for (MachineBasicBlock::reverse_iterator PBInst = PMBB->rbegin(),
366  PME = PMBB->rend();
367  PBInst != PME; ++PBInst) {
368  if (PBInst->isMetaInstruction())
369  continue;
370  PredCount++;
371  if (PredCount >= LimitLeft)
372  break;
373  if (PBInst->getDesc().isCall())
374  break;
375  PotentialBlockers.push_back(&*PBInst);
376  }
377  }
378  }
379  return PotentialBlockers;
380 }
381 
382 void X86AvoidSFBPass::buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode,
383  int64_t LoadDisp, MachineInstr *StoreInst,
384  unsigned NStoreOpcode, int64_t StoreDisp,
385  unsigned Size, int64_t LMMOffset,
386  int64_t SMMOffset) {
387  MachineOperand &LoadBase = getBaseOperand(LoadInst);
388  MachineOperand &StoreBase = getBaseOperand(StoreInst);
389  MachineBasicBlock *MBB = LoadInst->getParent();
390  MachineMemOperand *LMMO = *LoadInst->memoperands_begin();
391  MachineMemOperand *SMMO = *StoreInst->memoperands_begin();
392 
393  Register Reg1 = MRI->createVirtualRegister(
394  TII->getRegClass(TII->get(NLoadOpcode), 0, TRI, *(MBB->getParent())));
395  MachineInstr *NewLoad =
396  BuildMI(*MBB, LoadInst, LoadInst->getDebugLoc(), TII->get(NLoadOpcode),
397  Reg1)
398  .add(LoadBase)
399  .addImm(1)
400  .addReg(X86::NoRegister)
401  .addImm(LoadDisp)
402  .addReg(X86::NoRegister)
403  .addMemOperand(
404  MBB->getParent()->getMachineMemOperand(LMMO, LMMOffset, Size));
405  if (LoadBase.isReg())
406  getBaseOperand(NewLoad).setIsKill(false);
407  LLVM_DEBUG(NewLoad->dump());
408  // If the load and store are consecutive, use the loadInst location to
409  // reduce register pressure.
410  MachineInstr *StInst = StoreInst;
411  auto PrevInstrIt = skipDebugInstructionsBackward(
412  std::prev(MachineBasicBlock::instr_iterator(StoreInst)),
413  MBB->instr_begin());
414  if (PrevInstrIt.getNodePtr() == LoadInst)
415  StInst = LoadInst;
416  MachineInstr *NewStore =
417  BuildMI(*MBB, StInst, StInst->getDebugLoc(), TII->get(NStoreOpcode))
418  .add(StoreBase)
419  .addImm(1)
420  .addReg(X86::NoRegister)
421  .addImm(StoreDisp)
422  .addReg(X86::NoRegister)
423  .addReg(Reg1)
424  .addMemOperand(
425  MBB->getParent()->getMachineMemOperand(SMMO, SMMOffset, Size));
426  if (StoreBase.isReg())
427  getBaseOperand(NewStore).setIsKill(false);
428  MachineOperand &StoreSrcVReg = StoreInst->getOperand(X86::AddrNumOperands);
429  assert(StoreSrcVReg.isReg() && "Expected virtual register");
430  NewStore->getOperand(X86::AddrNumOperands).setIsKill(StoreSrcVReg.isKill());
431  LLVM_DEBUG(NewStore->dump());
432 }
433 
434 void X86AvoidSFBPass::buildCopies(int Size, MachineInstr *LoadInst,
435  int64_t LdDispImm, MachineInstr *StoreInst,
436  int64_t StDispImm, int64_t LMMOffset,
437  int64_t SMMOffset) {
438  int LdDisp = LdDispImm;
439  int StDisp = StDispImm;
440  while (Size > 0) {
441  if ((Size - MOV128SZ >= 0) && isYMMLoadOpcode(LoadInst->getOpcode())) {
442  Size = Size - MOV128SZ;
443  buildCopy(LoadInst, getYMMtoXMMLoadOpcode(LoadInst->getOpcode()), LdDisp,
444  StoreInst, getYMMtoXMMStoreOpcode(StoreInst->getOpcode()),
445  StDisp, MOV128SZ, LMMOffset, SMMOffset);
446  LdDisp += MOV128SZ;
447  StDisp += MOV128SZ;
448  LMMOffset += MOV128SZ;
449  SMMOffset += MOV128SZ;
450  continue;
451  }
452  if (Size - MOV64SZ >= 0) {
453  Size = Size - MOV64SZ;
454  buildCopy(LoadInst, X86::MOV64rm, LdDisp, StoreInst, X86::MOV64mr, StDisp,
455  MOV64SZ, LMMOffset, SMMOffset);
456  LdDisp += MOV64SZ;
457  StDisp += MOV64SZ;
458  LMMOffset += MOV64SZ;
459  SMMOffset += MOV64SZ;
460  continue;
461  }
462  if (Size - MOV32SZ >= 0) {
463  Size = Size - MOV32SZ;
464  buildCopy(LoadInst, X86::MOV32rm, LdDisp, StoreInst, X86::MOV32mr, StDisp,
465  MOV32SZ, LMMOffset, SMMOffset);
466  LdDisp += MOV32SZ;
467  StDisp += MOV32SZ;
468  LMMOffset += MOV32SZ;
469  SMMOffset += MOV32SZ;
470  continue;
471  }
472  if (Size - MOV16SZ >= 0) {
473  Size = Size - MOV16SZ;
474  buildCopy(LoadInst, X86::MOV16rm, LdDisp, StoreInst, X86::MOV16mr, StDisp,
475  MOV16SZ, LMMOffset, SMMOffset);
476  LdDisp += MOV16SZ;
477  StDisp += MOV16SZ;
478  LMMOffset += MOV16SZ;
479  SMMOffset += MOV16SZ;
480  continue;
481  }
482  if (Size - MOV8SZ >= 0) {
483  Size = Size - MOV8SZ;
484  buildCopy(LoadInst, X86::MOV8rm, LdDisp, StoreInst, X86::MOV8mr, StDisp,
485  MOV8SZ, LMMOffset, SMMOffset);
486  LdDisp += MOV8SZ;
487  StDisp += MOV8SZ;
488  LMMOffset += MOV8SZ;
489  SMMOffset += MOV8SZ;
490  continue;
491  }
492  }
493  assert(Size == 0 && "Wrong size division");
494 }
495 
496 static void updateKillStatus(MachineInstr *LoadInst, MachineInstr *StoreInst) {
497  MachineOperand &LoadBase = getBaseOperand(LoadInst);
498  MachineOperand &StoreBase = getBaseOperand(StoreInst);
499  auto StorePrevNonDbgInstr = skipDebugInstructionsBackward(
500  std::prev(MachineBasicBlock::instr_iterator(StoreInst)),
501  LoadInst->getParent()->instr_begin()).getNodePtr();
502  if (LoadBase.isReg()) {
503  MachineInstr *LastLoad = LoadInst->getPrevNode();
504  // If the original load and store to xmm/ymm were consecutive
505  // then the partial copies were also created in
506  // a consecutive order to reduce register pressure,
507  // and the location of the last load is before the last store.
508  if (StorePrevNonDbgInstr == LoadInst)
509  LastLoad = LoadInst->getPrevNode()->getPrevNode();
510  getBaseOperand(LastLoad).setIsKill(LoadBase.isKill());
511  }
512  if (StoreBase.isReg()) {
513  MachineInstr *StInst = StoreInst;
514  if (StorePrevNonDbgInstr == LoadInst)
515  StInst = LoadInst;
516  getBaseOperand(StInst->getPrevNode()).setIsKill(StoreBase.isKill());
517  }
518 }
519 
520 bool X86AvoidSFBPass::alias(const MachineMemOperand &Op1,
521  const MachineMemOperand &Op2) const {
522  if (!Op1.getValue() || !Op2.getValue())
523  return true;
524 
525  int64_t MinOffset = std::min(Op1.getOffset(), Op2.getOffset());
526  int64_t Overlapa = Op1.getSize() + Op1.getOffset() - MinOffset;
527  int64_t Overlapb = Op2.getSize() + Op2.getOffset() - MinOffset;
528 
529  AliasResult AAResult =
530  AA->alias(MemoryLocation(Op1.getValue(), Overlapa, Op1.getAAInfo()),
531  MemoryLocation(Op2.getValue(), Overlapb, Op2.getAAInfo()));
532  return AAResult != NoAlias;
533 }
534 
535 void X86AvoidSFBPass::findPotentiallylBlockedCopies(MachineFunction &MF) {
536  for (auto &MBB : MF)
537  for (auto &MI : MBB) {
538  if (!isPotentialBlockedMemCpyLd(MI.getOpcode()))
539  continue;
540  int DefVR = MI.getOperand(0).getReg();
541  if (!MRI->hasOneNonDBGUse(DefVR))
542  continue;
543  for (auto UI = MRI->use_nodbg_begin(DefVR), UE = MRI->use_nodbg_end();
544  UI != UE;) {
545  MachineOperand &StoreMO = *UI++;
546  MachineInstr &StoreMI = *StoreMO.getParent();
547  // Skip cases where the memcpy may overlap.
548  if (StoreMI.getParent() == MI.getParent() &&
549  isPotentialBlockedMemCpyPair(MI.getOpcode(), StoreMI.getOpcode()) &&
551  isRelevantAddressingMode(&StoreMI)) {
552  assert(MI.hasOneMemOperand() &&
553  "Expected one memory operand for load instruction");
554  assert(StoreMI.hasOneMemOperand() &&
555  "Expected one memory operand for store instruction");
556  if (!alias(**MI.memoperands_begin(), **StoreMI.memoperands_begin()))
557  BlockedLoadsStoresPairs.push_back(std::make_pair(&MI, &StoreMI));
558  }
559  }
560  }
561 }
562 
563 unsigned X86AvoidSFBPass::getRegSizeInBytes(MachineInstr *LoadInst) {
564  auto TRC = TII->getRegClass(TII->get(LoadInst->getOpcode()), 0, TRI,
565  *LoadInst->getParent()->getParent());
566  return TRI->getRegSizeInBits(*TRC) / 8;
567 }
568 
569 void X86AvoidSFBPass::breakBlockedCopies(
570  MachineInstr *LoadInst, MachineInstr *StoreInst,
571  const DisplacementSizeMap &BlockingStoresDispSizeMap) {
572  int64_t LdDispImm = getDispOperand(LoadInst).getImm();
573  int64_t StDispImm = getDispOperand(StoreInst).getImm();
574  int64_t LMMOffset = 0;
575  int64_t SMMOffset = 0;
576 
577  int64_t LdDisp1 = LdDispImm;
578  int64_t LdDisp2 = 0;
579  int64_t StDisp1 = StDispImm;
580  int64_t StDisp2 = 0;
581  unsigned Size1 = 0;
582  unsigned Size2 = 0;
583  int64_t LdStDelta = StDispImm - LdDispImm;
584 
585  for (auto DispSizePair : BlockingStoresDispSizeMap) {
586  LdDisp2 = DispSizePair.first;
587  StDisp2 = DispSizePair.first + LdStDelta;
588  Size2 = DispSizePair.second;
589  // Avoid copying overlapping areas.
590  if (LdDisp2 < LdDisp1) {
591  int OverlapDelta = LdDisp1 - LdDisp2;
592  LdDisp2 += OverlapDelta;
593  StDisp2 += OverlapDelta;
594  Size2 -= OverlapDelta;
595  }
596  Size1 = LdDisp2 - LdDisp1;
597 
598  // Build a copy for the point until the current blocking store's
599  // displacement.
600  buildCopies(Size1, LoadInst, LdDisp1, StoreInst, StDisp1, LMMOffset,
601  SMMOffset);
602  // Build a copy for the current blocking store.
603  buildCopies(Size2, LoadInst, LdDisp2, StoreInst, StDisp2, LMMOffset + Size1,
604  SMMOffset + Size1);
605  LdDisp1 = LdDisp2 + Size2;
606  StDisp1 = StDisp2 + Size2;
607  LMMOffset += Size1 + Size2;
608  SMMOffset += Size1 + Size2;
609  }
610  unsigned Size3 = (LdDispImm + getRegSizeInBytes(LoadInst)) - LdDisp1;
611  buildCopies(Size3, LoadInst, LdDisp1, StoreInst, StDisp1, LMMOffset,
612  LMMOffset);
613 }
614 
615 static bool hasSameBaseOpValue(MachineInstr *LoadInst,
616  MachineInstr *StoreInst) {
617  MachineOperand &LoadBase = getBaseOperand(LoadInst);
618  MachineOperand &StoreBase = getBaseOperand(StoreInst);
619  if (LoadBase.isReg() != StoreBase.isReg())
620  return false;
621  if (LoadBase.isReg())
622  return LoadBase.getReg() == StoreBase.getReg();
623  return LoadBase.getIndex() == StoreBase.getIndex();
624 }
625 
626 static bool isBlockingStore(int64_t LoadDispImm, unsigned LoadSize,
627  int64_t StoreDispImm, unsigned StoreSize) {
628  return ((StoreDispImm >= LoadDispImm) &&
629  (StoreDispImm <= LoadDispImm + (LoadSize - StoreSize)));
630 }
631 
632 // Keep track of all stores blocking a load
633 static void
634 updateBlockingStoresDispSizeMap(DisplacementSizeMap &BlockingStoresDispSizeMap,
635  int64_t DispImm, unsigned Size) {
636  if (BlockingStoresDispSizeMap.count(DispImm)) {
637  // Choose the smallest blocking store starting at this displacement.
638  if (BlockingStoresDispSizeMap[DispImm] > Size)
639  BlockingStoresDispSizeMap[DispImm] = Size;
640 
641  } else
642  BlockingStoresDispSizeMap[DispImm] = Size;
643 }
644 
645 // Remove blocking stores contained in each other.
646 static void
647 removeRedundantBlockingStores(DisplacementSizeMap &BlockingStoresDispSizeMap) {
648  if (BlockingStoresDispSizeMap.size() <= 1)
649  return;
650 
652  for (auto DispSizePair : BlockingStoresDispSizeMap) {
653  int64_t CurrDisp = DispSizePair.first;
654  unsigned CurrSize = DispSizePair.second;
655  while (DispSizeStack.size()) {
656  int64_t PrevDisp = DispSizeStack.back().first;
657  unsigned PrevSize = DispSizeStack.back().second;
658  if (CurrDisp + CurrSize > PrevDisp + PrevSize)
659  break;
660  DispSizeStack.pop_back();
661  }
662  DispSizeStack.push_back(DispSizePair);
663  }
664  BlockingStoresDispSizeMap.clear();
665  for (auto Disp : DispSizeStack)
666  BlockingStoresDispSizeMap.insert(Disp);
667 }
668 
669 bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) {
670  bool Changed = false;
671 
672  if (DisableX86AvoidStoreForwardBlocks || skipFunction(MF.getFunction()) ||
674  return false;
675 
676  MRI = &MF.getRegInfo();
677  assert(MRI->isSSA() && "Expected MIR to be in SSA form");
678  TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
679  TRI = MF.getSubtarget<X86Subtarget>().getRegisterInfo();
680  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
681  LLVM_DEBUG(dbgs() << "Start X86AvoidStoreForwardBlocks\n";);
682  // Look for a load then a store to XMM/YMM which look like a memcpy
683  findPotentiallylBlockedCopies(MF);
684 
685  for (auto LoadStoreInstPair : BlockedLoadsStoresPairs) {
686  MachineInstr *LoadInst = LoadStoreInstPair.first;
687  int64_t LdDispImm = getDispOperand(LoadInst).getImm();
688  DisplacementSizeMap BlockingStoresDispSizeMap;
689 
690  SmallVector<MachineInstr *, 2> PotentialBlockers =
691  findPotentialBlockers(LoadInst);
692  for (auto PBInst : PotentialBlockers) {
693  if (!isPotentialBlockingStoreInst(PBInst->getOpcode(),
694  LoadInst->getOpcode()) ||
695  !isRelevantAddressingMode(PBInst))
696  continue;
697  int64_t PBstDispImm = getDispOperand(PBInst).getImm();
698  assert(PBInst->hasOneMemOperand() && "Expected One Memory Operand");
699  unsigned PBstSize = (*PBInst->memoperands_begin())->getSize();
700  // This check doesn't cover all cases, but it will suffice for now.
701  // TODO: take branch probability into consideration, if the blocking
702  // store is in an unreached block, breaking the memcopy could lose
703  // performance.
704  if (hasSameBaseOpValue(LoadInst, PBInst) &&
705  isBlockingStore(LdDispImm, getRegSizeInBytes(LoadInst), PBstDispImm,
706  PBstSize))
707  updateBlockingStoresDispSizeMap(BlockingStoresDispSizeMap, PBstDispImm,
708  PBstSize);
709  }
710 
711  if (BlockingStoresDispSizeMap.empty())
712  continue;
713 
714  // We found a store forward block, break the memcpy's load and store
715  // into smaller copies such that each smaller store that was causing
716  // a store block would now be copied separately.
717  MachineInstr *StoreInst = LoadStoreInstPair.second;
718  LLVM_DEBUG(dbgs() << "Blocked load and store instructions: \n");
719  LLVM_DEBUG(LoadInst->dump());
720  LLVM_DEBUG(StoreInst->dump());
721  LLVM_DEBUG(dbgs() << "Replaced with:\n");
722  removeRedundantBlockingStores(BlockingStoresDispSizeMap);
723  breakBlockedCopies(LoadInst, StoreInst, BlockingStoresDispSizeMap);
724  updateKillStatus(LoadInst, StoreInst);
725  ForRemoval.push_back(LoadInst);
726  ForRemoval.push_back(StoreInst);
727  }
728  for (auto RemovedInst : ForRemoval) {
729  RemovedInst->eraseFromParent();
730  }
731  ForRemoval.clear();
732  BlockedLoadsStoresPairs.clear();
733  LLVM_DEBUG(dbgs() << "End X86AvoidStoreForwardBlocks\n";);
734 
735  return Changed;
736 }
bool is64Bit() const
Is this x86_64? (disregarding specific ABI / programming model)
Definition: X86Subtarget.h:542
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static bool isYMMLoadOpcode(unsigned Opcode)
instr_iterator instr_begin()
static void updateKillStatus(MachineInstr *LoadInst, MachineInstr *StoreInst)
This class represents lattice values for constants.
Definition: AllocatorList.h:23
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:385
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:178
The two locations do not alias at all.
Definition: AliasAnalysis.h:84
AddrNumOperands - Total number of operands in a memory reference.
Definition: X86BaseInfo.h:41
uint64_t getSize() const
Return the size in bytes of the memory reference.
unsigned const TargetRegisterInfo * TRI
An instruction for reading from memory.
Definition: Instructions.h:167
static void removeRedundantBlockingStores(DisplacementSizeMap &BlockingStoresDispSizeMap)
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
static cl::opt< bool > DisableX86AvoidStoreForwardBlocks("x86-disable-avoid-SFB", cl::Hidden, cl::desc("X86: Disable Store Forwarding Blocks fixup."), cl::init(false))
static void updateBlockingStoresDispSizeMap(DisplacementSizeMap &BlockingStoresDispSizeMap, int64_t DispImm, unsigned Size)
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
A description of a memory reference used in the backend.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
static bool hasSameBaseOpValue(MachineInstr *LoadInst, MachineInstr *StoreInst)
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i...
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:411
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
static bool isBlockingStore(int64_t LoadDispImm, unsigned LoadSize, int64_t StoreDispImm, unsigned StoreSize)
static bool isPotentialBlockingStoreInst(int Opcode, int LoadOpcode)
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:408
static const int MOV16SZ
static const int MOV32SZ
An instruction for storing to memory.
Definition: Instructions.h:320
COFF::MachineTypes Machine
Definition: COFFYAML.cpp:365
reverse_iterator rend()
reverse_iterator rbegin()
AliasResult
The possible results of an alias query.
Definition: AliasAnalysis.h:78
const Value * getValue() const
Return the base address of the memory access.
static cl::opt< unsigned > X86AvoidSFBInspectionLimit("x86-sfb-inspection-limit", cl::desc("X86: Number of instructions backward to " "inspect for store forwarding blocks."), cl::init(20), cl::Hidden)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
static int getAddrOffset(MachineInstr *MI)
unsigned const MachineRegisterInfo * MRI
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
AddrSegmentReg - The operand # of the segment in the memory operand.
Definition: X86BaseInfo.h:38
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static unsigned getYMMtoXMMStoreOpcode(unsigned StoreOpcode)
Represent the analysis usage information of a pass.
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
Definition: MachineInstr.h:551
static bool isPotentialBlockedMemCpyLd(unsigned Opcode)
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
unsigned getOperandBias(const MCInstrDesc &Desc)
getOperandBias - compute whether all of the def operands are repeated in the uses and therefore shoul...
Definition: X86BaseInfo.h:721
std::vector< MachineBasicBlock * >::iterator pred_iterator
static const int MOV128SZ
size_t size() const
Definition: SmallVector.h:52
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setIsKill(bool Val=true)
Representation for a specific memory location.
Iterator for intrusive lists based on ilist_node.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:536
static uint64_t add(uint64_t LeftOp, uint64_t RightOp)
Definition: FileCheck.cpp:243
static SmallVector< MachineInstr *, 2 > findPotentialBlockers(MachineInstr *LoadInst)
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
static MachineOperand & getBaseOperand(MachineInstr *MI)
int64_t getImm() const
IterT skipDebugInstructionsBackward(IterT It, IterT Begin)
Decrement It until it points to a non-debug instruction or to Begin and return the resulting iterator...
const Function & getFunction() const
Return the LLVM function that this machine code represents.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
static MachineOperand & getDispOperand(MachineInstr *MI)
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:256
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:64
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
static bool isRelevantAddressingMode(MachineInstr *MI)
INITIALIZE_PASS_BEGIN(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking", false, false) INITIALIZE_PASS_END(X86AvoidSFBPass
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isCall() const
Return true if the instruction is a call.
Definition: MCInstrDesc.h:272
static bool isXMMLoadOpcode(unsigned Opcode)
Machine code sinking
uint32_t Size
Definition: Profile.cpp:46
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static const int MOV64SZ
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
static const int MOV8SZ
FunctionPass * createX86AvoidStoreForwardingBlocks()
Return a pass that avoids creating store forward block issues in the hardware.
static unsigned getYMMtoXMMLoadOpcode(unsigned LoadOpcode)
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
static bool isPotentialBlockedMemCpyPair(int LdOpcode, int StOpcode)
Register getReg() const
getReg - Returns the register number.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
int getMemoryOperandNo(uint64_t TSFlags)
getMemoryOperandNo - The function returns the MCInst operand # for the first field of the memory oper...
Definition: X86BaseInfo.h:761