LLVM  4.0.0
InterleavedAccessPass.cpp
Go to the documentation of this file.
1 //===--------------------- InterleavedAccessPass.cpp ----------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the Interleaved Access pass, which identifies
11 // interleaved memory accesses and transforms them into target specific
12 // intrinsics.
13 //
14 // An interleaved load reads data from memory into several vectors, with
15 // DE-interleaving the data on a factor. An interleaved store writes several
16 // vectors to memory with RE-interleaving the data on a factor.
17 //
18 // As interleaved accesses are difficult to identified in CodeGen (mainly
19 // because the VECTOR_SHUFFLE DAG node is quite different from the shufflevector
20 // IR), we identify and transform them to intrinsics in this pass so the
21 // intrinsics can be easily matched into target specific instructions later in
22 // CodeGen.
23 //
24 // E.g. An interleaved load (Factor = 2):
25 // %wide.vec = load <8 x i32>, <8 x i32>* %ptr
26 // %v0 = shuffle <8 x i32> %wide.vec, <8 x i32> undef, <0, 2, 4, 6>
27 // %v1 = shuffle <8 x i32> %wide.vec, <8 x i32> undef, <1, 3, 5, 7>
28 //
29 // It could be transformed into a ld2 intrinsic in AArch64 backend or a vld2
30 // intrinsic in ARM backend.
31 //
32 // In X86, this can be further optimized into a set of target
33 // specific loads followed by an optimized sequence of shuffles.
34 //
35 // E.g. An interleaved store (Factor = 3):
36 // %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
37 // <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
38 // store <12 x i32> %i.vec, <12 x i32>* %ptr
39 //
40 // It could be transformed into a st3 intrinsic in AArch64 backend or a vst3
41 // intrinsic in ARM backend.
42 //
43 // Similarly, a set of interleaved stores can be transformed into an optimized
44 // sequence of shuffles followed by a set of target specific stores for X86.
45 //===----------------------------------------------------------------------===//
46 
47 #include "llvm/CodeGen/Passes.h"
48 #include "llvm/IR/Dominators.h"
49 #include "llvm/IR/InstIterator.h"
50 #include "llvm/Support/Debug.h"
55 
56 using namespace llvm;
57 
58 #define DEBUG_TYPE "interleaved-access"
59 
61  "lower-interleaved-accesses",
62  cl::desc("Enable lowering interleaved accesses to intrinsics"),
63  cl::init(true), cl::Hidden);
64 
65 namespace {
66 
67 class InterleavedAccess : public FunctionPass {
68 
69 public:
70  static char ID;
71  InterleavedAccess(const TargetMachine *TM = nullptr)
72  : FunctionPass(ID), DT(nullptr), TM(TM), TLI(nullptr) {
74  }
75 
76  StringRef getPassName() const override { return "Interleaved Access Pass"; }
77 
78  bool runOnFunction(Function &F) override;
79 
80  void getAnalysisUsage(AnalysisUsage &AU) const override {
83  }
84 
85 private:
86  DominatorTree *DT;
87  const TargetMachine *TM;
88  const TargetLowering *TLI;
89 
90  /// The maximum supported interleave factor.
91  unsigned MaxFactor;
92 
93  /// \brief Transform an interleaved load into target specific intrinsics.
94  bool lowerInterleavedLoad(LoadInst *LI,
96 
97  /// \brief Transform an interleaved store into target specific intrinsics.
98  bool lowerInterleavedStore(StoreInst *SI,
100 
101  /// \brief Returns true if the uses of an interleaved load by the
102  /// extractelement instructions in \p Extracts can be replaced by uses of the
103  /// shufflevector instructions in \p Shuffles instead. If so, the necessary
104  /// replacements are also performed.
105  bool tryReplaceExtracts(ArrayRef<ExtractElementInst *> Extracts,
107 };
108 } // end anonymous namespace.
109 
110 char InterleavedAccess::ID = 0;
112  InterleavedAccess, "interleaved-access",
113  "Lower interleaved memory accesses to target specific intrinsics", false,
114  false)
117  InterleavedAccess, "interleaved-access",
118  "Lower interleaved memory accesses to target specific intrinsics", false,
119  false)
120 
122  return new InterleavedAccess(TM);
123 }
124 
125 /// \brief Check if the mask is a DE-interleave mask of the given factor
126 /// \p Factor like:
127 /// <Index, Index+Factor, ..., Index+(NumElts-1)*Factor>
128 static bool isDeInterleaveMaskOfFactor(ArrayRef<int> Mask, unsigned Factor,
129  unsigned &Index) {
130  // Check all potential start indices from 0 to (Factor - 1).
131  for (Index = 0; Index < Factor; Index++) {
132  unsigned i = 0;
133 
134  // Check that elements are in ascending order by Factor. Ignore undef
135  // elements.
136  for (; i < Mask.size(); i++)
137  if (Mask[i] >= 0 && static_cast<unsigned>(Mask[i]) != Index + i * Factor)
138  break;
139 
140  if (i == Mask.size())
141  return true;
142  }
143 
144  return false;
145 }
146 
147 /// \brief Check if the mask is a DE-interleave mask for an interleaved load.
148 ///
149 /// E.g. DE-interleave masks (Factor = 2) could be:
150 /// <0, 2, 4, 6> (mask of index 0 to extract even elements)
151 /// <1, 3, 5, 7> (mask of index 1 to extract odd elements)
152 static bool isDeInterleaveMask(ArrayRef<int> Mask, unsigned &Factor,
153  unsigned &Index, unsigned MaxFactor) {
154  if (Mask.size() < 2)
155  return false;
156 
157  // Check potential Factors.
158  for (Factor = 2; Factor <= MaxFactor; Factor++)
159  if (isDeInterleaveMaskOfFactor(Mask, Factor, Index))
160  return true;
161 
162  return false;
163 }
164 
165 /// \brief Check if the mask can be used in an interleaved store.
166 //
167 /// It checks for a more general pattern than the RE-interleave mask.
168 /// I.e. <x, y, ... z, x+1, y+1, ...z+1, x+2, y+2, ...z+2, ...>
169 /// E.g. For a Factor of 2 (LaneLen=4): <4, 32, 5, 33, 6, 34, 7, 35>
170 /// E.g. For a Factor of 3 (LaneLen=4): <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
171 /// E.g. For a Factor of 4 (LaneLen=2): <8, 2, 12, 4, 9, 3, 13, 5>
172 ///
173 /// The particular case of an RE-interleave mask is:
174 /// I.e. <0, LaneLen, ... , LaneLen*(Factor - 1), 1, LaneLen + 1, ...>
175 /// E.g. For a Factor of 2 (LaneLen=4): <0, 4, 1, 5, 2, 6, 3, 7>
176 static bool isReInterleaveMask(ArrayRef<int> Mask, unsigned &Factor,
177  unsigned MaxFactor, unsigned OpNumElts) {
178  unsigned NumElts = Mask.size();
179  if (NumElts < 4)
180  return false;
181 
182  // Check potential Factors.
183  for (Factor = 2; Factor <= MaxFactor; Factor++) {
184  if (NumElts % Factor)
185  continue;
186 
187  unsigned LaneLen = NumElts / Factor;
188  if (!isPowerOf2_32(LaneLen))
189  continue;
190 
191  // Check whether each element matches the general interleaved rule.
192  // Ignore undef elements, as long as the defined elements match the rule.
193  // Outer loop processes all factors (x, y, z in the above example)
194  unsigned I = 0, J;
195  for (; I < Factor; I++) {
196  unsigned SavedLaneValue;
197  unsigned SavedNoUndefs = 0;
198 
199  // Inner loop processes consecutive accesses (x, x+1... in the example)
200  for (J = 0; J < LaneLen - 1; J++) {
201  // Lane computes x's position in the Mask
202  unsigned Lane = J * Factor + I;
203  unsigned NextLane = Lane + Factor;
204  int LaneValue = Mask[Lane];
205  int NextLaneValue = Mask[NextLane];
206 
207  // If both are defined, values must be sequential
208  if (LaneValue >= 0 && NextLaneValue >= 0 &&
209  LaneValue + 1 != NextLaneValue)
210  break;
211 
212  // If the next value is undef, save the current one as reference
213  if (LaneValue >= 0 && NextLaneValue < 0) {
214  SavedLaneValue = LaneValue;
215  SavedNoUndefs = 1;
216  }
217 
218  // Undefs are allowed, but defined elements must still be consecutive:
219  // i.e.: x,..., undef,..., x + 2,..., undef,..., undef,..., x + 5, ....
220  // Verify this by storing the last non-undef followed by an undef
221  // Check that following non-undef masks are incremented with the
222  // corresponding distance.
223  if (SavedNoUndefs > 0 && LaneValue < 0) {
224  SavedNoUndefs++;
225  if (NextLaneValue >= 0 &&
226  SavedLaneValue + SavedNoUndefs != (unsigned)NextLaneValue)
227  break;
228  }
229  }
230 
231  if (J < LaneLen - 1)
232  break;
233 
234  int StartMask = 0;
235  if (Mask[I] >= 0) {
236  // Check that the start of the I range (J=0) is greater than 0
237  StartMask = Mask[I];
238  } else if (Mask[(LaneLen - 1) * Factor + I] >= 0) {
239  // StartMask defined by the last value in lane
240  StartMask = Mask[(LaneLen - 1) * Factor + I] - J;
241  } else if (SavedNoUndefs > 0) {
242  // StartMask defined by some non-zero value in the j loop
243  StartMask = SavedLaneValue - (LaneLen - 1 - SavedNoUndefs);
244  }
245  // else StartMask remains set to 0, i.e. all elements are undefs
246 
247  if (StartMask < 0)
248  break;
249  // We must stay within the vectors; This case can happen with undefs.
250  if (StartMask + LaneLen > OpNumElts*2)
251  break;
252  }
253 
254  // Found an interleaved mask of current factor.
255  if (I == Factor)
256  return true;
257  }
258 
259  return false;
260 }
261 
262 bool InterleavedAccess::lowerInterleavedLoad(
263  LoadInst *LI, SmallVector<Instruction *, 32> &DeadInsts) {
264  if (!LI->isSimple())
265  return false;
266 
269 
270  // Check if all users of this load are shufflevectors. If we encounter any
271  // users that are extractelement instructions, we save them to later check if
272  // they can be modifed to extract from one of the shufflevectors instead of
273  // the load.
274  for (auto UI = LI->user_begin(), E = LI->user_end(); UI != E; UI++) {
275  auto *Extract = dyn_cast<ExtractElementInst>(*UI);
276  if (Extract && isa<ConstantInt>(Extract->getIndexOperand())) {
277  Extracts.push_back(Extract);
278  continue;
279  }
281  if (!SVI || !isa<UndefValue>(SVI->getOperand(1)))
282  return false;
283 
284  Shuffles.push_back(SVI);
285  }
286 
287  if (Shuffles.empty())
288  return false;
289 
290  unsigned Factor, Index;
291 
292  // Check if the first shufflevector is DE-interleave shuffle.
293  if (!isDeInterleaveMask(Shuffles[0]->getShuffleMask(), Factor, Index,
294  MaxFactor))
295  return false;
296 
297  // Holds the corresponding index for each DE-interleave shuffle.
298  SmallVector<unsigned, 4> Indices;
299  Indices.push_back(Index);
300 
301  Type *VecTy = Shuffles[0]->getType();
302 
303  // Check if other shufflevectors are also DE-interleaved of the same type
304  // and factor as the first shufflevector.
305  for (unsigned i = 1; i < Shuffles.size(); i++) {
306  if (Shuffles[i]->getType() != VecTy)
307  return false;
308 
309  if (!isDeInterleaveMaskOfFactor(Shuffles[i]->getShuffleMask(), Factor,
310  Index))
311  return false;
312 
313  Indices.push_back(Index);
314  }
315 
316  // Try and modify users of the load that are extractelement instructions to
317  // use the shufflevector instructions instead of the load.
318  if (!tryReplaceExtracts(Extracts, Shuffles))
319  return false;
320 
321  DEBUG(dbgs() << "IA: Found an interleaved load: " << *LI << "\n");
322 
323  // Try to create target specific intrinsics to replace the load and shuffles.
324  if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor))
325  return false;
326 
327  for (auto SVI : Shuffles)
328  DeadInsts.push_back(SVI);
329 
330  DeadInsts.push_back(LI);
331  return true;
332 }
333 
334 bool InterleavedAccess::tryReplaceExtracts(
337 
338  // If there aren't any extractelement instructions to modify, there's nothing
339  // to do.
340  if (Extracts.empty())
341  return true;
342 
343  // Maps extractelement instructions to vector-index pairs. The extractlement
344  // instructions will be modified to use the new vector and index operands.
346 
347  for (auto *Extract : Extracts) {
348 
349  // The vector index that is extracted.
350  auto *IndexOperand = cast<ConstantInt>(Extract->getIndexOperand());
351  auto Index = IndexOperand->getSExtValue();
352 
353  // Look for a suitable shufflevector instruction. The goal is to modify the
354  // extractelement instruction (which uses an interleaved load) to use one
355  // of the shufflevector instructions instead of the load.
356  for (auto *Shuffle : Shuffles) {
357 
358  // If the shufflevector instruction doesn't dominate the extract, we
359  // can't create a use of it.
360  if (!DT->dominates(Shuffle, Extract))
361  continue;
362 
363  // Inspect the indices of the shufflevector instruction. If the shuffle
364  // selects the same index that is extracted, we can modify the
365  // extractelement instruction.
366  SmallVector<int, 4> Indices;
367  Shuffle->getShuffleMask(Indices);
368  for (unsigned I = 0; I < Indices.size(); ++I)
369  if (Indices[I] == Index) {
370  assert(Extract->getOperand(0) == Shuffle->getOperand(0) &&
371  "Vector operations do not match");
372  ReplacementMap[Extract] = std::make_pair(Shuffle, I);
373  break;
374  }
375 
376  // If we found a suitable shufflevector instruction, stop looking.
377  if (ReplacementMap.count(Extract))
378  break;
379  }
380 
381  // If we did not find a suitable shufflevector instruction, the
382  // extractelement instruction cannot be modified, so we must give up.
383  if (!ReplacementMap.count(Extract))
384  return false;
385  }
386 
387  // Finally, perform the replacements.
388  IRBuilder<> Builder(Extracts[0]->getContext());
389  for (auto &Replacement : ReplacementMap) {
390  auto *Extract = Replacement.first;
391  auto *Vector = Replacement.second.first;
392  auto Index = Replacement.second.second;
393  Builder.SetInsertPoint(Extract);
394  Extract->replaceAllUsesWith(Builder.CreateExtractElement(Vector, Index));
395  Extract->eraseFromParent();
396  }
397 
398  return true;
399 }
400 
401 bool InterleavedAccess::lowerInterleavedStore(
402  StoreInst *SI, SmallVector<Instruction *, 32> &DeadInsts) {
403  if (!SI->isSimple())
404  return false;
405 
407  if (!SVI || !SVI->hasOneUse())
408  return false;
409 
410  // Check if the shufflevector is RE-interleave shuffle.
411  unsigned Factor;
412  unsigned OpNumElts = SVI->getOperand(0)->getType()->getVectorNumElements();
413  if (!isReInterleaveMask(SVI->getShuffleMask(), Factor, MaxFactor, OpNumElts))
414  return false;
415 
416  DEBUG(dbgs() << "IA: Found an interleaved store: " << *SI << "\n");
417 
418  // Try to create target specific intrinsics to replace the store and shuffle.
419  if (!TLI->lowerInterleavedStore(SI, SVI, Factor))
420  return false;
421 
422  // Already have a new target specific interleaved store. Erase the old store.
423  DeadInsts.push_back(SI);
424  DeadInsts.push_back(SVI);
425  return true;
426 }
427 
428 bool InterleavedAccess::runOnFunction(Function &F) {
429  if (!TM || !LowerInterleavedAccesses)
430  return false;
431 
432  DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n");
433 
434  DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
435  TLI = TM->getSubtargetImpl(F)->getTargetLowering();
436  MaxFactor = TLI->getMaxSupportedInterleaveFactor();
437 
438  // Holds dead instructions that will be erased later.
440  bool Changed = false;
441 
442  for (auto &I : instructions(F)) {
443  if (LoadInst *LI = dyn_cast<LoadInst>(&I))
444  Changed |= lowerInterleavedLoad(LI, DeadInsts);
445 
446  if (StoreInst *SI = dyn_cast<StoreInst>(&I))
447  Changed |= lowerInterleavedStore(SI, DeadInsts);
448  }
449 
450  for (auto I : DeadInsts)
451  I->eraseFromParent();
452 
453  return Changed;
454 }
Value * getValueOperand()
Definition: Instructions.h:391
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
size_t i
bool isSimple() const
Definition: Instructions.h:384
static cl::opt< bool > LowerInterleavedAccesses("lower-interleaved-accesses", cl::desc("Enable lowering interleaved accesses to intrinsics"), cl::init(true), cl::Hidden)
interleaved access
This instruction constructs a fixed permutation of two input vectors.
An instruction for reading from memory.
Definition: Instructions.h:164
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
Definition: Function.cpp:905
bool isSimple() const
Definition: Instructions.h:263
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:588
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
#define F(x, y, z)
Definition: MD5.cpp:51
interleaved Lower interleaved memory accesses to target specific false
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
An instruction for storing to memory.
Definition: Instructions.h:300
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:96
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:141
static bool isReInterleaveMask(ArrayRef< int > Mask, unsigned &Factor, unsigned MaxFactor, unsigned OpNumElts)
Check if the mask can be used in an interleaved store.
FunctionPass * createInterleavedAccessPass(const TargetMachine *TM)
InterleavedAccess Pass - This pass identifies and matches interleaved memory accesses to target speci...
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:399
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
#define INITIALIZE_TM_PASS_END(passName, arg, name, cfg, analysis)
Target machine pass initializer for passes with dependencies.
Represent the analysis usage information of a pass.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
Value * getOperand(unsigned i) const
Definition: User.h:145
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:136
void initializeInterleavedAccessPass(PassRegistry &)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:122
unsigned getVectorNumElements() const
Definition: DerivedTypes.h:438
static bool isDeInterleaveMask(ArrayRef< int > Mask, unsigned &Factor, unsigned &Index, unsigned MaxFactor)
Check if the mask is a DE-interleave mask for an interleaved load.
interleaved Lower interleaved memory accesses to target specific intrinsics
#define I(x, y, z)
Definition: MD5.cpp:54
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:383
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
This instruction extracts a single (scalar) element from a VectorType value.
INITIALIZE_TM_PASS_BEGIN(InterleavedAccess,"interleaved-access","Lower interleaved memory accesses to target specific intrinsics", false, false) INITIALIZE_TM_PASS_END(InterleavedAccess
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
Definition: Value.h:346
aarch64 promote const
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:81
#define DEBUG(X)
Definition: Debug.h:100
Primary interface to the complete machine description for the target machine.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
inst_range instructions(Function *F)
Definition: InstIterator.h:132
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:217
static bool isDeInterleaveMaskOfFactor(ArrayRef< int > Mask, unsigned Factor, unsigned &Index)
Check if the mask is a DE-interleave mask of the given factor Factor like: <Index, Index+Factor, ..., Index+(NumElts-1)*Factor>
static void getShuffleMask(Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
This file describes how to lower LLVM code to machine code.
user_iterator user_end()
Definition: Value.h:354