LLVM  10.0.0svn
X86ShuffleDecodeConstantPool.cpp
Go to the documentation of this file.
1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Define several functions to decode x86 specific shuffle semantics using
10 // constants from the constant pool.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "Utils/X86ShuffleDecode.h"
15 #include "llvm/ADT/APInt.h"
16 #include "llvm/IR/Constants.h"
17 
18 //===----------------------------------------------------------------------===//
19 // Vector Mask Decoding
20 //===----------------------------------------------------------------------===//
21 
22 namespace llvm {
23 
24 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
25  APInt &UndefElts,
26  SmallVectorImpl<uint64_t> &RawMask) {
27  // It is not an error for shuffle masks to not be a vector of
28  // MaskEltSizeInBits because the constant pool uniques constants by their
29  // bit representation.
30  // e.g. the following take up the same space in the constant pool:
31  // i128 -170141183420855150465331762880109871104
32  //
33  // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
34  //
35  // <4 x i32> <i32 -2147483648, i32 -2147483648,
36  // i32 -2147483648, i32 -2147483648>
37  Type *CstTy = C->getType();
38  if (!CstTy->isVectorTy())
39  return false;
40 
41  Type *CstEltTy = CstTy->getVectorElementType();
42  if (!CstEltTy->isIntegerTy())
43  return false;
44 
45  unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
46  unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
47  unsigned NumCstElts = CstTy->getVectorNumElements();
48 
49  assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
50  "Unaligned shuffle mask size");
51 
52  unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
53  UndefElts = APInt(NumMaskElts, 0);
54  RawMask.resize(NumMaskElts, 0);
55 
56  // Fast path - if the constants match the mask size then copy direct.
57  if (MaskEltSizeInBits == CstEltSizeInBits) {
58  assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
59  for (unsigned i = 0; i != NumMaskElts; ++i) {
60  Constant *COp = C->getAggregateElement(i);
61  if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
62  return false;
63 
64  if (isa<UndefValue>(COp)) {
65  UndefElts.setBit(i);
66  RawMask[i] = 0;
67  continue;
68  }
69 
70  auto *Elt = cast<ConstantInt>(COp);
71  RawMask[i] = Elt->getValue().getZExtValue();
72  }
73  return true;
74  }
75 
76  // Extract all the undef/constant element data and pack into single bitsets.
77  APInt UndefBits(CstSizeInBits, 0);
78  APInt MaskBits(CstSizeInBits, 0);
79  for (unsigned i = 0; i != NumCstElts; ++i) {
80  Constant *COp = C->getAggregateElement(i);
81  if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
82  return false;
83 
84  unsigned BitOffset = i * CstEltSizeInBits;
85 
86  if (isa<UndefValue>(COp)) {
87  UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
88  continue;
89  }
90 
91  MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
92  }
93 
94  // Now extract the undef/constant bit data into the raw shuffle masks.
95  for (unsigned i = 0; i != NumMaskElts; ++i) {
96  unsigned BitOffset = i * MaskEltSizeInBits;
97  APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
98 
99  // Only treat the element as UNDEF if all bits are UNDEF, otherwise
100  // treat it as zero.
101  if (EltUndef.isAllOnesValue()) {
102  UndefElts.setBit(i);
103  RawMask[i] = 0;
104  continue;
105  }
106 
107  APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
108  RawMask[i] = EltBits.getZExtValue();
109  }
110 
111  return true;
112 }
113 
114 void DecodePSHUFBMask(const Constant *C, unsigned Width,
115  SmallVectorImpl<int> &ShuffleMask) {
116  assert((Width == 128 || Width == 256 || Width == 512) &&
117  C->getType()->getPrimitiveSizeInBits() >= Width &&
118  "Unexpected vector size.");
119 
120  // The shuffle mask requires a byte vector.
121  APInt UndefElts;
123  if (!extractConstantMask(C, 8, UndefElts, RawMask))
124  return;
125 
126  unsigned NumElts = Width / 8;
127  assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
128  "Unexpected number of vector elements.");
129 
130  for (unsigned i = 0; i != NumElts; ++i) {
131  if (UndefElts[i]) {
132  ShuffleMask.push_back(SM_SentinelUndef);
133  continue;
134  }
135 
136  uint64_t Element = RawMask[i];
137  // If the high bit (7) of the byte is set, the element is zeroed.
138  if (Element & (1 << 7))
139  ShuffleMask.push_back(SM_SentinelZero);
140  else {
141  // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
142  // lane of the vector we're inside.
143  unsigned Base = i & ~0xf;
144 
145  // Only the least significant 4 bits of the byte are used.
146  int Index = Base + (Element & 0xf);
147  ShuffleMask.push_back(Index);
148  }
149  }
150 }
151 
152 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width,
153  SmallVectorImpl<int> &ShuffleMask) {
154  assert((Width == 128 || Width == 256 || Width == 512) &&
155  C->getType()->getPrimitiveSizeInBits() >= Width &&
156  "Unexpected vector size.");
157  assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
158 
159  // The shuffle mask requires elements the same size as the target.
160  APInt UndefElts;
162  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
163  return;
164 
165  unsigned NumElts = Width / ElSize;
166  unsigned NumEltsPerLane = 128 / ElSize;
167  assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
168  "Unexpected number of vector elements.");
169 
170  for (unsigned i = 0; i != NumElts; ++i) {
171  if (UndefElts[i]) {
172  ShuffleMask.push_back(SM_SentinelUndef);
173  continue;
174  }
175 
176  int Index = i & ~(NumEltsPerLane - 1);
177  uint64_t Element = RawMask[i];
178  if (ElSize == 64)
179  Index += (Element >> 1) & 0x1;
180  else
181  Index += Element & 0x3;
182 
183  ShuffleMask.push_back(Index);
184  }
185 }
186 
187 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
188  unsigned Width,
189  SmallVectorImpl<int> &ShuffleMask) {
190  Type *MaskTy = C->getType();
191  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
192  (void)MaskTySize;
193  assert((MaskTySize == 128 || MaskTySize == 256) &&
194  Width >= MaskTySize && "Unexpected vector size.");
195 
196  // The shuffle mask requires elements the same size as the target.
197  APInt UndefElts;
198  SmallVector<uint64_t, 8> RawMask;
199  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
200  return;
201 
202  unsigned NumElts = Width / ElSize;
203  unsigned NumEltsPerLane = 128 / ElSize;
204  assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
205  "Unexpected number of vector elements.");
206 
207  for (unsigned i = 0; i != NumElts; ++i) {
208  if (UndefElts[i]) {
209  ShuffleMask.push_back(SM_SentinelUndef);
210  continue;
211  }
212 
213  // VPERMIL2 Operation.
214  // Bits[3] - Match Bit.
215  // Bits[2:1] - (Per Lane) PD Shuffle Mask.
216  // Bits[2:0] - (Per Lane) PS Shuffle Mask.
217  uint64_t Selector = RawMask[i];
218  unsigned MatchBit = (Selector >> 3) & 0x1;
219 
220  // M2Z[0:1] MatchBit
221  // 0Xb X Source selected by Selector index.
222  // 10b 0 Source selected by Selector index.
223  // 10b 1 Zero.
224  // 11b 0 Zero.
225  // 11b 1 Source selected by Selector index.
226  if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
227  ShuffleMask.push_back(SM_SentinelZero);
228  continue;
229  }
230 
231  int Index = i & ~(NumEltsPerLane - 1);
232  if (ElSize == 64)
233  Index += (Selector >> 1) & 0x1;
234  else
235  Index += Selector & 0x3;
236 
237  int Src = (Selector >> 2) & 0x1;
238  Index += Src * NumElts;
239  ShuffleMask.push_back(Index);
240  }
241 }
242 
243 void DecodeVPPERMMask(const Constant *C, unsigned Width,
244  SmallVectorImpl<int> &ShuffleMask) {
245  Type *MaskTy = C->getType();
246  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
247  (void)MaskTySize;
248  assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size.");
249 
250  // The shuffle mask requires a byte vector.
251  APInt UndefElts;
253  if (!extractConstantMask(C, 8, UndefElts, RawMask))
254  return;
255 
256  unsigned NumElts = Width / 8;
257  assert(NumElts == 16 && "Unexpected number of vector elements.");
258 
259  for (unsigned i = 0; i != NumElts; ++i) {
260  if (UndefElts[i]) {
261  ShuffleMask.push_back(SM_SentinelUndef);
262  continue;
263  }
264 
265  // VPPERM Operation
266  // Bits[4:0] - Byte Index (0 - 31)
267  // Bits[7:5] - Permute Operation
268  //
269  // Permute Operation:
270  // 0 - Source byte (no logical operation).
271  // 1 - Invert source byte.
272  // 2 - Bit reverse of source byte.
273  // 3 - Bit reverse of inverted source byte.
274  // 4 - 00h (zero - fill).
275  // 5 - FFh (ones - fill).
276  // 6 - Most significant bit of source byte replicated in all bit positions.
277  // 7 - Invert most significant bit of source byte and replicate in all bit
278  // positions.
279  uint64_t Element = RawMask[i];
280  uint64_t Index = Element & 0x1F;
281  uint64_t PermuteOp = (Element >> 5) & 0x7;
282 
283  if (PermuteOp == 4) {
284  ShuffleMask.push_back(SM_SentinelZero);
285  continue;
286  }
287  if (PermuteOp != 0) {
288  ShuffleMask.clear();
289  return;
290  }
291  ShuffleMask.push_back((int)Index);
292  }
293 }
294 
295 void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
296  SmallVectorImpl<int> &ShuffleMask) {
297  assert((Width == 128 || Width == 256 || Width == 512) &&
298  C->getType()->getPrimitiveSizeInBits() >= Width &&
299  "Unexpected vector size.");
300  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
301  "Unexpected vector element size.");
302 
303  // The shuffle mask requires elements the same size as the target.
304  APInt UndefElts;
306  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
307  return;
308 
309  unsigned NumElts = Width / ElSize;
310 
311  for (unsigned i = 0; i != NumElts; ++i) {
312  if (UndefElts[i]) {
313  ShuffleMask.push_back(SM_SentinelUndef);
314  continue;
315  }
316  int Index = RawMask[i] & (NumElts - 1);
317  ShuffleMask.push_back(Index);
318  }
319 }
320 
321 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
322  SmallVectorImpl<int> &ShuffleMask) {
323  assert((Width == 128 || Width == 256 || Width == 512) &&
324  C->getType()->getPrimitiveSizeInBits() >= Width &&
325  "Unexpected vector size.");
326  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
327  "Unexpected vector element size.");
328 
329  // The shuffle mask requires elements the same size as the target.
330  APInt UndefElts;
332  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
333  return;
334 
335  unsigned NumElts = Width / ElSize;
336 
337  for (unsigned i = 0; i != NumElts; ++i) {
338  if (UndefElts[i]) {
339  ShuffleMask.push_back(SM_SentinelUndef);
340  continue;
341  }
342  int Index = RawMask[i] & (NumElts*2 - 1);
343  ShuffleMask.push_back(Index);
344  }
345 }
346 } // llvm namespace
Type * getVectorElementType() const
Definition: Type.h:375
uint64_t CallInst * C
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1571
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition: APInt.h:1417
This class represents lattice values for constants.
Definition: AllocatorList.h:23
void DecodeVPPERMMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPPERM mask from a raw array of constants such as from BUILD_VECTOR.
void DecodeVPERMV3Mask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERMT2 W/D/Q/PS/PD mask from a raw array of constants.
void push_back(const T &Elt)
Definition: SmallVector.h:211
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:230
void setBit(unsigned BitPosition)
Set a given bit to 1.
Definition: APInt.h:1402
void DecodeVPERMILPMask(unsigned NumElts, unsigned ScalarBits, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERMILPD/VPERMILPS variable mask from a raw array of constants.
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:197
This file implements a class to represent arbitrary precision integral constant values and operations...
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:246
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition: APInt.cpp:431
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:359
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition: APInt.cpp:345
bool isAllOnesValue() const
Determine if all bits are set.
Definition: APInt.h:395
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:115
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
This is an important base class in LLVM.
Definition: Constant.h:41
This file contains the declarations for the subclasses of Constant, which represent the different fla...
void DecodeVPERMIL2PMask(unsigned NumElts, unsigned ScalarBits, unsigned M2Z, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERMIL2PD/VPERMIL2PS variable mask from a raw array of constants.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type...
Definition: Type.cpp:134
unsigned getVectorNumElements() const
Definition: DerivedTypes.h:566
Class for arbitrary precision integers.
Definition: APInt.h:69
static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits, APInt &UndefElts, SmallVectorImpl< uint64_t > &RawMask)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void DecodeVPERMVMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERM W/D/Q/PS/PD mask from a raw array of constants.
void DecodePSHUFBMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a PSHUFB mask from a raw array of constants such as from BUILD_VECTOR.
void resize(size_type N)
Definition: SmallVector.h:344