LLVM  16.0.0git
VNCoercion.cpp
Go to the documentation of this file.
4 #include "llvm/IR/IRBuilder.h"
6 #include "llvm/Support/Debug.h"
7 
8 #define DEBUG_TYPE "vncoerce"
9 
10 namespace llvm {
11 namespace VNCoercion {
12 
14  return Ty->isStructTy() || Ty->isArrayTy() || isa<ScalableVectorType>(Ty);
15 }
16 
17 /// Return true if coerceAvailableValueToLoadType will succeed.
18 bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy,
19  const DataLayout &DL) {
20  Type *StoredTy = StoredVal->getType();
21 
22  if (StoredTy == LoadTy)
23  return true;
24 
25  // If the loaded/stored value is a first class array/struct, or scalable type,
26  // don't try to transform them. We need to be able to bitcast to integer.
29  return false;
30 
31  uint64_t StoreSize = DL.getTypeSizeInBits(StoredTy).getFixedSize();
32 
33  // The store size must be byte-aligned to support future type casts.
34  if (llvm::alignTo(StoreSize, 8) != StoreSize)
35  return false;
36 
37  // The store has to be at least as big as the load.
38  if (StoreSize < DL.getTypeSizeInBits(LoadTy).getFixedSize())
39  return false;
40 
41  bool StoredNI = DL.isNonIntegralPointerType(StoredTy->getScalarType());
42  bool LoadNI = DL.isNonIntegralPointerType(LoadTy->getScalarType());
43  // Don't coerce non-integral pointers to integers or vice versa.
44  if (StoredNI != LoadNI) {
45  // As a special case, allow coercion of memset used to initialize
46  // an array w/null. Despite non-integral pointers not generally having a
47  // specific bit pattern, we do assume null is zero.
48  if (auto *CI = dyn_cast<Constant>(StoredVal))
49  return CI->isNullValue();
50  return false;
51  } else if (StoredNI && LoadNI &&
52  StoredTy->getPointerAddressSpace() !=
53  LoadTy->getPointerAddressSpace()) {
54  return false;
55  }
56 
57 
58  // The implementation below uses inttoptr for vectors of unequal size; we
59  // can't allow this for non integral pointers. We could teach it to extract
60  // exact subvectors if desired.
61  if (StoredNI && StoreSize != DL.getTypeSizeInBits(LoadTy).getFixedSize())
62  return false;
63 
64  return true;
65 }
66 
67 /// If we saw a store of a value to memory, and
68 /// then a load from a must-aliased pointer of a different type, try to coerce
69 /// the stored value. LoadedTy is the type of the load we want to replace.
70 /// IRB is IRBuilder used to insert new instructions.
71 ///
72 /// If we can't do it, return null.
74  IRBuilderBase &Helper,
75  const DataLayout &DL) {
76  assert(canCoerceMustAliasedValueToLoad(StoredVal, LoadedTy, DL) &&
77  "precondition violation - materialization can't fail");
78  if (auto *C = dyn_cast<Constant>(StoredVal))
79  StoredVal = ConstantFoldConstant(C, DL);
80 
81  // If this is already the right type, just return it.
82  Type *StoredValTy = StoredVal->getType();
83 
84  uint64_t StoredValSize = DL.getTypeSizeInBits(StoredValTy).getFixedSize();
85  uint64_t LoadedValSize = DL.getTypeSizeInBits(LoadedTy).getFixedSize();
86 
87  // If the store and reload are the same size, we can always reuse it.
88  if (StoredValSize == LoadedValSize) {
89  // Pointer to Pointer -> use bitcast.
90  if (StoredValTy->isPtrOrPtrVectorTy() && LoadedTy->isPtrOrPtrVectorTy()) {
91  StoredVal = Helper.CreateBitCast(StoredVal, LoadedTy);
92  } else {
93  // Convert source pointers to integers, which can be bitcast.
94  if (StoredValTy->isPtrOrPtrVectorTy()) {
95  StoredValTy = DL.getIntPtrType(StoredValTy);
96  StoredVal = Helper.CreatePtrToInt(StoredVal, StoredValTy);
97  }
98 
99  Type *TypeToCastTo = LoadedTy;
100  if (TypeToCastTo->isPtrOrPtrVectorTy())
101  TypeToCastTo = DL.getIntPtrType(TypeToCastTo);
102 
103  if (StoredValTy != TypeToCastTo)
104  StoredVal = Helper.CreateBitCast(StoredVal, TypeToCastTo);
105 
106  // Cast to pointer if the load needs a pointer type.
107  if (LoadedTy->isPtrOrPtrVectorTy())
108  StoredVal = Helper.CreateIntToPtr(StoredVal, LoadedTy);
109  }
110 
111  if (auto *C = dyn_cast<ConstantExpr>(StoredVal))
112  StoredVal = ConstantFoldConstant(C, DL);
113 
114  return StoredVal;
115  }
116  // If the loaded value is smaller than the available value, then we can
117  // extract out a piece from it. If the available value is too small, then we
118  // can't do anything.
119  assert(StoredValSize >= LoadedValSize &&
120  "canCoerceMustAliasedValueToLoad fail");
121 
122  // Convert source pointers to integers, which can be manipulated.
123  if (StoredValTy->isPtrOrPtrVectorTy()) {
124  StoredValTy = DL.getIntPtrType(StoredValTy);
125  StoredVal = Helper.CreatePtrToInt(StoredVal, StoredValTy);
126  }
127 
128  // Convert vectors and fp to integer, which can be manipulated.
129  if (!StoredValTy->isIntegerTy()) {
130  StoredValTy = IntegerType::get(StoredValTy->getContext(), StoredValSize);
131  StoredVal = Helper.CreateBitCast(StoredVal, StoredValTy);
132  }
133 
134  // If this is a big-endian system, we need to shift the value down to the low
135  // bits so that a truncate will work.
136  if (DL.isBigEndian()) {
137  uint64_t ShiftAmt = DL.getTypeStoreSizeInBits(StoredValTy).getFixedSize() -
138  DL.getTypeStoreSizeInBits(LoadedTy).getFixedSize();
139  StoredVal = Helper.CreateLShr(
140  StoredVal, ConstantInt::get(StoredVal->getType(), ShiftAmt));
141  }
142 
143  // Truncate the integer to the right size now.
144  Type *NewIntTy = IntegerType::get(StoredValTy->getContext(), LoadedValSize);
145  StoredVal = Helper.CreateTruncOrBitCast(StoredVal, NewIntTy);
146 
147  if (LoadedTy != NewIntTy) {
148  // If the result is a pointer, inttoptr.
149  if (LoadedTy->isPtrOrPtrVectorTy())
150  StoredVal = Helper.CreateIntToPtr(StoredVal, LoadedTy);
151  else
152  // Otherwise, bitcast.
153  StoredVal = Helper.CreateBitCast(StoredVal, LoadedTy);
154  }
155 
156  if (auto *C = dyn_cast<Constant>(StoredVal))
157  StoredVal = ConstantFoldConstant(C, DL);
158 
159  return StoredVal;
160 }
161 
162 /// This function is called when we have a memdep query of a load that ends up
163 /// being a clobbering memory write (store, memset, memcpy, memmove). This
164 /// means that the write *may* provide bits used by the load but we can't be
165 /// sure because the pointers don't must-alias.
166 ///
167 /// Check this case to see if there is anything more we can do before we give
168 /// up. This returns -1 if we have to give up, or a byte number in the stored
169 /// value of the piece that feeds the load.
170 static int analyzeLoadFromClobberingWrite(Type *LoadTy, Value *LoadPtr,
171  Value *WritePtr,
172  uint64_t WriteSizeInBits,
173  const DataLayout &DL) {
174  // If the loaded/stored value is a first class array/struct, or scalable type,
175  // don't try to transform them. We need to be able to bitcast to integer.
177  return -1;
178 
179  int64_t StoreOffset = 0, LoadOffset = 0;
180  Value *StoreBase =
181  GetPointerBaseWithConstantOffset(WritePtr, StoreOffset, DL);
182  Value *LoadBase = GetPointerBaseWithConstantOffset(LoadPtr, LoadOffset, DL);
183  if (StoreBase != LoadBase)
184  return -1;
185 
186  uint64_t LoadSize = DL.getTypeSizeInBits(LoadTy).getFixedSize();
187 
188  if ((WriteSizeInBits & 7) | (LoadSize & 7))
189  return -1;
190  uint64_t StoreSize = WriteSizeInBits / 8; // Convert to bytes.
191  LoadSize /= 8;
192 
193  // If the Load isn't completely contained within the stored bits, we don't
194  // have all the bits to feed it. We could do something crazy in the future
195  // (issue a smaller load then merge the bits in) but this seems unlikely to be
196  // valuable.
197  if (StoreOffset > LoadOffset ||
198  StoreOffset + int64_t(StoreSize) < LoadOffset + int64_t(LoadSize))
199  return -1;
200 
201  // Okay, we can do this transformation. Return the number of bytes into the
202  // store that the load is.
203  return LoadOffset - StoreOffset;
204 }
205 
206 /// This function is called when we have a
207 /// memdep query of a load that ends up being a clobbering store.
209  StoreInst *DepSI, const DataLayout &DL) {
210  auto *StoredVal = DepSI->getValueOperand();
211 
212  // Cannot handle reading from store of first-class aggregate or scalable type.
213  if (isFirstClassAggregateOrScalableType(StoredVal->getType()))
214  return -1;
215 
216  if (!canCoerceMustAliasedValueToLoad(StoredVal, LoadTy, DL))
217  return -1;
218 
219  Value *StorePtr = DepSI->getPointerOperand();
220  uint64_t StoreSize =
221  DL.getTypeSizeInBits(DepSI->getValueOperand()->getType()).getFixedSize();
222  return analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, StorePtr, StoreSize,
223  DL);
224 }
225 
226 /// Looks at a memory location for a load (specified by MemLocBase, Offs, and
227 /// Size) and compares it against a load.
228 ///
229 /// If the specified load could be safely widened to a larger integer load
230 /// that is 1) still efficient, 2) safe for the target, and 3) would provide
231 /// the specified memory location value, then this function returns the size
232 /// in bytes of the load width to use. If not, this returns zero.
233 static unsigned getLoadLoadClobberFullWidthSize(const Value *MemLocBase,
234  int64_t MemLocOffs,
235  unsigned MemLocSize,
236  const LoadInst *LI) {
237  // We can only extend simple integer loads.
238  if (!isa<IntegerType>(LI->getType()) || !LI->isSimple())
239  return 0;
240 
241  // Load widening is hostile to ThreadSanitizer: it may cause false positives
242  // or make the reports more cryptic (access sizes are wrong).
243  if (LI->getParent()->getParent()->hasFnAttribute(Attribute::SanitizeThread))
244  return 0;
245 
246  const DataLayout &DL = LI->getModule()->getDataLayout();
247 
248  // Get the base of this load.
249  int64_t LIOffs = 0;
250  const Value *LIBase =
252 
253  // If the two pointers are not based on the same pointer, we can't tell that
254  // they are related.
255  if (LIBase != MemLocBase)
256  return 0;
257 
258  // Okay, the two values are based on the same pointer, but returned as
259  // no-alias. This happens when we have things like two byte loads at "P+1"
260  // and "P+3". Check to see if increasing the size of the "LI" load up to its
261  // alignment (or the largest native integer type) will allow us to load all
262  // the bits required by MemLoc.
263 
264  // If MemLoc is before LI, then no widening of LI will help us out.
265  if (MemLocOffs < LIOffs)
266  return 0;
267 
268  // Get the alignment of the load in bytes. We assume that it is safe to load
269  // any legal integer up to this size without a problem. For example, if we're
270  // looking at an i8 load on x86-32 that is known 1024 byte aligned, we can
271  // widen it up to an i32 load. If it is known 2-byte aligned, we can widen it
272  // to i16.
273  unsigned LoadAlign = LI->getAlign().value();
274 
275  int64_t MemLocEnd = MemLocOffs + MemLocSize;
276 
277  // If no amount of rounding up will let MemLoc fit into LI, then bail out.
278  if (LIOffs + LoadAlign < MemLocEnd)
279  return 0;
280 
281  // This is the size of the load to try. Start with the next larger power of
282  // two.
283  unsigned NewLoadByteSize = LI->getType()->getPrimitiveSizeInBits() / 8U;
284  NewLoadByteSize = NextPowerOf2(NewLoadByteSize);
285 
286  while (true) {
287  // If this load size is bigger than our known alignment or would not fit
288  // into a native integer register, then we fail.
289  if (NewLoadByteSize > LoadAlign ||
290  !DL.fitsInLegalInteger(NewLoadByteSize * 8))
291  return 0;
292 
293  if (LIOffs + NewLoadByteSize > MemLocEnd &&
295  Attribute::SanitizeAddress) ||
297  Attribute::SanitizeHWAddress)))
298  // We will be reading past the location accessed by the original program.
299  // While this is safe in a regular build, Address Safety analysis tools
300  // may start reporting false warnings. So, don't do widening.
301  return 0;
302 
303  // If a load of this width would include all of MemLoc, then we succeed.
304  if (LIOffs + NewLoadByteSize >= MemLocEnd)
305  return NewLoadByteSize;
306 
307  NewLoadByteSize <<= 1;
308  }
309 }
310 
311 /// This function is called when we have a
312 /// memdep query of a load that ends up being clobbered by another load. See if
313 /// the other load can feed into the second load.
314 int analyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr, LoadInst *DepLI,
315  const DataLayout &DL) {
316  // Cannot handle reading from store of first-class aggregate yet.
317  if (DepLI->getType()->isStructTy() || DepLI->getType()->isArrayTy())
318  return -1;
319 
320  if (!canCoerceMustAliasedValueToLoad(DepLI, LoadTy, DL))
321  return -1;
322 
323  Value *DepPtr = DepLI->getPointerOperand();
324  uint64_t DepSize = DL.getTypeSizeInBits(DepLI->getType()).getFixedSize();
325  int R = analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, DepPtr, DepSize, DL);
326  if (R != -1)
327  return R;
328 
329  // If we have a load/load clobber an DepLI can be widened to cover this load,
330  // then we should widen it!
331  int64_t LoadOffs = 0;
332  const Value *LoadBase =
333  GetPointerBaseWithConstantOffset(LoadPtr, LoadOffs, DL);
334  unsigned LoadSize = DL.getTypeStoreSize(LoadTy).getFixedSize();
335 
336  unsigned Size =
337  getLoadLoadClobberFullWidthSize(LoadBase, LoadOffs, LoadSize, DepLI);
338  if (Size == 0)
339  return -1;
340 
341  // Check non-obvious conditions enforced by MDA which we rely on for being
342  // able to materialize this potentially available value
343  assert(DepLI->isSimple() && "Cannot widen volatile/atomic load!");
344  assert(DepLI->getType()->isIntegerTy() && "Can't widen non-integer load");
345 
346  return analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, DepPtr, Size * 8, DL);
347 }
348 
350  MemIntrinsic *MI, const DataLayout &DL) {
351  // If the mem operation is a non-constant size, we can't handle it.
352  ConstantInt *SizeCst = dyn_cast<ConstantInt>(MI->getLength());
353  if (!SizeCst)
354  return -1;
355  uint64_t MemSizeInBits = SizeCst->getZExtValue() * 8;
356 
357  // If this is memset, we just need to see if the offset is valid in the size
358  // of the memset..
359  if (const auto *memset_inst = dyn_cast<MemSetInst>(MI)) {
360  if (DL.isNonIntegralPointerType(LoadTy->getScalarType())) {
361  auto *CI = dyn_cast<ConstantInt>(memset_inst->getValue());
362  if (!CI || !CI->isZero())
363  return -1;
364  }
365  return analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, MI->getDest(),
366  MemSizeInBits, DL);
367  }
368 
369  // If we have a memcpy/memmove, the only case we can handle is if this is a
370  // copy from constant memory. In that case, we can read directly from the
371  // constant memory.
372  MemTransferInst *MTI = cast<MemTransferInst>(MI);
373 
374  Constant *Src = dyn_cast<Constant>(MTI->getSource());
375  if (!Src)
376  return -1;
377 
378  GlobalVariable *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(Src));
379  if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
380  return -1;
381 
382  // See if the access is within the bounds of the transfer.
383  int Offset = analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, MI->getDest(),
384  MemSizeInBits, DL);
385  if (Offset == -1)
386  return Offset;
387 
388  // Otherwise, see if we can constant fold a load from the constant with the
389  // offset applied as appropriate.
390  unsigned IndexSize = DL.getIndexTypeSizeInBits(Src->getType());
391  if (ConstantFoldLoadFromConstPtr(Src, LoadTy, APInt(IndexSize, Offset), DL))
392  return Offset;
393  return -1;
394 }
395 
396 static Value *getStoreValueForLoadHelper(Value *SrcVal, unsigned Offset,
397  Type *LoadTy, IRBuilderBase &Builder,
398  const DataLayout &DL) {
399  LLVMContext &Ctx = SrcVal->getType()->getContext();
400 
401  // If two pointers are in the same address space, they have the same size,
402  // so we don't need to do any truncation, etc. This avoids introducing
403  // ptrtoint instructions for pointers that may be non-integral.
404  if (SrcVal->getType()->isPointerTy() && LoadTy->isPointerTy() &&
405  cast<PointerType>(SrcVal->getType())->getAddressSpace() ==
406  cast<PointerType>(LoadTy)->getAddressSpace()) {
407  return SrcVal;
408  }
409 
410  uint64_t StoreSize =
411  (DL.getTypeSizeInBits(SrcVal->getType()).getFixedSize() + 7) / 8;
412  uint64_t LoadSize = (DL.getTypeSizeInBits(LoadTy).getFixedSize() + 7) / 8;
413  // Compute which bits of the stored value are being used by the load. Convert
414  // to an integer type to start with.
415  if (SrcVal->getType()->isPtrOrPtrVectorTy())
416  SrcVal =
417  Builder.CreatePtrToInt(SrcVal, DL.getIntPtrType(SrcVal->getType()));
418  if (!SrcVal->getType()->isIntegerTy())
419  SrcVal =
420  Builder.CreateBitCast(SrcVal, IntegerType::get(Ctx, StoreSize * 8));
421 
422  // Shift the bits to the least significant depending on endianness.
423  unsigned ShiftAmt;
424  if (DL.isLittleEndian())
425  ShiftAmt = Offset * 8;
426  else
427  ShiftAmt = (StoreSize - LoadSize - Offset) * 8;
428  if (ShiftAmt)
429  SrcVal = Builder.CreateLShr(SrcVal,
430  ConstantInt::get(SrcVal->getType(), ShiftAmt));
431 
432  if (LoadSize != StoreSize)
433  SrcVal = Builder.CreateTruncOrBitCast(SrcVal,
434  IntegerType::get(Ctx, LoadSize * 8));
435  return SrcVal;
436 }
437 
438 /// This function is called when we have a memdep query of a load that ends up
439 /// being a clobbering store. This means that the store provides bits used by
440 /// the load but the pointers don't must-alias. Check this case to see if
441 /// there is anything more we can do before we give up.
442 Value *getStoreValueForLoad(Value *SrcVal, unsigned Offset, Type *LoadTy,
443  Instruction *InsertPt, const DataLayout &DL) {
444 
445  IRBuilder<> Builder(InsertPt);
446  SrcVal = getStoreValueForLoadHelper(SrcVal, Offset, LoadTy, Builder, DL);
447  return coerceAvailableValueToLoadType(SrcVal, LoadTy, Builder, DL);
448 }
449 
451  Type *LoadTy, const DataLayout &DL) {
452  return ConstantFoldLoadFromConst(SrcVal, LoadTy, APInt(32, Offset), DL);
453 }
454 
455 /// This function is called when we have a memdep query of a load that ends up
456 /// being a clobbering load. This means that the load *may* provide bits used
457 /// by the load but we can't be sure because the pointers don't must-alias.
458 /// Check this case to see if there is anything more we can do before we give
459 /// up.
460 Value *getLoadValueForLoad(LoadInst *SrcVal, unsigned Offset, Type *LoadTy,
461  Instruction *InsertPt, const DataLayout &DL) {
462  // If Offset+LoadTy exceeds the size of SrcVal, then we must be wanting to
463  // widen SrcVal out to a larger load.
464  unsigned SrcValStoreSize =
465  DL.getTypeStoreSize(SrcVal->getType()).getFixedSize();
466  unsigned LoadSize = DL.getTypeStoreSize(LoadTy).getFixedSize();
467  if (Offset + LoadSize > SrcValStoreSize) {
468  assert(SrcVal->isSimple() && "Cannot widen volatile/atomic load!");
469  assert(SrcVal->getType()->isIntegerTy() && "Can't widen non-integer load");
470  // If we have a load/load clobber an DepLI can be widened to cover this
471  // load, then we should widen it to the next power of 2 size big enough!
472  unsigned NewLoadSize = Offset + LoadSize;
473  if (!isPowerOf2_32(NewLoadSize))
474  NewLoadSize = NextPowerOf2(NewLoadSize);
475 
476  Value *PtrVal = SrcVal->getPointerOperand();
477  // Insert the new load after the old load. This ensures that subsequent
478  // memdep queries will find the new load. We can't easily remove the old
479  // load completely because it is already in the value numbering table.
480  IRBuilder<> Builder(SrcVal->getParent(), ++BasicBlock::iterator(SrcVal));
481  Type *DestTy = IntegerType::get(LoadTy->getContext(), NewLoadSize * 8);
482  Type *DestPTy =
483  PointerType::get(DestTy, PtrVal->getType()->getPointerAddressSpace());
484  Builder.SetCurrentDebugLocation(SrcVal->getDebugLoc());
485  PtrVal = Builder.CreateBitCast(PtrVal, DestPTy);
486  LoadInst *NewLoad = Builder.CreateLoad(DestTy, PtrVal);
487  NewLoad->takeName(SrcVal);
488  NewLoad->setAlignment(SrcVal->getAlign());
489 
490  LLVM_DEBUG(dbgs() << "GVN WIDENED LOAD: " << *SrcVal << "\n");
491  LLVM_DEBUG(dbgs() << "TO: " << *NewLoad << "\n");
492 
493  // Replace uses of the original load with the wider load. On a big endian
494  // system, we need to shift down to get the relevant bits.
495  Value *RV = NewLoad;
496  if (DL.isBigEndian())
497  RV = Builder.CreateLShr(RV, (NewLoadSize - SrcValStoreSize) * 8);
498  RV = Builder.CreateTrunc(RV, SrcVal->getType());
499  SrcVal->replaceAllUsesWith(RV);
500 
501  SrcVal = NewLoad;
502  }
503 
504  return getStoreValueForLoad(SrcVal, Offset, LoadTy, InsertPt, DL);
505 }
506 
508  Type *LoadTy, const DataLayout &DL) {
509  unsigned SrcValStoreSize =
510  DL.getTypeStoreSize(SrcVal->getType()).getFixedSize();
511  unsigned LoadSize = DL.getTypeStoreSize(LoadTy).getFixedSize();
512  if (Offset + LoadSize > SrcValStoreSize)
513  return nullptr;
514  return getConstantStoreValueForLoad(SrcVal, Offset, LoadTy, DL);
515 }
516 
517 /// This function is called when we have a
518 /// memdep query of a load that ends up being a clobbering mem intrinsic.
519 Value *getMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset,
520  Type *LoadTy, Instruction *InsertPt,
521  const DataLayout &DL) {
522  LLVMContext &Ctx = LoadTy->getContext();
523  uint64_t LoadSize = DL.getTypeSizeInBits(LoadTy).getFixedSize() / 8;
524  IRBuilder<> Builder(InsertPt);
525 
526  // We know that this method is only called when the mem transfer fully
527  // provides the bits for the load.
528  if (MemSetInst *MSI = dyn_cast<MemSetInst>(SrcInst)) {
529  // memset(P, 'x', 1234) -> splat('x'), even if x is a variable, and
530  // independently of what the offset is.
531  Value *Val = MSI->getValue();
532  if (LoadSize != 1)
533  Val =
534  Builder.CreateZExtOrBitCast(Val, IntegerType::get(Ctx, LoadSize * 8));
535  Value *OneElt = Val;
536 
537  // Splat the value out to the right number of bits.
538  for (unsigned NumBytesSet = 1; NumBytesSet != LoadSize;) {
539  // If we can double the number of bytes set, do it.
540  if (NumBytesSet * 2 <= LoadSize) {
541  Value *ShVal = Builder.CreateShl(
542  Val, ConstantInt::get(Val->getType(), NumBytesSet * 8));
543  Val = Builder.CreateOr(Val, ShVal);
544  NumBytesSet <<= 1;
545  continue;
546  }
547 
548  // Otherwise insert one byte at a time.
549  Value *ShVal =
550  Builder.CreateShl(Val, ConstantInt::get(Val->getType(), 1 * 8));
551  Val = Builder.CreateOr(OneElt, ShVal);
552  ++NumBytesSet;
553  }
554 
555  return coerceAvailableValueToLoadType(Val, LoadTy, Builder, DL);
556  }
557 
558  // Otherwise, this is a memcpy/memmove from a constant global.
559  MemTransferInst *MTI = cast<MemTransferInst>(SrcInst);
560  Constant *Src = cast<Constant>(MTI->getSource());
561  unsigned IndexSize = DL.getIndexTypeSizeInBits(Src->getType());
562  return ConstantFoldLoadFromConstPtr(Src, LoadTy, APInt(IndexSize, Offset),
563  DL);
564 }
565 
567  Type *LoadTy, const DataLayout &DL) {
568  LLVMContext &Ctx = LoadTy->getContext();
569  uint64_t LoadSize = DL.getTypeSizeInBits(LoadTy).getFixedSize() / 8;
570 
571  // We know that this method is only called when the mem transfer fully
572  // provides the bits for the load.
573  if (MemSetInst *MSI = dyn_cast<MemSetInst>(SrcInst)) {
574  auto *Val = dyn_cast<ConstantInt>(MSI->getValue());
575  if (!Val)
576  return nullptr;
577 
578  Val = ConstantInt::get(Ctx, APInt::getSplat(LoadSize * 8, Val->getValue()));
579  return ConstantFoldLoadFromConst(Val, LoadTy, DL);
580  }
581 
582  // Otherwise, this is a memcpy/memmove from a constant global.
583  MemTransferInst *MTI = cast<MemTransferInst>(SrcInst);
584  Constant *Src = cast<Constant>(MTI->getSource());
585  unsigned IndexSize = DL.getIndexTypeSizeInBits(Src->getType());
586  return ConstantFoldLoadFromConstPtr(Src, LoadTy, APInt(IndexSize, Offset),
587  DL);
588 }
589 } // namespace VNCoercion
590 } // namespace llvm
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:156
llvm::VNCoercion::analyzeLoadFromClobberingMemInst
int analyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr, MemIntrinsic *DepMI, const DataLayout &DL)
This function determines whether a value for the pointer LoadPtr can be extracted from the memory int...
Definition: VNCoercion.cpp:349
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:108
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:69
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::VNCoercion::getStoreValueForLoad
Value * getStoreValueForLoad(Value *SrcVal, unsigned Offset, Type *LoadTy, Instruction *InsertPt, const DataLayout &DL)
If analyzeLoadFromClobberingStore returned an offset, this function can be used to actually perform t...
Definition: VNCoercion.cpp:442
llvm::BasicBlock::iterator
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:87
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:104
IntrinsicInst.h
llvm::Type::isPointerTy
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:237
llvm::VNCoercion::analyzeLoadFromClobberingWrite
static int analyzeLoadFromClobberingWrite(Type *LoadTy, Value *LoadPtr, Value *WritePtr, uint64_t WriteSizeInBits, const DataLayout &DL)
This function is called when we have a memdep query of a load that ends up being a clobbering memory ...
Definition: VNCoercion.cpp:170
llvm::MemTransferInst
This class wraps the llvm.memcpy/memmove intrinsics.
Definition: IntrinsicInst.h:1106
llvm::IRBuilderBase::CreatePtrToInt
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1975
llvm::VNCoercion::getStoreValueForLoadHelper
static Value * getStoreValueForLoadHelper(Value *SrcVal, unsigned Offset, Type *LoadTy, IRBuilderBase &Builder, const DataLayout &DL)
Definition: VNCoercion.cpp:396
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Definition: Type.cpp:727
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:328
llvm::Type::getPointerAddressSpace
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:729
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
ValueTracking.h
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::IRBuilderBase::CreateTruncOrBitCast
Value * CreateTruncOrBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2013
llvm::MemIntrinsic
This is the common base class for memset/memcpy/memmove.
Definition: IntrinsicInst.h:1041
llvm::VNCoercion::coerceAvailableValueToLoadType
Value * coerceAvailableValueToLoadType(Value *StoredVal, Type *LoadedTy, IRBuilderBase &IRB, const DataLayout &DL)
If we saw a store of a value to memory, and then a load from a must-aliased pointer of a different ty...
Definition: VNCoercion.cpp:73
llvm::LoadInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:260
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:458
llvm::LoadInst::getAlign
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:216
ConstantFolding.h
llvm::IRBuilderBase::CreateIntToPtr
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1980
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::Type::isArrayTy
bool isArrayTy() const
True if this is an instance of ArrayType.
Definition: Type.h:234
llvm::StoreInst::getValueOperand
Value * getValueOperand()
Definition: Instructions.h:386
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::NextPowerOf2
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:611
llvm::Instruction
Definition: Instruction.h:42
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::getUnderlyingObject
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
Definition: ValueTracking.cpp:4498
llvm::MemSetInst
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
Definition: IntrinsicInst.h:1073
llvm::VNCoercion::getMemInstValueForLoad
Value * getMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset, Type *LoadTy, Instruction *InsertPt, const DataLayout &DL)
If analyzeLoadFromClobberingMemInst returned an offset, this function can be used to actually perform...
Definition: VNCoercion.cpp:519
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:210
llvm::Function::hasFnAttribute
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:629
llvm::IRBuilderBase::CreateBitCast
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1985
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:297
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
uint64_t
llvm::GlobalVariable::hasDefinitiveInitializer
bool hasDefinitiveInitializer() const
hasDefinitiveInitializer - Whether the global variable has an initializer, and any other instances of...
Definition: GlobalVariable.h:109
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
VNCoercion.h
llvm::LoadInst::setAlignment
void setAlignment(Align Align)
Definition: Instructions.h:220
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::ConstantFoldLoadFromConst
Constant * ConstantFoldLoadFromConst(Constant *C, Type *Ty, const APInt &Offset, const DataLayout &DL)
Extract value of C at the given Offset reinterpreted as Ty.
Definition: ConstantFolding.cpp:698
llvm::VNCoercion::getConstantLoadValueForLoad
Constant * getConstantLoadValueForLoad(Constant *SrcVal, unsigned Offset, Type *LoadTy, const DataLayout &DL)
Definition: VNCoercion.cpp:507
llvm::LoadInst::isSimple
bool isSimple() const
Definition: Instructions.h:252
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:93
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition: Type.h:243
llvm::VNCoercion::isFirstClassAggregateOrScalableType
static bool isFirstClassAggregateOrScalableType(Type *Ty)
Definition: VNCoercion.cpp:13
llvm::Type::getContext
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:173
llvm::VNCoercion::getLoadValueForLoad
Value * getLoadValueForLoad(LoadInst *SrcVal, unsigned Offset, Type *LoadTy, Instruction *InsertPt, const DataLayout &DL)
If analyzeLoadFromClobberingLoad returned an offset, this function can be used to actually perform th...
Definition: VNCoercion.cpp:460
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
llvm::ConstantFoldLoadFromConstPtr
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
Definition: ConstantFolding.cpp:729
llvm::VNCoercion::getConstantMemInstValueForLoad
Constant * getConstantMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset, Type *LoadTy, const DataLayout &DL)
Definition: VNCoercion.cpp:566
llvm::VNCoercion::getLoadLoadClobberFullWidthSize
static unsigned getLoadLoadClobberFullWidthSize(const Value *MemLocBase, int64_t MemLocOffs, unsigned MemLocSize, const LoadInst *LI)
Looks at a memory location for a load (specified by MemLocBase, Offs, and Size) and compares it again...
Definition: VNCoercion.cpp:233
llvm::Align::value
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
llvm::VNCoercion::analyzeLoadFromClobberingStore
int analyzeLoadFromClobberingStore(Type *LoadTy, Value *LoadPtr, StoreInst *DepSI, const DataLayout &DL)
This function determines whether a value for the pointer LoadPtr can be extracted from the store at D...
Definition: VNCoercion.cpp:208
llvm::GetPointerBaseWithConstantOffset
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
Definition: ValueTracking.h:272
llvm::APInt::getSplat
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.cpp:612
llvm::VNCoercion::canCoerceMustAliasedValueToLoad
bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy, const DataLayout &DL)
Return true if CoerceAvailableValueToLoadType would succeed if it was called.
Definition: VNCoercion.cpp:18
llvm::VNCoercion::getConstantStoreValueForLoad
Constant * getConstantStoreValueForLoad(Constant *SrcVal, unsigned Offset, Type *LoadTy, const DataLayout &DL)
Definition: VNCoercion.cpp:450
llvm::Type::isStructTy
bool isStructTy() const
True if this is an instance of StructType.
Definition: Type.h:231
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:354
llvm::StoreInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:389
llvm::GlobalVariable::isConstant
bool isConstant() const
If the value is a global constant, its value is immutable throughout the runtime execution of the pro...
Definition: GlobalVariable.h:152
llvm::IRBuilderBase::CreateLShr
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1343
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::VNCoercion::analyzeLoadFromClobberingLoad
int analyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr, LoadInst *DepLI, const DataLayout &DL)
This function determines whether a value for the pointer LoadPtr can be extracted from the load at De...
Definition: VNCoercion.cpp:314
llvm::ConstantFoldConstant
Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
Definition: ConstantFolding.cpp:1207
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
llvm::IntegerType::get
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:311
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:381
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
Debug.h
llvm::Type::getPrimitiveSizeInBits
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:164
llvm::MemTransferBase::getSource
Value * getSource() const
This is just like getRawSource, but it strips off any cast instructions that feed it,...
Definition: IntrinsicInst.h:884