LLVM 24.0.0git
ConstantFolding.cpp
Go to the documentation of this file.
1//===-- ConstantFolding.cpp - Fold instructions into constants ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines routines for folding instructions into constants.
10//
11// Also, to supplement the basic IR ConstantExpr simplifications,
12// this file defines some additional folding routines that can make use of
13// DataLayout information. These functions cannot go in IR due to library
14// dependency issues.
15//
16//===----------------------------------------------------------------------===//
17
19#include "llvm/ADT/APFloat.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/ADT/APSInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/StringRef.h"
32#include "llvm/Config/config.h"
33#include "llvm/IR/Constant.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DataLayout.h"
38#include "llvm/IR/Function.h"
39#include "llvm/IR/GlobalValue.h"
41#include "llvm/IR/InstrTypes.h"
42#include "llvm/IR/Instruction.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/IntrinsicsAArch64.h"
47#include "llvm/IR/IntrinsicsAMDGPU.h"
48#include "llvm/IR/IntrinsicsARM.h"
49#include "llvm/IR/IntrinsicsNVPTX.h"
50#include "llvm/IR/IntrinsicsWebAssembly.h"
51#include "llvm/IR/IntrinsicsX86.h"
53#include "llvm/IR/Operator.h"
54#include "llvm/IR/Type.h"
55#include "llvm/IR/Value.h"
56#include "llvm/Support/CRC.h"
60#include <cassert>
61#include <cerrno>
62#include <cfenv>
63#include <cmath>
64#include <cstdint>
65
66using namespace llvm;
67
69 "disable-fp-call-folding",
70 cl::desc("Disable constant-folding of FP intrinsics and libcalls."),
71 cl::init(false), cl::Hidden);
72
73namespace {
74
75//===----------------------------------------------------------------------===//
76// Constant Folding internal helper functions
77//===----------------------------------------------------------------------===//
78
79static Constant *foldConstVectorToAPInt(APInt &Result, Type *DestTy,
80 Constant *C, Type *SrcEltTy,
81 unsigned NumSrcElts,
82 const DataLayout &DL) {
83 // Now that we know that the input value is a vector of integers, just shift
84 // and insert them into our result.
85 unsigned BitShift = DL.getTypeSizeInBits(SrcEltTy);
86 for (unsigned i = 0; i != NumSrcElts; ++i) {
87 Constant *Element;
88 if (DL.isLittleEndian())
89 Element = C->getAggregateElement(NumSrcElts - i - 1);
90 else
91 Element = C->getAggregateElement(i);
92
93 if (isa_and_nonnull<UndefValue>(Element)) {
94 Result <<= BitShift;
95 continue;
96 }
97
98 auto *ElementCI = dyn_cast_or_null<ConstantInt>(Element);
99 if (!ElementCI)
100 return ConstantExpr::getBitCast(C, DestTy);
101
102 Result <<= BitShift;
103 Result |= ElementCI->getValue().zext(Result.getBitWidth());
104 }
105
106 return nullptr;
107}
108
109/// Check whether folding this bitcast into a byte vector would mix poison and
110/// non-poison bits in the same output lane. While integer types track poison on
111/// a per-value basis, byte types track it on a per-bit basis. However,
112/// `ConstantByte` cannot represent values with both poison and non-poison bits.
113///
114/// Source elements are grouped by the output lane they map to. Returns true if
115/// any group contains both poison and non-poison elements.
116static bool foldMixesPoisonBits(Constant *C, unsigned NumSrcElt,
117 unsigned NumDstElt) {
118 // If element counts don't divide evenly, bail out if a poison source element
119 // might span multiple destination lanes.
120 if (NumSrcElt % NumDstElt != 0)
121 return C->containsPoisonElement();
122 unsigned Ratio = NumSrcElt / NumDstElt;
123 for (unsigned i = 0; i != NumSrcElt; i += Ratio) {
124 bool HasPoison = false;
125 bool HasNonPoison = false;
126 for (unsigned j = 0; j != Ratio; ++j) {
127 Constant *Src = C->getAggregateElement(i + j);
128 // Conservatively bail out.
129 if (!Src)
130 return true;
131 if (isa<PoisonValue>(Src))
132 HasPoison = true;
133 else
134 HasNonPoison = true;
135 }
136 if (HasPoison && HasNonPoison)
137 return true;
138 }
139 return false;
140}
141
142/// Track which destination lanes of a bitcast are produced from poison bytes.
143/// A destination lane is marked if any source element mapped to it is poison.
144/// Returns false if an aggregate element cannot be inspected. The caller should
145/// bail out of folding.
146static bool computePoisonDstLanes(Constant *C, unsigned NumSrcElt,
147 unsigned NumDstElt,
148 SmallBitVector &PoisonDstElts) {
149 // If element counts don't divide evenly, bail out if a poison source element
150 // might span multiple destination lanes.
151 if ((NumDstElt < NumSrcElt ? NumSrcElt % NumDstElt : NumDstElt % NumSrcElt))
152 return !C->containsPoisonElement();
153 if (NumDstElt < NumSrcElt) {
154 unsigned Ratio = NumSrcElt / NumDstElt;
155 for (unsigned i = 0; i != NumDstElt; ++i) {
156 for (unsigned j = 0; j != Ratio; ++j) {
157 Constant *Src = C->getAggregateElement(i * Ratio + j);
158 if (!Src)
159 return false;
160 if (isa<PoisonValue>(Src)) {
161 PoisonDstElts[i] = true;
162 break;
163 }
164 }
165 }
166 } else {
167 unsigned Ratio = NumDstElt / NumSrcElt;
168 for (unsigned i = 0; i != NumSrcElt; ++i) {
169 Constant *Src = C->getAggregateElement(i);
170 if (!Src)
171 return false;
172 if (isa<PoisonValue>(Src))
173 PoisonDstElts.set(i * Ratio, (i + 1) * Ratio);
174 }
175 }
176 return true;
177}
178
179/// Constant fold bitcast, symbolically evaluating it with DataLayout.
180/// This always returns a non-null constant, but it may be a
181/// ConstantExpr if unfoldable.
182Constant *FoldBitCast(Constant *C, Type *DestTy, const DataLayout &DL) {
183 assert(CastInst::castIsValid(Instruction::BitCast, C, DestTy) &&
184 "Invalid constantexpr bitcast!");
185
186 // Catch the obvious splat cases.
187 if (Constant *Res = ConstantFoldLoadFromUniformValue(C, DestTy, DL))
188 return Res;
189
190 if (auto *VTy = dyn_cast<VectorType>(C->getType())) {
191 // Handle a vector->scalar integer/fp/byte cast.
192 if (isa<IntegerType>(DestTy) || DestTy->isFloatingPointTy() ||
193 DestTy->isByteTy()) {
194 unsigned NumSrcElts = cast<FixedVectorType>(VTy)->getNumElements();
195 Type *SrcEltTy = VTy->getElementType();
196
197 // A byte constant can't represent partial poison or pointer provenance.
198 if (DestTy->isByteTy() &&
199 (C->containsPoisonElement() || SrcEltTy->isPointerTy()))
200 return ConstantExpr::getBitCast(C, DestTy);
201
202 // Bitcasting a byte containing any poison bit to an integer or fp type
203 // yields poison.
204 if (SrcEltTy->isByteTy() && C->containsPoisonElement())
205 return PoisonValue::get(DestTy);
206
207 // If the vector is a vector of floating point or bytes, convert it to a
208 // vector of int to simplify things.
209 Constant *OrigC = C;
210 if (SrcEltTy->isFloatingPointTy() || SrcEltTy->isByteTy()) {
211 unsigned Width = SrcEltTy->getPrimitiveSizeInBits();
212 auto *SrcIVTy = FixedVectorType::get(
213 IntegerType::get(C->getContext(), Width), NumSrcElts);
214 // Ask IR to do the conversion now that #elts line up.
215 C = ConstantExpr::getBitCast(C, SrcIVTy);
216 }
217
218 APInt Result(DL.getTypeSizeInBits(DestTy), 0);
219 // An element that isn't a plain constant, like a pointer cast to a byte,
220 // can't be folded. Byte destinations fall back to a bitcast of the
221 // original vector, as the integer vector C would drop its provenance.
222 if (Constant *CE = foldConstVectorToAPInt(Result, DestTy, C, SrcEltTy,
223 NumSrcElts, DL))
224 return DestTy->isByteTy() ? ConstantExpr::getBitCast(OrigC, DestTy)
225 : CE;
226
227 if (isa<IntegerType>(DestTy))
228 return ConstantInt::get(DestTy, Result);
229
230 if (isa<ByteType>(DestTy))
231 return ConstantByte::get(DestTy->getContext(), Result);
232
233 APFloat FP(DestTy->getFltSemantics(), Result);
234 return ConstantFP::get(DestTy->getContext(), FP);
235 }
236 }
237
238 // The code below only handles casts to vectors currently.
239 auto *DestVTy = dyn_cast<VectorType>(DestTy);
240 if (!DestVTy)
241 return ConstantExpr::getBitCast(C, DestTy);
242
243 // If this is a scalar -> vector cast, convert the input into a <1 x scalar>
244 // vector so the code below can handle it uniformly.
245 if (!isa<VectorType>(C->getType()) &&
247 Constant *Ops = C; // don't take the address of C!
248 return FoldBitCast(ConstantVector::get(Ops), DestTy, DL);
249 }
250
251 // Some of what follows may extend to cover scalable vectors but the current
252 // implementation is fixed length specific.
253 if (!isa<FixedVectorType>(C->getType()))
254 return ConstantExpr::getBitCast(C, DestTy);
255
256 // If this is a bitcast from constant vector -> vector, fold it.
259 return ConstantExpr::getBitCast(C, DestTy);
260
261 // If the element types match, IR can fold it.
262 unsigned NumDstElt = cast<FixedVectorType>(DestVTy)->getNumElements();
263 unsigned NumSrcElt = cast<FixedVectorType>(C->getType())->getNumElements();
264 if (NumDstElt == NumSrcElt)
265 return ConstantExpr::getBitCast(C, DestTy);
266
267 Type *SrcEltTy = cast<VectorType>(C->getType())->getElementType();
268 Type *DstEltTy = DestVTy->getElementType();
269
270 // Otherwise, we're changing the number of elements in a vector, which
271 // requires endianness information to do the right thing. For example,
272 // bitcast (<2 x i64> <i64 0, i64 1> to <4 x i32>)
273 // folds to (little endian):
274 // <4 x i32> <i32 0, i32 0, i32 1, i32 0>
275 // and to (big endian):
276 // <4 x i32> <i32 0, i32 0, i32 0, i32 1>
277
278 // First thing is first. We only want to think about integer here, so if
279 // we have something in FP form, recast it as integer.
280 if (DstEltTy->isFloatingPointTy()) {
281 // Fold to an vector of integers with same size as our FP type.
282 unsigned FPWidth = DstEltTy->getPrimitiveSizeInBits();
283 auto *DestIVTy = FixedVectorType::get(
284 IntegerType::get(C->getContext(), FPWidth), NumDstElt);
285 // Recursively handle this integer conversion, if possible.
286 C = FoldBitCast(C, DestIVTy, DL);
287
288 // Finally, IR can handle this now that #elts line up.
289 return ConstantExpr::getBitCast(C, DestTy);
290 }
291
292 // Handle byte destination type by folding through integers.
293 if (DstEltTy->isByteTy()) {
294 // When combining elements into larger byte values, bail out if the fold
295 // mixes poison and non-poison bits in the same destination element. Byte
296 // types track poison per bit, and no constant value can represent that.
297 if (NumDstElt < NumSrcElt && foldMixesPoisonBits(C, NumSrcElt, NumDstElt))
298 return ConstantExpr::getBitCast(C, DestTy);
299
300 // Fold to a vector of integers with same size as the byte type.
301 unsigned ByteWidth = DstEltTy->getPrimitiveSizeInBits();
302 auto *DestIVTy = FixedVectorType::get(
303 IntegerType::get(C->getContext(), ByteWidth), NumDstElt);
304 C = FoldBitCast(C, DestIVTy, DL);
305 return ConstantExpr::getBitCast(C, DestTy);
306 }
307
308 // Okay, we know the destination is integer, if the input is FP, convert
309 // it to integer first.
310 if (SrcEltTy->isFloatingPointTy()) {
311 unsigned FPWidth = SrcEltTy->getPrimitiveSizeInBits();
312 auto *SrcIVTy = FixedVectorType::get(
313 IntegerType::get(C->getContext(), FPWidth), NumSrcElt);
314 // Ask IR to do the conversion now that #elts line up.
315 C = ConstantExpr::getBitCast(C, SrcIVTy);
316 assert((isa<ConstantVector>(C) || // FIXME: Remove ConstantVector.
318 "Constant folding cannot fail for plain fp->int bitcast!");
319 }
320
321 // Handle byte source type by folding through integers. Byte types track
322 // poison per bit, so any poison bit makes the destination lane poison.
323 // Record which destination lanes contain poison bits, before the generic
324 // fold below refines them to undef/zero, so they can be restored.
325 SmallBitVector PoisonDstElts(NumDstElt);
326 if (SrcEltTy->isByteTy()) {
327 if (!computePoisonDstLanes(C, NumSrcElt, NumDstElt, PoisonDstElts))
328 return ConstantExpr::getBitCast(C, DestTy);
329
330 unsigned ByteWidth = SrcEltTy->getPrimitiveSizeInBits();
331 auto *SrcIVTy = FixedVectorType::get(
332 IntegerType::get(C->getContext(), ByteWidth), NumSrcElt);
333 // Ask IR to do the conversion now that #elts line up.
334 C = ConstantExpr::getBitCast(C, SrcIVTy);
335 assert((isa<ConstantVector>(C) || // FIXME: Remove ConstantVector.
337 "Constant folding cannot fail for plain byte->int bitcast!");
338 }
339
340 // Now we know that the input and output vectors are both integer vectors
341 // of the same size, and that their #elements is not the same.
342 // Use data buffer for easy non-integer element ratio vectors handling,
343 // For example: <4 x i24> to <3 x i32>.
344 bool isLittleEndian = DL.isLittleEndian();
345 unsigned SrcBitSize = SrcEltTy->getPrimitiveSizeInBits();
346 unsigned DstBitSize = DstEltTy->getPrimitiveSizeInBits();
348 unsigned SrcElt = 0;
349
350 APInt Buffer(2 * std::max(SrcBitSize, DstBitSize), 0);
351 APInt UndefMask(Buffer.getBitWidth(), 0);
352 APInt PoisonMask(Buffer.getBitWidth(), 0);
353 unsigned BufferBitSize = 0;
354
355 while (Result.size() != NumDstElt) {
356 // Load SrcElts into Buffer.
357 while (BufferBitSize < DstBitSize) {
358 Constant *Element = C->getAggregateElement(SrcElt++);
359 if (!Element) // Reject constantexpr elements
360 return ConstantExpr::getBitCast(C, DestTy);
361
362 // Shift Buffer & Masks to fit next SrcElt.
363 if (!isLittleEndian) {
364 Buffer <<= SrcBitSize;
365 UndefMask <<= SrcBitSize;
366 PoisonMask <<= SrcBitSize;
367 }
368
369 APInt SrcValue;
370 unsigned BitPosition = isLittleEndian ? BufferBitSize : 0;
371 if (isa<UndefValue>(Element)) {
372 // Set masks fragments bits.
373 UndefMask.setBits(BitPosition, BitPosition + SrcBitSize);
374 if (isa<PoisonValue>(Element))
375 PoisonMask.setBits(BitPosition, BitPosition + SrcBitSize);
376 SrcValue = APInt::getZero(SrcBitSize);
377 } else {
378 auto *Src = dyn_cast<ConstantInt>(Element);
379 if (!Src)
380 return ConstantExpr::getBitCast(C, DestTy);
381 SrcValue = Src->getValue();
382 }
383
384 // Insert src element bits into Buffer on correct position.
385 Buffer.insertBits(SrcValue, BitPosition);
386 BufferBitSize += SrcBitSize;
387 }
388
389 // Create DstElts from Buffer.
390 while (BufferBitSize >= DstBitSize) {
391 unsigned ShiftAmt = isLittleEndian ? 0 : BufferBitSize - DstBitSize;
392 // Emit undef/poison, if all undef mask fragment bits are set.
393 if (UndefMask.extractBits(DstBitSize, ShiftAmt).isAllOnes()) {
394 // Push poison, if any bit in poison mask fragment is set.
395 if (!PoisonMask.extractBits(DstBitSize, ShiftAmt).isZero()) {
396 Result.push_back(PoisonValue::get(DstEltTy));
397 } else {
398 Result.push_back(UndefValue::get(DstEltTy));
399 }
400 } else {
401 // Create and push DstElt.
402 APInt Elt = Buffer.extractBits(DstBitSize, ShiftAmt);
403 Result.push_back(ConstantInt::get(DstEltTy, Elt));
404 }
405
406 // Shift unused Buffer fragment to lower bits.
407 if (isLittleEndian) {
408 Buffer.lshrInPlace(DstBitSize);
409 UndefMask.lshrInPlace(DstBitSize);
410 PoisonMask.lshrInPlace(DstBitSize);
411 }
412 BufferBitSize -= DstBitSize;
413 }
414 }
415
416 // Restore destination lanes whose source bytes contained poison bits.
417 for (unsigned I : PoisonDstElts.set_bits())
418 Result[I] = PoisonValue::get(DstEltTy);
419
420 return ConstantVector::get(Result);
421}
422
423} // end anonymous namespace
424
425/// If this constant is a constant offset from a global, return the global and
426/// the constant. Because of constantexprs, this function is recursive.
428 APInt &Offset, const DataLayout &DL,
429 DSOLocalEquivalent **DSOEquiv) {
430 if (DSOEquiv)
431 *DSOEquiv = nullptr;
432
433 // Trivial case, constant is the global.
434 if ((GV = dyn_cast<GlobalValue>(C))) {
435 unsigned BitWidth = DL.getIndexTypeSizeInBits(GV->getType());
436 Offset = APInt(BitWidth, 0);
437 return true;
438 }
439
440 if (auto *FoundDSOEquiv = dyn_cast<DSOLocalEquivalent>(C)) {
441 if (DSOEquiv)
442 *DSOEquiv = FoundDSOEquiv;
443 GV = FoundDSOEquiv->getGlobalValue();
444 unsigned BitWidth = DL.getIndexTypeSizeInBits(GV->getType());
445 Offset = APInt(BitWidth, 0);
446 return true;
447 }
448
449 // Otherwise, if this isn't a constant expr, bail out.
450 auto *CE = dyn_cast<ConstantExpr>(C);
451 if (!CE) return false;
452
453 // Look through ptr->int and ptr->ptr casts.
454 if (CE->getOpcode() == Instruction::PtrToInt ||
455 CE->getOpcode() == Instruction::PtrToAddr)
456 return IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, DL,
457 DSOEquiv);
458
459 // i32* getelementptr ([5 x i32]* @a, i32 0, i32 5)
460 auto *GEP = dyn_cast<GEPOperator>(CE);
461 if (!GEP)
462 return false;
463
464 unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType());
465 APInt TmpOffset(BitWidth, 0);
466
467 // If the base isn't a global+constant, we aren't either.
468 if (!IsConstantOffsetFromGlobal(CE->getOperand(0), GV, TmpOffset, DL,
469 DSOEquiv))
470 return false;
471
472 // Otherwise, add any offset that our operands provide.
473 if (!GEP->accumulateConstantOffset(DL, TmpOffset))
474 return false;
475
476 Offset = TmpOffset;
477 return true;
478}
479
481 const DataLayout &DL) {
482 do {
483 Type *SrcTy = C->getType();
484 if (SrcTy == DestTy)
485 return C;
486
487 TypeSize DestSize = DL.getTypeSizeInBits(DestTy);
488 TypeSize SrcSize = DL.getTypeSizeInBits(SrcTy);
489 if (!TypeSize::isKnownGE(SrcSize, DestSize))
490 return nullptr;
491
492 // Catch the obvious splat cases (since all-zeros can coerce non-integral
493 // pointers legally).
494 if (Constant *Res = ConstantFoldLoadFromUniformValue(C, DestTy, DL))
495 return Res;
496
497 // If the type sizes are the same and a cast is legal, just directly
498 // cast the constant.
499 // But be careful not to coerce non-integral pointers illegally.
500 if (SrcSize == DestSize &&
501 DL.isNonIntegralPointerType(SrcTy->getScalarType()) ==
502 DL.isNonIntegralPointerType(DestTy->getScalarType())) {
503 Instruction::CastOps Cast = Instruction::BitCast;
504 // If we are going from a pointer to int or vice versa, we spell the cast
505 // differently.
506 if (SrcTy->isIntegerTy() && DestTy->isPointerTy())
507 Cast = Instruction::IntToPtr;
508 else if (SrcTy->isPointerTy() && DestTy->isIntegerTy())
509 Cast = Instruction::PtrToInt;
510
511 if (CastInst::castIsValid(Cast, C, DestTy))
512 return ConstantFoldCastOperand(Cast, C, DestTy, DL);
513 }
514
515 // If this isn't an aggregate type, there is nothing we can do to drill down
516 // and find a bitcastable constant.
517 if (!SrcTy->isAggregateType() && !SrcTy->isVectorTy())
518 return nullptr;
519
520 // We're simulating a load through a pointer that was bitcast to point to
521 // a different type, so we can try to walk down through the initial
522 // elements of an aggregate to see if some part of the aggregate is
523 // castable to implement the "load" semantic model.
524 if (SrcTy->isStructTy()) {
525 // Struct types might have leading zero-length elements like [0 x i32],
526 // which are certainly not what we are looking for, so skip them.
527 unsigned Elem = 0;
528 Constant *ElemC;
529 do {
530 ElemC = C->getAggregateElement(Elem++);
531 } while (ElemC && DL.getTypeSizeInBits(ElemC->getType()).isZero());
532 C = ElemC;
533 } else {
534 // For non-byte-sized vector elements, the first element is not
535 // necessarily located at the vector base address.
536 if (auto *VT = dyn_cast<VectorType>(SrcTy))
537 if (!DL.typeSizeEqualsStoreSize(VT->getElementType()))
538 return nullptr;
539
540 C = C->getAggregateElement(0u);
541 }
542 } while (C);
543
544 return nullptr;
545}
546
547/// Return true if \p Ty is or contains a byte type.
548static bool containsByteType(Type *Ty) {
549 if (auto *ATy = dyn_cast<ArrayType>(Ty))
550 return containsByteType(ATy->getElementType());
551 if (auto *STy = dyn_cast<StructType>(Ty))
552 return any_of(STy->elements(), containsByteType);
553 return Ty->isByteOrByteVectorTy();
554}
555
556namespace {
557
558/// Recursive helper to read bits out of global. C is the constant being copied
559/// out of. ByteOffset is an offset into C. CurPtr is the pointer to copy
560/// results into and BytesLeft is the number of bytes left in
561/// the CurPtr buffer. DL is the DataLayout. When IsByteLoad is true, do not
562/// unwrap inttoptr constant expressions. The caller would reconstruct those
563/// bits as a ConstantByte, dropping the pointer's provenance.
564bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, unsigned char *CurPtr,
565 unsigned BytesLeft, const DataLayout &DL,
566 bool IsByteLoad = false) {
567 assert(ByteOffset <= DL.getTypeAllocSize(C->getType()) &&
568 "Out of range access");
569
570 // Reading type padding, return zero.
571 if (ByteOffset >= DL.getTypeStoreSize(C->getType()))
572 return true;
573
574 // If this element is zero or undefined, we can just return since *CurPtr is
575 // zero initialized.
577 return true;
578
579 auto *CI = dyn_cast<ConstantInt>(C);
580 if (CI && CI->getType()->isIntegerTy()) {
581 if ((CI->getBitWidth() & 7) != 0)
582 return false;
583 const APInt &Val = CI->getValue();
584 unsigned IntBytes = unsigned(CI->getBitWidth()/8);
585
586 for (unsigned i = 0; i != BytesLeft && ByteOffset != IntBytes; ++i) {
587 unsigned n = ByteOffset;
588 if (!DL.isLittleEndian())
589 n = IntBytes - n - 1;
590 CurPtr[i] = Val.extractBits(8, n * 8).getZExtValue();
591 ++ByteOffset;
592 }
593 return true;
594 }
595
596 // Read a `ConstantByte` as an integer with the same bits.
597 auto *CB = dyn_cast<ConstantByte>(C);
598 if (CB && CB->getType()->isByteTy())
599 return ReadDataFromGlobal(ConstantInt::get(C->getContext(), CB->getValue()),
600 ByteOffset, CurPtr, BytesLeft, DL, IsByteLoad);
601
602 auto *CFP = dyn_cast<ConstantFP>(C);
603 if (CFP && CFP->getType()->isFloatingPointTy()) {
604 if (CFP->getType()->isDoubleTy()) {
605 C = FoldBitCast(C, Type::getInt64Ty(C->getContext()), DL);
606 return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, DL,
607 IsByteLoad);
608 }
609 if (CFP->getType()->isFloatTy()){
610 C = FoldBitCast(C, Type::getInt32Ty(C->getContext()), DL);
611 return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, DL,
612 IsByteLoad);
613 }
614 if (CFP->getType()->isHalfTy()){
615 C = FoldBitCast(C, Type::getInt16Ty(C->getContext()), DL);
616 return ReadDataFromGlobal(C, ByteOffset, CurPtr, BytesLeft, DL,
617 IsByteLoad);
618 }
619 return false;
620 }
621
622 if (auto *CS = dyn_cast<ConstantStruct>(C)) {
623 const StructLayout *SL = DL.getStructLayout(CS->getType());
624 unsigned Index = SL->getElementContainingOffset(ByteOffset);
625 uint64_t CurEltOffset = SL->getElementOffset(Index);
626 ByteOffset -= CurEltOffset;
627
628 while (true) {
629 // If the element access is to the element itself and not to tail padding,
630 // read the bytes from the element.
631 uint64_t EltSize = DL.getTypeAllocSize(CS->getOperand(Index)->getType());
632
633 if (ByteOffset < EltSize &&
634 !ReadDataFromGlobal(CS->getOperand(Index), ByteOffset, CurPtr,
635 BytesLeft, DL, IsByteLoad))
636 return false;
637
638 ++Index;
639
640 // Check to see if we read from the last struct element, if so we're done.
641 if (Index == CS->getType()->getNumElements())
642 return true;
643
644 // If we read all of the bytes we needed from this element we're done.
645 uint64_t NextEltOffset = SL->getElementOffset(Index);
646
647 if (BytesLeft <= NextEltOffset - CurEltOffset - ByteOffset)
648 return true;
649
650 // Move to the next element of the struct.
651 CurPtr += NextEltOffset - CurEltOffset - ByteOffset;
652 BytesLeft -= NextEltOffset - CurEltOffset - ByteOffset;
653 ByteOffset = 0;
654 CurEltOffset = NextEltOffset;
655 }
656 // not reached.
657 }
658
662 uint64_t NumElts, EltSize;
663 Type *EltTy;
664 if (auto *AT = dyn_cast<ArrayType>(C->getType())) {
665 NumElts = AT->getNumElements();
666 EltTy = AT->getElementType();
667 EltSize = DL.getTypeAllocSize(EltTy);
668 } else {
669 NumElts = cast<FixedVectorType>(C->getType())->getNumElements();
670 EltTy = cast<FixedVectorType>(C->getType())->getElementType();
671 // TODO: For non-byte-sized vectors, current implementation assumes there is
672 // padding to the next byte boundary between elements.
673 if (!DL.typeSizeEqualsStoreSize(EltTy))
674 return false;
675
676 EltSize = DL.getTypeStoreSize(EltTy);
677 }
678 uint64_t Index = ByteOffset / EltSize;
679 uint64_t Offset = ByteOffset - Index * EltSize;
680
681 for (; Index != NumElts; ++Index) {
682 if (!ReadDataFromGlobal(C->getAggregateElement(Index), Offset, CurPtr,
683 BytesLeft, DL, IsByteLoad))
684 return false;
685
686 uint64_t BytesWritten = EltSize - Offset;
687 assert(BytesWritten <= EltSize && "Not indexing into this element?");
688 if (BytesWritten >= BytesLeft)
689 return true;
690
691 Offset = 0;
692 BytesLeft -= BytesWritten;
693 CurPtr += BytesWritten;
694 }
695 return true;
696 }
697
698 if (auto *CE = dyn_cast<ConstantExpr>(C)) {
699 if (CE->getOpcode() == Instruction::IntToPtr &&
700 CE->getOperand(0)->getType() == DL.getIntPtrType(CE->getType())) {
701 // Folding byte loads through the integer operand would rebuild the result
702 // as a `ConstantByte`, dropping the pointer's provenance.
703 if (IsByteLoad)
704 return false;
705 return ReadDataFromGlobal(CE->getOperand(0), ByteOffset, CurPtr,
706 BytesLeft, DL, IsByteLoad);
707 }
708 }
709
710 // Otherwise, unknown initializer type.
711 return false;
712}
713
714/// OrigLoadTy is the original type being loaded, while LoadTy is the type
715/// currently being folded (which may be integer type mapped from OrigLoadTy).
716Constant *FoldReinterpretLoadFromConst(Constant *C, Type *LoadTy,
717 Type *OrigLoadTy, int64_t Offset,
718 const DataLayout &DL) {
719 // Bail out early. Not expect to load from scalable global variable.
720 if (isa<ScalableVectorType>(LoadTy))
721 return nullptr;
722
723 auto *IntType = dyn_cast<IntegerType>(LoadTy);
724
725 // If this isn't an integer load we can't fold it directly.
726 if (!IntType) {
727 // If this is a non-integer load, we can try folding it as an int load and
728 // then bitcast the result. This can be useful for union cases. Note
729 // that address spaces don't matter here since we're not going to result in
730 // an actual new load.
731 if (!LoadTy->isFloatingPointTy() && !LoadTy->isPointerTy() &&
732 !LoadTy->isByteTy() && !LoadTy->isVectorTy())
733 return nullptr;
734
735 // Reinterpret pointers read from `ConstantByte` as a bitcast, rather than
736 // an inttoptr.
737 if (LoadTy->isPtrOrPtrVectorTy() &&
738 !DL.isNonIntegralPointerType(LoadTy->getScalarType()) &&
739 containsByteType(C->getType())) {
740 Type *ByteTy = DL.getBytePtrType(LoadTy);
741 if (Constant *Res =
742 FoldReinterpretLoadFromConst(C, ByteTy, ByteTy, Offset, DL))
743 return ConstantFoldCastOperand(Instruction::BitCast, Res, LoadTy, DL);
744 }
745
746 Type *MapTy = Type::getIntNTy(C->getContext(),
747 DL.getTypeSizeInBits(LoadTy).getFixedValue());
748 if (Constant *Res =
749 FoldReinterpretLoadFromConst(C, MapTy, OrigLoadTy, Offset, DL)) {
750 if (Res->isNullValue() && !LoadTy->isX86_AMXTy())
751 // Materializing a zero can be done trivially without a bitcast
752 return Constant::getNullValue(LoadTy);
753 Type *CastTy = LoadTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(LoadTy) : LoadTy;
754 Res = FoldBitCast(Res, CastTy, DL);
755 if (LoadTy->isPtrOrPtrVectorTy()) {
756 // For vector of pointer, we needed to first convert to a vector of integer, then do vector inttoptr
757 if (Res->isNullValue() && !LoadTy->isX86_AMXTy())
758 return Constant::getNullValue(LoadTy);
759 if (DL.isNonIntegralPointerType(LoadTy->getScalarType()))
760 // Be careful not to replace a load of an addrspace value with an inttoptr here
761 return nullptr;
762 Res = ConstantExpr::getIntToPtr(Res, LoadTy);
763 }
764 return Res;
765 }
766 return nullptr;
767 }
768
769 unsigned BytesLoaded = (IntType->getBitWidth() + 7) / 8;
770 // Allow folding of large type loads (e.g. <16 x double>).
771 if (BytesLoaded > 128 || BytesLoaded == 0)
772 return nullptr;
773
774 // For scalar integer load, use smaller limit to avoid regression during
775 // memcmp expansion. Codegen may generate inefficient string operations.
776 if (BytesLoaded > 32 && OrigLoadTy->isIntegerTy())
777 return nullptr;
778
779 // If we're not accessing anything in this constant, the result is undefined.
780 if (Offset <= -1 * static_cast<int64_t>(BytesLoaded))
781 return PoisonValue::get(IntType);
782
783 // TODO: We should be able to support scalable types.
784 TypeSize InitializerSize = DL.getTypeAllocSize(C->getType());
785 if (InitializerSize.isScalable())
786 return nullptr;
787
788 // If we're not accessing anything in this constant, the result is undefined.
789 if (Offset >= (int64_t)InitializerSize.getFixedValue())
790 return PoisonValue::get(IntType);
791
792 SmallVector<unsigned char, 64> RawBytes(BytesLoaded);
793 unsigned char *CurPtr = RawBytes.data();
794 unsigned BytesLeft = BytesLoaded;
795
796 // If we're loading off the beginning of the global, some bytes may be valid.
797 if (Offset < 0) {
798 CurPtr += -Offset;
799 BytesLeft += Offset;
800 Offset = 0;
801 }
802
803 if (!ReadDataFromGlobal(C, Offset, CurPtr, BytesLeft, DL,
804 /*IsByteLoad=*/OrigLoadTy->isByteOrByteVectorTy()))
805 return nullptr;
806
807 APInt ResultVal = APInt(IntType->getBitWidth(), 0);
808 if (DL.isLittleEndian()) {
809 ResultVal = RawBytes[BytesLoaded - 1];
810 for (unsigned i = 1; i != BytesLoaded; ++i) {
811 ResultVal <<= 8;
812 ResultVal |= RawBytes[BytesLoaded - 1 - i];
813 }
814 } else {
815 ResultVal = RawBytes[0];
816 for (unsigned i = 1; i != BytesLoaded; ++i) {
817 ResultVal <<= 8;
818 ResultVal |= RawBytes[i];
819 }
820 }
821
822 return ConstantInt::get(IntType->getContext(), ResultVal);
823}
824
825} // anonymous namespace
826
827// If GV is a constant with an initializer read its representation starting
828// at Offset and return it as a constant array of unsigned char. Otherwise
829// return null.
831 uint64_t Offset) {
832 if (!GV->isConstant() || !GV->hasDefinitiveInitializer())
833 return nullptr;
834
835 const DataLayout &DL = GV->getDataLayout();
836 Constant *Init = const_cast<Constant *>(GV->getInitializer());
837 TypeSize InitSize = DL.getTypeAllocSize(Init->getType());
838 if (InitSize < Offset)
839 return nullptr;
840
841 uint64_t NBytes = InitSize - Offset;
842 if (NBytes > UINT16_MAX)
843 // Bail for large initializers in excess of 64K to avoid allocating
844 // too much memory.
845 // Offset is assumed to be less than or equal than InitSize (this
846 // is enforced in ReadDataFromGlobal).
847 return nullptr;
848
849 SmallVector<unsigned char, 256> RawBytes(static_cast<size_t>(NBytes));
850 unsigned char *CurPtr = RawBytes.data();
851
852 if (!ReadDataFromGlobal(Init, Offset, CurPtr, NBytes, DL))
853 return nullptr;
854
855 return ConstantDataArray::get(GV->getContext(), RawBytes);
856}
857
858/// If this Offset points exactly to the start of an aggregate element, return
859/// that element, otherwise return nullptr.
861 const DataLayout &DL) {
862 if (Offset.isZero())
863 return Base;
864
866 return nullptr;
867
868 Type *ElemTy = Base->getType();
869 SmallVector<APInt> Indices = DL.getGEPIndicesForOffset(ElemTy, Offset);
870 if (!Offset.isZero() || !Indices[0].isZero())
871 return nullptr;
872
873 Constant *C = Base;
874 for (const APInt &Index : drop_begin(Indices)) {
875 if (Index.isNegative() || Index.getActiveBits() >= 32)
876 return nullptr;
877
878 C = C->getAggregateElement(Index.getZExtValue());
879 if (!C)
880 return nullptr;
881 }
882
883 return C;
884}
885
887 const APInt &Offset,
888 const DataLayout &DL) {
889 if (Constant *AtOffset = getConstantAtOffset(C, Offset, DL))
890 if (Constant *Result = ConstantFoldLoadThroughBitcast(AtOffset, Ty, DL))
891 return Result;
892
893 // Explicitly check for out-of-bounds access, so we return poison even if the
894 // constant is a uniform value.
895 TypeSize Size = DL.getTypeAllocSize(C->getType());
896 if (!Size.isScalable() && Offset.sge(Size.getFixedValue()))
897 return PoisonValue::get(Ty);
898
899 // Try an offset-independent fold of a uniform value.
900 if (Constant *Result = ConstantFoldLoadFromUniformValue(C, Ty, DL))
901 return Result;
902
903 // Try hard to fold loads from bitcasted strange and non-type-safe things.
904 if (Offset.getSignificantBits() <= 64)
905 if (Constant *Result =
906 FoldReinterpretLoadFromConst(C, Ty, Ty, Offset.getSExtValue(), DL))
907 return Result;
908
909 return nullptr;
910}
911
916
919 const DataLayout &DL) {
920 // We can only fold loads from constant globals with a definitive initializer.
921 // Check this upfront, to skip expensive offset calculations.
923 if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
924 return nullptr;
925
926 C = cast<Constant>(C->stripAndAccumulateConstantOffsets(
927 DL, Offset, /* AllowNonInbounds */ true));
928
929 if (C == GV)
930 if (Constant *Result = ConstantFoldLoadFromConst(GV->getInitializer(), Ty,
931 Offset, DL))
932 return Result;
933
934 // If this load comes from anywhere in a uniform constant global, the value
935 // is always the same, regardless of the loaded offset.
936 return ConstantFoldLoadFromUniformValue(GV->getInitializer(), Ty, DL);
937}
938
940 const DataLayout &DL) {
941 APInt Offset(DL.getIndexTypeSizeInBits(C->getType()), 0);
942 return ConstantFoldLoadFromConstPtr(C, Ty, std::move(Offset), DL);
943}
944
946 const DataLayout &DL) {
947 if (isa<PoisonValue>(C))
948 return PoisonValue::get(Ty);
949 if (isa<UndefValue>(C))
950 return UndefValue::get(Ty);
951 // If padding is needed when storing C to memory, then it isn't considered as
952 // uniform.
953 if (!DL.typeSizeEqualsStoreSize(C->getType()))
954 return nullptr;
955 if (C->isNullValue() && !Ty->isX86_AMXTy())
956 return Constant::getNullValue(Ty);
957 if (C->isAllOnesValue() &&
958 (Ty->isIntOrIntVectorTy() || Ty->isByteOrByteVectorTy() ||
959 Ty->isFPOrFPVectorTy()))
960 return Constant::getAllOnesValue(Ty);
961 return nullptr;
962}
963
964namespace {
965
966/// One of Op0/Op1 is a constant expression.
967/// Attempt to symbolically evaluate the result of a binary operator merging
968/// these together. If target data info is available, it is provided as DL,
969/// otherwise DL is null.
970Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0, Constant *Op1,
971 const DataLayout &DL) {
972 // SROA
973
974 // Fold (and 0xffffffff00000000, (shl x, 32)) -> shl.
975 // Fold (lshr (or X, Y), 32) -> (lshr [X/Y], 32) if one doesn't contribute
976 // bits.
977
978 if (Opc == Instruction::And) {
979 KnownBits Known0 = computeKnownBits(Op0, DL);
980 KnownBits Known1 = computeKnownBits(Op1, DL);
981 if ((Known1.One | Known0.Zero).isAllOnes()) {
982 // All the bits of Op0 that the 'and' could be masking are already zero.
983 return Op0;
984 }
985 if ((Known0.One | Known1.Zero).isAllOnes()) {
986 // All the bits of Op1 that the 'and' could be masking are already zero.
987 return Op1;
988 }
989
990 Known0 &= Known1;
991 if (Known0.isConstant())
992 return ConstantInt::get(Op0->getType(), Known0.getConstant());
993 }
994
995 // If the constant expr is something like &A[123] - &A[4].f, fold this into a
996 // constant. This happens frequently when iterating over a global array.
997 if (Opc == Instruction::Sub) {
998 GlobalValue *GV1, *GV2;
999 APInt Offs1, Offs2;
1000
1001 if (IsConstantOffsetFromGlobal(Op0, GV1, Offs1, DL))
1002 if (IsConstantOffsetFromGlobal(Op1, GV2, Offs2, DL) && GV1 == GV2) {
1003 unsigned OpSize = DL.getTypeSizeInBits(Op0->getType());
1004
1005 // (&GV+C1) - (&GV+C2) -> C1-C2, pointer arithmetic cannot overflow.
1006 // PtrToInt may change the bitwidth so we have convert to the right size
1007 // first.
1008 return ConstantInt::get(Op0->getType(), Offs1.zextOrTrunc(OpSize) -
1009 Offs2.zextOrTrunc(OpSize));
1010 }
1011 }
1012
1013 return nullptr;
1014}
1015
1016/// If array indices are not pointer-sized integers, explicitly cast them so
1017/// that they aren't implicitly casted by the getelementptr.
1018Constant *CastGEPIndices(Type *SrcElemTy, ArrayRef<Constant *> Ops,
1019 Type *ResultTy, GEPNoWrapFlags NW,
1020 std::optional<ConstantRange> InRange,
1021 const DataLayout &DL, const TargetLibraryInfo *TLI) {
1022 Type *IntIdxTy = DL.getIndexType(ResultTy);
1023 Type *IntIdxScalarTy = IntIdxTy->getScalarType();
1024
1025 bool Any = false;
1027 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
1028 if ((i == 1 ||
1030 SrcElemTy, Ops.slice(1, i - 1)))) &&
1031 Ops[i]->getType()->getScalarType() != IntIdxScalarTy) {
1032 Any = true;
1033 Type *NewType =
1034 Ops[i]->getType()->isVectorTy() ? IntIdxTy : IntIdxScalarTy;
1036 CastInst::getCastOpcode(Ops[i], true, NewType, true), Ops[i], NewType,
1037 DL);
1038 if (!NewIdx)
1039 return nullptr;
1040 NewIdxs.push_back(NewIdx);
1041 } else
1042 NewIdxs.push_back(Ops[i]);
1043 }
1044
1045 if (!Any)
1046 return nullptr;
1047
1048 Constant *C = ConstantExpr::getGetElementPtr(DL, SrcElemTy, Ops[0], NewIdxs,
1049 NW, InRange);
1050 if (!C)
1051 return nullptr;
1052 return ConstantFoldConstant(C, DL, TLI);
1053}
1054
1055/// If we can symbolically evaluate the GEP constant expression, do so.
1056Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
1058 const DataLayout &DL,
1059 const TargetLibraryInfo *TLI) {
1060 Type *SrcElemTy = GEP->getSourceElementType();
1061 Type *ResTy = GEP->getType();
1062 if (!SrcElemTy->isSized() || isa<ScalableVectorType>(SrcElemTy))
1063 return nullptr;
1064
1065 if (Constant *C = CastGEPIndices(SrcElemTy, Ops, ResTy, GEP->getNoWrapFlags(),
1066 GEP->getInRange(), DL, TLI))
1067 return C;
1068
1069 Constant *Ptr = Ops[0];
1070 if (!Ptr->getType()->isPointerTy())
1071 return nullptr;
1072
1073 Type *IntIdxTy = DL.getIndexType(Ptr->getType());
1074
1075 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
1076 if (!isa<ConstantInt>(Ops[i]) || !Ops[i]->getType()->isIntegerTy())
1077 return nullptr;
1078
1079 unsigned BitWidth = DL.getTypeSizeInBits(IntIdxTy);
1080 APInt Offset(BitWidth, 0);
1082 reinterpret_cast<const Value *const *>(Ops.data()) + 1, Ops.size() - 1);
1083 if (!GEPOperator::accumulateConstantOffset(SrcElemTy, Indices, DL, Offset))
1084 return nullptr;
1085
1086 std::optional<ConstantRange> InRange = GEP->getInRange();
1087 if (InRange)
1088 InRange = InRange->sextOrTrunc(BitWidth);
1089
1090 // If this is a GEP of a GEP, fold it all into a single GEP.
1091 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
1092 bool Overflow = false;
1093 while (auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
1094 NW &= GEP->getNoWrapFlags();
1095
1096 // Do not try to incorporate the sub-GEP if some index is not a number.
1097 APInt NestedOffset(BitWidth, 0);
1098 if (!GEP->accumulateConstantOffset(DL, NestedOffset))
1099 break;
1100
1101 // Adjust inrange offset and intersect inrange attributes
1102 if (auto GEPRange = GEP->getInRange()) {
1103 auto AdjustedGEPRange = GEPRange->sextOrTrunc(BitWidth).subtract(Offset);
1104 InRange =
1105 InRange ? InRange->intersectWith(AdjustedGEPRange) : AdjustedGEPRange;
1106 }
1107
1108 Ptr = cast<Constant>(GEP->getOperand(0));
1109 SrcElemTy = GEP->getSourceElementType();
1110 Offset = Offset.sadd_ov(NestedOffset, Overflow);
1111 }
1112
1113 // Preserving nusw (without inbounds) also requires that the offset
1114 // additions did not overflow.
1115 if (NW.hasNoUnsignedSignedWrap() && !NW.isInBounds() && Overflow)
1117
1118 // If the base value for this address is a literal integer value, fold the
1119 // getelementptr to the resulting integer value casted to the pointer type.
1120 APInt BaseIntVal(DL.getPointerTypeSizeInBits(Ptr->getType()), 0);
1121 if (auto *CE = dyn_cast<ConstantExpr>(Ptr)) {
1122 if (CE->getOpcode() == Instruction::IntToPtr) {
1123 if (auto *Base = dyn_cast<ConstantInt>(CE->getOperand(0)))
1124 BaseIntVal = Base->getValue().zextOrTrunc(BaseIntVal.getBitWidth());
1125 }
1126 }
1127
1128 if ((Ptr->isNullValue() || BaseIntVal != 0) &&
1129 !DL.mustNotIntroduceIntToPtr(Ptr->getType())) {
1130
1131 // If the index size is smaller than the pointer size, add to the low
1132 // bits only.
1133 BaseIntVal.insertBits(BaseIntVal.trunc(BitWidth) + Offset, 0);
1134 Constant *C = ConstantInt::get(Ptr->getContext(), BaseIntVal);
1135 return ConstantExpr::getIntToPtr(C, ResTy);
1136 }
1137
1138 // Try to infer inbounds for GEPs of globals.
1139 if (!NW.isInBounds() && Offset.isNonNegative()) {
1140 bool CanBeNull;
1141 uint64_t DerefBytes = Ptr->getPointerDereferenceableBytes(
1142 DL, CanBeNull, /*CanBeFreed=*/nullptr);
1143 if (DerefBytes != 0 && !CanBeNull && Offset.sle(DerefBytes))
1145 }
1146
1147 // nusw + nneg -> nuw
1148 if (NW.hasNoUnsignedSignedWrap() && Offset.isNonNegative())
1150
1151 // Otherwise canonicalize this to a single ptradd.
1152 LLVMContext &Ctx = Ptr->getContext();
1153 return ConstantExpr::getPtrAdd(Ptr, ConstantInt::get(Ctx, Offset), NW,
1154 InRange);
1155}
1156
1157/// Attempt to constant fold an instruction with the
1158/// specified opcode and operands. If successful, the constant result is
1159/// returned, if not, null is returned. Note that this function can fail when
1160/// attempting to fold instructions like loads and stores, which have no
1161/// constant expression form.
1162Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
1164 const DataLayout &DL,
1165 const TargetLibraryInfo *TLI,
1166 bool AllowNonDeterministic) {
1167 Type *DestTy = InstOrCE->getType();
1168
1169 if (Instruction::isUnaryOp(Opcode))
1170 return ConstantFoldUnaryOpOperand(Opcode, Ops[0], DL);
1171
1172 if (Instruction::isBinaryOp(Opcode)) {
1173 switch (Opcode) {
1174 default:
1175 break;
1176 case Instruction::FAdd:
1177 case Instruction::FSub:
1178 case Instruction::FMul:
1179 case Instruction::FDiv:
1180 case Instruction::FRem:
1181 // Handle floating point instructions separately to account for denormals
1182 // TODO: If a constant expression is being folded rather than an
1183 // instruction, denormals will not be flushed/treated as zero
1184 if (const auto *I = dyn_cast<Instruction>(InstOrCE)) {
1185 return ConstantFoldFPInstOperands(Opcode, Ops[0], Ops[1], DL, I,
1186 AllowNonDeterministic);
1187 }
1188 }
1189 return ConstantFoldBinaryOpOperands(Opcode, Ops[0], Ops[1], DL);
1190 }
1191
1192 if (Instruction::isCast(Opcode))
1193 return ConstantFoldCastOperand(Opcode, Ops[0], DestTy, DL);
1194
1195 if (auto *GEP = dyn_cast<GEPOperator>(InstOrCE)) {
1196 Type *SrcElemTy = GEP->getSourceElementType();
1198 return nullptr;
1199
1200 if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI))
1201 return C;
1202
1203 return ConstantExpr::getGetElementPtr(DL, SrcElemTy, Ops[0], Ops.slice(1),
1204 GEP->getNoWrapFlags(),
1205 GEP->getInRange());
1206 }
1207
1208 if (auto *CE = dyn_cast<ConstantExpr>(InstOrCE))
1209 return CE->getWithOperands(Ops);
1210
1211 switch (Opcode) {
1212 default: return nullptr;
1213 case Instruction::ICmp:
1214 case Instruction::FCmp: {
1215 auto *C = cast<CmpInst>(InstOrCE);
1216 return ConstantFoldCompareInstOperands(C->getPredicate(), Ops[0], Ops[1],
1217 DL, TLI, C->getFunction());
1218 }
1219 case Instruction::Freeze:
1220 return isGuaranteedNotToBeUndefOrPoison(Ops[0]) ? Ops[0] : nullptr;
1221 case Instruction::Call:
1222 if (auto *F = dyn_cast<Function>(Ops.back())) {
1223 const auto *Call = cast<CallBase>(InstOrCE);
1225 return ConstantFoldCall(Call, F, Ops.slice(0, Ops.size() - 1), TLI,
1226 AllowNonDeterministic);
1227 }
1228 return nullptr;
1229 case Instruction::Select:
1230 return ConstantFoldSelectInstruction(Ops[0], Ops[1], Ops[2]);
1231 case Instruction::ExtractElement:
1233 case Instruction::ExtractValue:
1235 Ops[0], cast<ExtractValueInst>(InstOrCE)->getIndices());
1236 case Instruction::InsertElement:
1237 return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]);
1238 case Instruction::InsertValue:
1240 Ops[0], Ops[1], cast<InsertValueInst>(InstOrCE)->getIndices());
1241 case Instruction::ShuffleVector:
1243 Ops[0], Ops[1], cast<ShuffleVectorInst>(InstOrCE)->getShuffleMask());
1244 case Instruction::BitInsert:
1245 return ConstantFoldBitInsertOperands(Ops[0], Ops[1], Ops[2], DL);
1246 case Instruction::BitExtract:
1247 return ConstantFoldBitExtractOperands(InstOrCE->getType(), Ops[0], Ops[1],
1248 DL);
1249 case Instruction::Load: {
1250 const auto *LI = dyn_cast<LoadInst>(InstOrCE);
1251 if (LI->isVolatile())
1252 return nullptr;
1253 return ConstantFoldLoadFromConstPtr(Ops[0], LI->getType(), DL);
1254 }
1255 }
1256}
1257
1258} // end anonymous namespace
1259
1260//===----------------------------------------------------------------------===//
1261// Constant Folding public APIs
1262//===----------------------------------------------------------------------===//
1263
1264namespace {
1265
1266Constant *
1267ConstantFoldConstantImpl(const Constant *C, const DataLayout &DL,
1268 const TargetLibraryInfo *TLI,
1271 return const_cast<Constant *>(C);
1272
1274 for (const Use &OldU : C->operands()) {
1275 Constant *OldC = cast<Constant>(&OldU);
1276 Constant *NewC = OldC;
1277 // Recursively fold the ConstantExpr's operands. If we have already folded
1278 // a ConstantExpr, we don't have to process it again.
1279 if (isa<ConstantVector>(OldC) || isa<ConstantExpr>(OldC)) {
1280 auto It = FoldedOps.find(OldC);
1281 if (It == FoldedOps.end()) {
1282 NewC = ConstantFoldConstantImpl(OldC, DL, TLI, FoldedOps);
1283 FoldedOps.insert({OldC, NewC});
1284 } else {
1285 NewC = It->second;
1286 }
1287 }
1288 Ops.push_back(NewC);
1289 }
1290
1291 if (auto *CE = dyn_cast<ConstantExpr>(C)) {
1292 if (Constant *Res = ConstantFoldInstOperandsImpl(
1293 CE, CE->getOpcode(), Ops, DL, TLI, /*AllowNonDeterministic=*/true))
1294 return Res;
1295 return const_cast<Constant *>(C);
1296 }
1297
1299 return ConstantVector::get(Ops);
1300}
1301
1302} // end anonymous namespace
1303
1305 const DataLayout &DL,
1306 const TargetLibraryInfo *TLI) {
1307 // Handle PHI nodes quickly here...
1308 if (auto *PN = dyn_cast<PHINode>(I)) {
1309 Constant *CommonValue = nullptr;
1310
1312 for (Value *Incoming : PN->incoming_values()) {
1313 // If the incoming value is undef then skip it. Note that while we could
1314 // skip the value if it is equal to the phi node itself we choose not to
1315 // because that would break the rule that constant folding only applies if
1316 // all operands are constants.
1317 if (isa<UndefValue>(Incoming))
1318 continue;
1319 // If the incoming value is not a constant, then give up.
1320 auto *C = dyn_cast<Constant>(Incoming);
1321 if (!C)
1322 return nullptr;
1323 // Fold the PHI's operands.
1324 C = ConstantFoldConstantImpl(C, DL, TLI, FoldedOps);
1325 // If the incoming value is a different constant to
1326 // the one we saw previously, then give up.
1327 if (CommonValue && C != CommonValue)
1328 return nullptr;
1329 CommonValue = C;
1330 }
1331
1332 // If we reach here, all incoming values are the same constant or undef.
1333 return CommonValue ? CommonValue : UndefValue::get(PN->getType());
1334 }
1335
1336 // Scan the operand list, checking to see if they are all constants, if so,
1337 // hand off to ConstantFoldInstOperandsImpl.
1338 if (!all_of(I->operands(), [](const Use &U) { return isa<Constant>(U); }))
1339 return nullptr;
1340
1343 for (const Use &OpU : I->operands()) {
1344 auto *Op = cast<Constant>(&OpU);
1345 // Fold the Instruction's operands.
1346 Op = ConstantFoldConstantImpl(Op, DL, TLI, FoldedOps);
1347 Ops.push_back(Op);
1348 }
1349
1350 return ConstantFoldInstOperands(I, Ops, DL, TLI);
1351}
1352
1354 const TargetLibraryInfo *TLI) {
1356 return ConstantFoldConstantImpl(C, DL, TLI, FoldedOps);
1357}
1358
1361 const DataLayout &DL,
1362 const TargetLibraryInfo *TLI,
1363 bool AllowNonDeterministic) {
1364 return ConstantFoldInstOperandsImpl(I, I->getOpcode(), Ops, DL, TLI,
1365 AllowNonDeterministic);
1366}
1367
1369 Constant *Ops0, Constant *Ops1,
1370 const DataLayout &DL,
1371 const TargetLibraryInfo *TLI,
1372 const Function *CtxF) {
1373 CmpInst::Predicate Predicate = (CmpInst::Predicate)IntPredicate;
1374 // fold: icmp (inttoptr x), null -> icmp x, 0
1375 // fold: icmp null, (inttoptr x) -> icmp 0, x
1376 // fold: icmp (ptrtoint x), 0 -> icmp x, null
1377 // fold: icmp 0, (ptrtoint x) -> icmp null, x
1378 // fold: icmp (inttoptr x), (inttoptr y) -> icmp trunc/zext x, trunc/zext y
1379 // fold: icmp (ptrtoint x), (ptrtoint y) -> icmp x, y
1380 //
1381 // FIXME: The following comment is out of data and the DataLayout is here now.
1382 // ConstantExpr::getCompare cannot do this, because it doesn't have DL
1383 // around to know if bit truncation is happening.
1384 if (auto *CE0 = dyn_cast<ConstantExpr>(Ops0)) {
1385 if (Ops1->isNullValue()) {
1386 if (CE0->getOpcode() == Instruction::IntToPtr) {
1387 Type *IntPtrTy = DL.getIntPtrType(CE0->getType());
1388 // Convert the integer value to the right size to ensure we get the
1389 // proper extension or truncation.
1390 if (Constant *C = ConstantFoldIntegerCast(CE0->getOperand(0), IntPtrTy,
1391 /*IsSigned*/ false, DL)) {
1392 Constant *Null = Constant::getNullValue(C->getType());
1393 return ConstantFoldCompareInstOperands(Predicate, C, Null, DL, TLI);
1394 }
1395 }
1396
1397 // icmp only compares the address part of the pointer, so only do this
1398 // transform if the integer size matches the address size.
1399 if (CE0->getOpcode() == Instruction::PtrToInt ||
1400 CE0->getOpcode() == Instruction::PtrToAddr) {
1401 Type *AddrTy = DL.getAddressType(CE0->getOperand(0)->getType());
1402 if (CE0->getType() == AddrTy) {
1403 Constant *C = CE0->getOperand(0);
1404 Constant *Null = Constant::getNullValue(C->getType());
1405 return ConstantFoldCompareInstOperands(Predicate, C, Null, DL, TLI);
1406 }
1407 }
1408 }
1409
1410 if (auto *CE1 = dyn_cast<ConstantExpr>(Ops1)) {
1411 if (CE0->getOpcode() == CE1->getOpcode()) {
1412 if (CE0->getOpcode() == Instruction::IntToPtr) {
1413 Type *IntPtrTy = DL.getIntPtrType(CE0->getType());
1414
1415 // Convert the integer value to the right size to ensure we get the
1416 // proper extension or truncation.
1417 Constant *C0 = ConstantFoldIntegerCast(CE0->getOperand(0), IntPtrTy,
1418 /*IsSigned*/ false, DL);
1419 Constant *C1 = ConstantFoldIntegerCast(CE1->getOperand(0), IntPtrTy,
1420 /*IsSigned*/ false, DL);
1421 if (C0 && C1)
1422 return ConstantFoldCompareInstOperands(Predicate, C0, C1, DL, TLI);
1423 }
1424
1425 // icmp only compares the address part of the pointer, so only do this
1426 // transform if the integer size matches the address size.
1427 if (CE0->getOpcode() == Instruction::PtrToInt ||
1428 CE0->getOpcode() == Instruction::PtrToAddr) {
1429 Type *AddrTy = DL.getAddressType(CE0->getOperand(0)->getType());
1430 if (CE0->getType() == AddrTy &&
1431 CE0->getOperand(0)->getType() == CE1->getOperand(0)->getType()) {
1433 Predicate, CE0->getOperand(0), CE1->getOperand(0), DL, TLI);
1434 }
1435 }
1436 }
1437 }
1438
1439 // Convert pointer comparison (base+offset1) pred (base+offset2) into
1440 // offset1 pred offset2, for the case where the offset is inbounds. This
1441 // only works for equality and unsigned comparison, as inbounds permits
1442 // crossing the sign boundary. However, the offset comparison itself is
1443 // signed.
1444 if (Ops0->getType()->isPointerTy() && !ICmpInst::isSigned(Predicate)) {
1445 unsigned IndexWidth = DL.getIndexTypeSizeInBits(Ops0->getType());
1446 APInt Offset0(IndexWidth, 0);
1447 bool IsEqPred = ICmpInst::isEquality(Predicate);
1448 Value *Stripped0 = Ops0->stripAndAccumulateConstantOffsets(
1449 DL, Offset0, /*AllowNonInbounds=*/IsEqPred,
1450 /*AllowInvariantGroup=*/false, /*ExternalAnalysis=*/nullptr,
1451 /*LookThroughIntToPtr=*/IsEqPred);
1452 APInt Offset1(IndexWidth, 0);
1453 Value *Stripped1 = Ops1->stripAndAccumulateConstantOffsets(
1454 DL, Offset1, /*AllowNonInbounds=*/IsEqPred,
1455 /*AllowInvariantGroup=*/false, /*ExternalAnalysis=*/nullptr,
1456 /*LookThroughIntToPtr=*/IsEqPred);
1457 if (Stripped0 == Stripped1)
1458 return ConstantInt::getBool(
1459 Ops0->getContext(),
1460 ICmpInst::compare(Offset0, Offset1,
1461 ICmpInst::getSignedPredicate(Predicate)));
1462 }
1463 } else if (isa<ConstantExpr>(Ops1)) {
1464 // If RHS is a constant expression, but the left side isn't, swap the
1465 // operands and try again.
1466 Predicate = ICmpInst::getSwappedPredicate(Predicate);
1467 return ConstantFoldCompareInstOperands(Predicate, Ops1, Ops0, DL, TLI);
1468 }
1469
1470 if (CmpInst::isFPPredicate(Predicate)) {
1471 // Flush any denormal constant float input according to denormal handling
1472 // mode.
1473 Ops0 = FlushFPConstant(Ops0, CtxF, /*IsOutput=*/false);
1474 if (!Ops0)
1475 return nullptr;
1476 Ops1 = FlushFPConstant(Ops1, CtxF, /*IsOutput=*/false);
1477 if (!Ops1)
1478 return nullptr;
1479 }
1480
1481 return ConstantFoldCompareInstruction(Predicate, Ops0, Ops1);
1482}
1483
1485 const DataLayout &DL) {
1487
1488 return ConstantFoldUnaryInstruction(Opcode, Op);
1489}
1490
1492 Constant *RHS,
1493 const DataLayout &DL) {
1495 if (isa<ConstantExpr>(LHS) || isa<ConstantExpr>(RHS))
1496 if (Constant *C = SymbolicallyEvaluateBinop(Opcode, LHS, RHS, DL))
1497 return C;
1498
1500 return ConstantExpr::get(Opcode, LHS, RHS);
1501 return ConstantFoldBinaryInstruction(Opcode, LHS, RHS);
1502}
1503
1506 switch (Mode) {
1508 return nullptr;
1509 case DenormalMode::IEEE:
1510 return ConstantFP::get(Ty, APF);
1512 return ConstantFP::get(
1513 Ty, APFloat::getZero(APF.getSemantics(), APF.isNegative()));
1515 return ConstantFP::get(Ty, APFloat::getZero(APF.getSemantics(), false));
1516 default:
1517 break;
1518 }
1519
1520 llvm_unreachable("unknown denormal mode");
1521}
1522
1523/// Return the denormal mode that can be assumed when executing a floating point
1524/// operation at \p CtxI.
1526 if (!CtxF)
1527 return DenormalMode::getDynamic();
1528 return CtxF->getDenormalMode(Ty->getScalarType()->getFltSemantics());
1529}
1530
1531static ConstantFP *
1532flushDenormalConstantFP(ConstantFP *CFP, const Function *CtxF, bool IsOutput) {
1533 const APFloat &APF = CFP->getValueAPF();
1534 if (!APF.isDenormal())
1535 return CFP;
1536
1538 return flushDenormalConstant(CFP->getType(), APF,
1539 IsOutput ? Mode.Output : Mode.Input);
1540}
1541
1543 bool IsOutput) {
1544 if (ConstantFP *CFP = dyn_cast<ConstantFP>(Operand))
1545 return flushDenormalConstantFP(CFP, CtxF, IsOutput);
1546
1548 return Operand;
1549
1550 Type *Ty = Operand->getType();
1551 VectorType *VecTy = dyn_cast<VectorType>(Ty);
1552 if (VecTy) {
1553 if (auto *Splat = dyn_cast_or_null<ConstantFP>(Operand->getSplatValue())) {
1554 ConstantFP *Folded = flushDenormalConstantFP(Splat, CtxF, IsOutput);
1555 if (!Folded)
1556 return nullptr;
1558 }
1559
1560 Ty = VecTy->getElementType();
1561 }
1562
1563 if (isa<ConstantExpr>(Operand))
1564 return Operand;
1565
1566 if (const auto *CV = dyn_cast<ConstantVector>(Operand)) {
1568 for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
1569 Constant *Element = CV->getAggregateElement(i);
1570 if (isa<UndefValue>(Element)) {
1571 NewElts.push_back(Element);
1572 continue;
1573 }
1574
1575 ConstantFP *CFP = dyn_cast<ConstantFP>(Element);
1576 if (!CFP)
1577 return nullptr;
1578
1579 ConstantFP *Folded = flushDenormalConstantFP(CFP, CtxF, IsOutput);
1580 if (!Folded)
1581 return nullptr;
1582 NewElts.push_back(Folded);
1583 }
1584
1585 return ConstantVector::get(NewElts);
1586 }
1587
1588 if (const auto *CDV = dyn_cast<ConstantDataVector>(Operand)) {
1590 for (unsigned I = 0, E = CDV->getNumElements(); I < E; ++I) {
1591 const APFloat &Elt = CDV->getElementAsAPFloat(I);
1592 if (!Elt.isDenormal()) {
1593 NewElts.push_back(ConstantFP::get(Ty, Elt));
1594 } else {
1595 DenormalMode Mode = getInstrDenormalMode(CtxF, Ty);
1597 flushDenormalConstant(Ty, Elt, IsOutput ? Mode.Output : Mode.Input);
1598 if (!Folded)
1599 return nullptr;
1600 NewElts.push_back(Folded);
1601 }
1602 }
1603
1604 return ConstantVector::get(NewElts);
1605 }
1606
1607 return nullptr;
1608}
1609
1611 Constant *RHS, const DataLayout &DL,
1612 const Instruction *I,
1613 bool AllowNonDeterministic) {
1614 if (Instruction::isBinaryOp(Opcode)) {
1615 // Flush denormal inputs if needed.
1616 Constant *Op0 =
1617 FlushFPConstant(LHS, I->getFunction(), /* IsOutput */ false);
1618 if (!Op0)
1619 return nullptr;
1620 Constant *Op1 =
1621 FlushFPConstant(RHS, I->getFunction(), /* IsOutput */ false);
1622 if (!Op1)
1623 return nullptr;
1624
1625 // If nsz or an algebraic FMF flag is set, the result of the FP operation
1626 // may change due to future optimization. Don't constant fold them if
1627 // non-deterministic results are not allowed.
1628 if (!AllowNonDeterministic)
1630 if (FP->hasNoSignedZeros() || FP->hasAllowReassoc() ||
1631 FP->hasAllowContract() || FP->hasAllowReciprocal())
1632 return nullptr;
1633
1634 // Calculate constant result.
1635 Constant *C = ConstantFoldBinaryOpOperands(Opcode, Op0, Op1, DL);
1636 if (!C)
1637 return nullptr;
1638
1639 // Flush denormal output if needed.
1640 C = FlushFPConstant(C, I->getFunction(), /* IsOutput */ true);
1641 if (!C)
1642 return nullptr;
1643
1644 // The precise NaN value is non-deterministic.
1645 if (!AllowNonDeterministic && C->isNaN())
1646 return nullptr;
1647
1648 return C;
1649 }
1650 // If instruction lacks a parent/function and the denormal mode cannot be
1651 // determined, use the default (IEEE).
1652 return ConstantFoldBinaryOpOperands(Opcode, LHS, RHS, DL);
1653}
1654
1656 Type *DestTy, const DataLayout &DL) {
1657 assert(Instruction::isCast(Opcode));
1658
1659 if (auto *CE = dyn_cast<ConstantExpr>(C))
1660 if (CE->isCast())
1661 if (unsigned NewOp = CastInst::isEliminableCastPair(
1662 Instruction::CastOps(CE->getOpcode()),
1663 Instruction::CastOps(Opcode), CE->getOperand(0)->getType(),
1664 C->getType(), DestTy, &DL))
1665 return ConstantFoldCastOperand(NewOp, CE->getOperand(0), DestTy, DL);
1666
1667 switch (Opcode) {
1668 default:
1669 llvm_unreachable("Missing case");
1670 case Instruction::PtrToAddr:
1671 case Instruction::PtrToInt:
1672 if (auto *CE = dyn_cast<ConstantExpr>(C)) {
1673 Constant *FoldedValue = nullptr;
1674 // If the input is an inttoptr, eliminate the pair. This requires knowing
1675 // the width of a pointer, so it can't be done in ConstantExpr::getCast.
1676 if (CE->getOpcode() == Instruction::IntToPtr) {
1677 // zext/trunc the inttoptr to pointer size. ptrtoaddr is truncated to
1678 // the address size below.
1679 FoldedValue = ConstantFoldIntegerCast(CE->getOperand(0),
1680 DL.getIntPtrType(CE->getType()),
1681 /*IsSigned=*/false, DL);
1682 } else if (auto *GEP = dyn_cast<GEPOperator>(CE)) {
1683 // If we have GEP, we can perform the following folds:
1684 // (ptrtoint/ptrtoaddr (gep null, x)) -> x
1685 // (ptrtoint/ptrtoaddr (gep (gep null, x), y) -> x + y, etc.
1686 unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType());
1687 APInt BaseOffset(BitWidth, 0);
1688 auto *Base = cast<Constant>(GEP->stripAndAccumulateConstantOffsets(
1689 DL, BaseOffset, /*AllowNonInbounds=*/true));
1690 if (Base->isNullValue()) {
1691 FoldedValue = ConstantInt::get(CE->getContext(), BaseOffset);
1692 } else {
1693 // ptrtoint/ptrtoaddr (gep i8, Ptr, (sub 0, V))
1694 // -> sub (ptrtoint/ptrtoaddr Ptr), V
1695 if (GEP->getNumIndices() == 1 &&
1696 GEP->getSourceElementType()->isIntegerTy(8)) {
1697 auto *Ptr = cast<Constant>(GEP->getPointerOperand());
1698 auto *Sub = dyn_cast<ConstantExpr>(GEP->getOperand(1));
1699 Type *IntIdxTy = DL.getIndexType(Ptr->getType());
1700 if (Sub && Sub->getType() == IntIdxTy &&
1701 Sub->getOpcode() == Instruction::Sub &&
1702 Sub->getOperand(0)->isNullValue())
1703 FoldedValue = ConstantExpr::getSub(
1704 ConstantExpr::getCast(Opcode, Ptr, IntIdxTy),
1705 Sub->getOperand(1));
1706 }
1707 }
1708 }
1709 if (FoldedValue) {
1710 // Do a zext or trunc to get to the ptrtoint/ptrtoaddr dest size.
1711 return ConstantFoldIntegerCast(FoldedValue, DestTy, /*IsSigned=*/false,
1712 DL);
1713 }
1714 }
1715 break;
1716 case Instruction::IntToPtr:
1717 case Instruction::Trunc:
1718 case Instruction::ZExt:
1719 case Instruction::SExt:
1720 case Instruction::FPTrunc:
1721 case Instruction::FPExt:
1722 case Instruction::UIToFP:
1723 case Instruction::SIToFP:
1724 case Instruction::FPToUI:
1725 case Instruction::FPToSI:
1726 case Instruction::AddrSpaceCast:
1727 break;
1728 case Instruction::BitCast:
1729 return FoldBitCast(C, DestTy, DL);
1730 }
1731
1733 return ConstantExpr::getCast(Opcode, C, DestTy);
1734 return ConstantFoldCastInstruction(Opcode, C, DestTy);
1735}
1736
1738 bool IsSigned, const DataLayout &DL) {
1739 Type *SrcTy = C->getType();
1740 if (SrcTy == DestTy)
1741 return C;
1742 if (SrcTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits())
1743 return ConstantFoldCastOperand(Instruction::Trunc, C, DestTy, DL);
1744 if (IsSigned)
1745 return ConstantFoldCastOperand(Instruction::SExt, C, DestTy, DL);
1746 return ConstantFoldCastOperand(Instruction::ZExt, C, DestTy, DL);
1747}
1748
1751 const DataLayout &DL) {
1752 // bitinsert C, C, undef -> poison
1754 return PoisonValue::get(Base->getType());
1755
1756 // bitinsert poison, poison, C -> poison
1758 return Base;
1759
1760 if (auto *VTy = dyn_cast<VectorType>(Base->getType())) {
1761 // Fast path for splatted constants.
1762 Constant *BaseSplat = Base->getSplatValue();
1763 Constant *ValSplat = Val->getSplatValue();
1764 Constant *OffsetSplat = Offset->getSplatValue();
1765 if (BaseSplat && ValSplat && OffsetSplat) {
1766 Constant *Res =
1767 ConstantFoldBitInsertOperands(BaseSplat, ValSplat, OffsetSplat, DL);
1768 if (!Res)
1769 return nullptr;
1770 return ConstantVector::getSplat(VTy->getElementCount(), Res);
1771 }
1772
1773 auto *FVTy = dyn_cast<FixedVectorType>(VTy);
1774 if (!FVTy)
1775 return nullptr;
1776
1777 // Fold each element and create a vector constant from those constants.
1779 for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
1780 Constant *BaseElt = Base->getAggregateElement(I);
1781 Constant *ValElt = Val->getAggregateElement(I);
1782 Constant *OffsetElt = Offset->getAggregateElement(I);
1783 if (!BaseElt || !ValElt || !OffsetElt)
1784 return nullptr;
1785 Constant *Res =
1786 ConstantFoldBitInsertOperands(BaseElt, ValElt, OffsetElt, DL);
1787 if (!Res)
1788 return nullptr;
1789 Result.push_back(Res);
1790 }
1791
1792 return ConstantVector::get(Result);
1793 }
1794
1795 auto *COffset = dyn_cast<ConstantInt>(Offset);
1796 if (!COffset)
1797 return nullptr;
1798
1799 // bitinsert C, C, out_of_range -> poison
1800 unsigned BaseBits = DL.getTypeSizeInBits(Base->getType());
1801 unsigned Bits = DL.getTypeSizeInBits(Val->getType());
1802 uint64_t Off = COffset->getZExtValue();
1803 if (Off + Bits > BaseBits)
1804 return PoisonValue::get(Base->getType());
1805
1806 // Overwriting every bit of the base is a bitcast of the value.
1807 if (Bits == BaseBits)
1808 return ConstantFoldCastOperand(Instruction::BitCast, Val, Base->getType(),
1809 DL);
1810
1811 // bitinsert undef, undef, C -> undef
1812 // Any poison bits are refined to undef.
1814 return UndefValue::get(Base->getType());
1815
1816 // A byte constant can't mix poison or undef bits with other bits.
1817 auto *CB = dyn_cast<ConstantByte>(Base);
1818 // The value bits are unknown.
1819 if (!CB || isa<ConstantExpr>(Val) || Val->getType()->isPointerTy())
1820 return nullptr;
1821
1823 ConstantFoldCastOperand(Instruction::BitCast, Val,
1824 IntegerType::get(Base->getContext(), Bits), DL));
1825 if (!CI)
1826 return nullptr;
1827
1828 APInt Res = CB->getValue();
1829 Res.insertBits(CI->getValue(), Off);
1830 return ConstantByte::get(Base->getType(), Res);
1831}
1832
1835 const DataLayout &DL) {
1836 // bitextract poison, C -> poison
1837 // bitextract C, undef -> poison
1839 return PoisonValue::get(Ty);
1840
1841 if (auto *VTy = dyn_cast<VectorType>(Ty)) {
1842 Type *EltTy = VTy->getElementType();
1843 // Fast path for splatted constants.
1844 Constant *SrcSplat = Src->getSplatValue();
1845 Constant *OffsetSplat = Offset->getSplatValue();
1846 if (SrcSplat && OffsetSplat) {
1847 Constant *Res =
1848 ConstantFoldBitExtractOperands(EltTy, SrcSplat, OffsetSplat, DL);
1849 if (!Res)
1850 return nullptr;
1851 return ConstantVector::getSplat(VTy->getElementCount(), Res);
1852 }
1853
1854 auto *FVTy = dyn_cast<FixedVectorType>(VTy);
1855 if (!FVTy)
1856 return nullptr;
1857
1858 // Fold each element and create a vector constant from those constants.
1860 for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
1861 Constant *SrcElt = Src->getAggregateElement(I);
1862 Constant *OffsetElt = Offset->getAggregateElement(I);
1863 if (!SrcElt || !OffsetElt)
1864 return nullptr;
1865 Constant *Res =
1866 ConstantFoldBitExtractOperands(EltTy, SrcElt, OffsetElt, DL);
1867 if (!Res)
1868 return nullptr;
1869 Result.push_back(Res);
1870 }
1871
1872 return ConstantVector::get(Result);
1873 }
1874
1875 auto *COffset = dyn_cast<ConstantInt>(Offset);
1876 if (!COffset)
1877 return nullptr;
1878
1879 // bitextract C, out_of_range -> poison
1880 unsigned Bits = DL.getTypeSizeInBits(Ty);
1881 uint64_t Off = COffset->getZExtValue();
1882 if (Off + Bits > DL.getTypeSizeInBits(Src->getType()))
1883 return PoisonValue::get(Ty);
1884
1885 // bitextract undef, C -> undef
1886 if (isa<UndefValue>(Src))
1887 return UndefValue::get(Ty);
1888
1889 auto *CB = dyn_cast<ConstantByte>(Src);
1890 // An integer can't be bitcast to a pointer.
1891 if (!CB || Ty->isPointerTy())
1892 return nullptr;
1893
1894 APInt Res = CB->getValue().extractBits(Bits, Off);
1896 Instruction::BitCast, ConstantInt::get(Src->getContext(), Res), Ty, DL);
1897}
1898
1899//===----------------------------------------------------------------------===//
1900// Constant Folding for Calls
1901//
1902
1903/// Returns true if the intrinsic can be constant folded, given \p IsStrictFP.
1904static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
1905 switch (ID) {
1906 // Operations that do not operate floating-point numbers and do not depend on
1907 // FP environment can be folded even in strictfp functions.
1908 case Intrinsic::bswap:
1909 case Intrinsic::ctpop:
1910 case Intrinsic::ctlz:
1911 case Intrinsic::cttz:
1912 case Intrinsic::fshl:
1913 case Intrinsic::fshr:
1914 case Intrinsic::clmul:
1915 case Intrinsic::pdep:
1916 case Intrinsic::pext:
1917 case Intrinsic::launder_invariant_group:
1918 case Intrinsic::masked_load:
1919 case Intrinsic::get_active_lane_mask:
1920 case Intrinsic::mask_beforefirst:
1921 case Intrinsic::abs:
1922 case Intrinsic::smax:
1923 case Intrinsic::smin:
1924 case Intrinsic::umax:
1925 case Intrinsic::umin:
1926 case Intrinsic::scmp:
1927 case Intrinsic::ucmp:
1928 case Intrinsic::sadd_with_overflow:
1929 case Intrinsic::uadd_with_overflow:
1930 case Intrinsic::ssub_with_overflow:
1931 case Intrinsic::usub_with_overflow:
1932 case Intrinsic::smul_with_overflow:
1933 case Intrinsic::umul_with_overflow:
1934 case Intrinsic::smulh:
1935 case Intrinsic::umulh:
1936 case Intrinsic::sadd_sat:
1937 case Intrinsic::uadd_sat:
1938 case Intrinsic::ssub_sat:
1939 case Intrinsic::usub_sat:
1940 case Intrinsic::sshl_sat:
1941 case Intrinsic::ushl_sat:
1942 case Intrinsic::smul_fix:
1943 case Intrinsic::smul_fix_sat:
1944 case Intrinsic::bitreverse:
1945 case Intrinsic::is_constant:
1946 case Intrinsic::vector_reduce_add:
1947 case Intrinsic::vector_reduce_mul:
1948 case Intrinsic::vector_reduce_and:
1949 case Intrinsic::vector_reduce_or:
1950 case Intrinsic::vector_reduce_xor:
1951 case Intrinsic::vector_reduce_smin:
1952 case Intrinsic::vector_reduce_smax:
1953 case Intrinsic::vector_reduce_umin:
1954 case Intrinsic::vector_reduce_umax:
1955 case Intrinsic::vector_partial_reduce_add:
1956 case Intrinsic::vector_extract:
1957 case Intrinsic::vector_insert:
1958 case Intrinsic::vector_interleave2:
1959 case Intrinsic::vector_interleave3:
1960 case Intrinsic::vector_interleave4:
1961 case Intrinsic::vector_interleave5:
1962 case Intrinsic::vector_interleave6:
1963 case Intrinsic::vector_interleave7:
1964 case Intrinsic::vector_interleave8:
1965 case Intrinsic::vector_deinterleave2:
1966 case Intrinsic::vector_deinterleave3:
1967 case Intrinsic::vector_deinterleave4:
1968 case Intrinsic::vector_deinterleave5:
1969 case Intrinsic::vector_deinterleave6:
1970 case Intrinsic::vector_deinterleave7:
1971 case Intrinsic::vector_deinterleave8:
1972 // Target intrinsics
1973 case Intrinsic::amdgcn_perm:
1974 case Intrinsic::amdgcn_wave_reduce_umin:
1975 case Intrinsic::amdgcn_wave_reduce_umax:
1976 case Intrinsic::amdgcn_wave_reduce_max:
1977 case Intrinsic::amdgcn_wave_reduce_min:
1978 case Intrinsic::amdgcn_wave_reduce_and:
1979 case Intrinsic::amdgcn_wave_reduce_or:
1980 case Intrinsic::amdgcn_wave_reduce_xor:
1981 case Intrinsic::amdgcn_wave_reduce_add:
1982 case Intrinsic::amdgcn_wave_reduce_sub:
1983 case Intrinsic::amdgcn_s_wqm:
1984 case Intrinsic::amdgcn_s_quadmask:
1985 case Intrinsic::amdgcn_s_bitreplicate:
1986 case Intrinsic::arm_crc32b:
1987 case Intrinsic::arm_crc32h:
1988 case Intrinsic::arm_crc32w:
1989 case Intrinsic::arm_crc32cb:
1990 case Intrinsic::arm_crc32ch:
1991 case Intrinsic::arm_crc32cw:
1992 case Intrinsic::arm_mve_vctp8:
1993 case Intrinsic::arm_mve_vctp16:
1994 case Intrinsic::arm_mve_vctp32:
1995 case Intrinsic::arm_mve_vctp64:
1996 case Intrinsic::aarch64_crc32b:
1997 case Intrinsic::aarch64_crc32h:
1998 case Intrinsic::aarch64_crc32w:
1999 case Intrinsic::aarch64_crc32x:
2000 case Intrinsic::aarch64_crc32cb:
2001 case Intrinsic::aarch64_crc32ch:
2002 case Intrinsic::aarch64_crc32cw:
2003 case Intrinsic::aarch64_crc32cx:
2004 case Intrinsic::aarch64_sve_convert_from_svbool:
2005 case Intrinsic::wasm_alltrue:
2006 case Intrinsic::wasm_anytrue:
2007 case Intrinsic::wasm_dot:
2008 // WebAssembly float semantics are always known
2009 case Intrinsic::wasm_trunc_signed:
2010 case Intrinsic::wasm_trunc_unsigned:
2011 case Intrinsic::x86_sse42_crc32_32_8:
2012 case Intrinsic::x86_sse42_crc32_32_16:
2013 case Intrinsic::x86_sse42_crc32_32_32:
2014 case Intrinsic::x86_sse42_crc32_64_64:
2015 return true;
2016
2017 // Floating point operations cannot be folded in strictfp functions in
2018 // general case. They can be folded if FP environment is known to compiler.
2019 case Intrinsic::minnum:
2020 case Intrinsic::maxnum:
2021 case Intrinsic::minimum:
2022 case Intrinsic::maximum:
2023 case Intrinsic::minimumnum:
2024 case Intrinsic::maximumnum:
2025 case Intrinsic::log:
2026 case Intrinsic::log2:
2027 case Intrinsic::log10:
2028 case Intrinsic::exp:
2029 case Intrinsic::exp2:
2030 case Intrinsic::exp10:
2031 case Intrinsic::sqrt:
2032 case Intrinsic::sin:
2033 case Intrinsic::cos:
2034 case Intrinsic::sincos:
2035 case Intrinsic::sinh:
2036 case Intrinsic::cosh:
2037 case Intrinsic::atan:
2038 case Intrinsic::pow:
2039 case Intrinsic::powi:
2040 case Intrinsic::ldexp:
2041 case Intrinsic::fma:
2042 case Intrinsic::fmuladd:
2043 case Intrinsic::frexp:
2044 case Intrinsic::fptoui_sat:
2045 case Intrinsic::fptosi_sat:
2046 case Intrinsic::amdgcn_cos:
2047 case Intrinsic::amdgcn_cubeid:
2048 case Intrinsic::amdgcn_cubema:
2049 case Intrinsic::amdgcn_cubesc:
2050 case Intrinsic::amdgcn_cubetc:
2051 case Intrinsic::amdgcn_fmul_legacy:
2052 case Intrinsic::amdgcn_fma_legacy:
2053 case Intrinsic::amdgcn_fract:
2054 case Intrinsic::amdgcn_sin:
2055 // The intrinsics below depend on rounding mode in MXCSR.
2056 case Intrinsic::x86_sse_cvtss2si:
2057 case Intrinsic::x86_sse_cvtss2si64:
2058 case Intrinsic::x86_sse_cvttss2si:
2059 case Intrinsic::x86_sse_cvttss2si64:
2060 case Intrinsic::x86_sse2_cvtsd2si:
2061 case Intrinsic::x86_sse2_cvtsd2si64:
2062 case Intrinsic::x86_sse2_cvttsd2si:
2063 case Intrinsic::x86_sse2_cvttsd2si64:
2064 case Intrinsic::x86_avx512_vcvtss2si32:
2065 case Intrinsic::x86_avx512_vcvtss2si64:
2066 case Intrinsic::x86_avx512_cvttss2si:
2067 case Intrinsic::x86_avx512_cvttss2si64:
2068 case Intrinsic::x86_avx512_vcvtsd2si32:
2069 case Intrinsic::x86_avx512_vcvtsd2si64:
2070 case Intrinsic::x86_avx512_cvttsd2si:
2071 case Intrinsic::x86_avx512_cvttsd2si64:
2072 case Intrinsic::x86_avx512_vcvtss2usi32:
2073 case Intrinsic::x86_avx512_vcvtss2usi64:
2074 case Intrinsic::x86_avx512_cvttss2usi:
2075 case Intrinsic::x86_avx512_cvttss2usi64:
2076 case Intrinsic::x86_avx512_vcvtsd2usi32:
2077 case Intrinsic::x86_avx512_vcvtsd2usi64:
2078 case Intrinsic::x86_avx512_cvttsd2usi:
2079 case Intrinsic::x86_avx512_cvttsd2usi64:
2080
2081 // NVVM FMax intrinsics
2082 case Intrinsic::nvvm_fmax_d:
2083 case Intrinsic::nvvm_fmax_f:
2084 case Intrinsic::nvvm_fmax_ftz_f:
2085 case Intrinsic::nvvm_fmax_ftz_nan_f:
2086 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
2087 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
2088 case Intrinsic::nvvm_fmax_nan_f:
2089 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
2090 case Intrinsic::nvvm_fmax_xorsign_abs_f:
2091
2092 // NVVM FMin intrinsics
2093 case Intrinsic::nvvm_fmin_d:
2094 case Intrinsic::nvvm_fmin_f:
2095 case Intrinsic::nvvm_fmin_ftz_f:
2096 case Intrinsic::nvvm_fmin_ftz_nan_f:
2097 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
2098 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
2099 case Intrinsic::nvvm_fmin_nan_f:
2100 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
2101 case Intrinsic::nvvm_fmin_xorsign_abs_f:
2102
2103 // NVVM float/double to int32/uint32 conversion intrinsics
2104 case Intrinsic::nvvm_f2i_rm:
2105 case Intrinsic::nvvm_f2i_rn:
2106 case Intrinsic::nvvm_f2i_rp:
2107 case Intrinsic::nvvm_f2i_rz:
2108 case Intrinsic::nvvm_f2i_rm_ftz:
2109 case Intrinsic::nvvm_f2i_rn_ftz:
2110 case Intrinsic::nvvm_f2i_rp_ftz:
2111 case Intrinsic::nvvm_f2i_rz_ftz:
2112 case Intrinsic::nvvm_f2ui_rm:
2113 case Intrinsic::nvvm_f2ui_rn:
2114 case Intrinsic::nvvm_f2ui_rp:
2115 case Intrinsic::nvvm_f2ui_rz:
2116 case Intrinsic::nvvm_f2ui_rm_ftz:
2117 case Intrinsic::nvvm_f2ui_rn_ftz:
2118 case Intrinsic::nvvm_f2ui_rp_ftz:
2119 case Intrinsic::nvvm_f2ui_rz_ftz:
2120 case Intrinsic::nvvm_d2i_rm:
2121 case Intrinsic::nvvm_d2i_rn:
2122 case Intrinsic::nvvm_d2i_rp:
2123 case Intrinsic::nvvm_d2i_rz:
2124 case Intrinsic::nvvm_d2ui_rm:
2125 case Intrinsic::nvvm_d2ui_rn:
2126 case Intrinsic::nvvm_d2ui_rp:
2127 case Intrinsic::nvvm_d2ui_rz:
2128
2129 // NVVM float/double to int64/uint64 conversion intrinsics
2130 case Intrinsic::nvvm_f2ll_rm:
2131 case Intrinsic::nvvm_f2ll_rn:
2132 case Intrinsic::nvvm_f2ll_rp:
2133 case Intrinsic::nvvm_f2ll_rz:
2134 case Intrinsic::nvvm_f2ll_rm_ftz:
2135 case Intrinsic::nvvm_f2ll_rn_ftz:
2136 case Intrinsic::nvvm_f2ll_rp_ftz:
2137 case Intrinsic::nvvm_f2ll_rz_ftz:
2138 case Intrinsic::nvvm_f2ull_rm:
2139 case Intrinsic::nvvm_f2ull_rn:
2140 case Intrinsic::nvvm_f2ull_rp:
2141 case Intrinsic::nvvm_f2ull_rz:
2142 case Intrinsic::nvvm_f2ull_rm_ftz:
2143 case Intrinsic::nvvm_f2ull_rn_ftz:
2144 case Intrinsic::nvvm_f2ull_rp_ftz:
2145 case Intrinsic::nvvm_f2ull_rz_ftz:
2146 case Intrinsic::nvvm_d2ll_rm:
2147 case Intrinsic::nvvm_d2ll_rn:
2148 case Intrinsic::nvvm_d2ll_rp:
2149 case Intrinsic::nvvm_d2ll_rz:
2150 case Intrinsic::nvvm_d2ull_rm:
2151 case Intrinsic::nvvm_d2ull_rn:
2152 case Intrinsic::nvvm_d2ull_rp:
2153 case Intrinsic::nvvm_d2ull_rz:
2154
2155 // NVVM math intrinsics:
2156 case Intrinsic::nvvm_ceil_d:
2157 case Intrinsic::nvvm_ceil_f:
2158 case Intrinsic::nvvm_ceil_ftz_f:
2159
2160 case Intrinsic::nvvm_fabs:
2161 case Intrinsic::nvvm_fabs_ftz:
2162
2163 case Intrinsic::nvvm_floor_d:
2164 case Intrinsic::nvvm_floor_f:
2165 case Intrinsic::nvvm_floor_ftz_f:
2166
2167 case Intrinsic::nvvm_rcp_rm_d:
2168 case Intrinsic::nvvm_rcp_rm_f:
2169 case Intrinsic::nvvm_rcp_rm_ftz_f:
2170 case Intrinsic::nvvm_rcp_rn_d:
2171 case Intrinsic::nvvm_rcp_rn_f:
2172 case Intrinsic::nvvm_rcp_rn_ftz_f:
2173 case Intrinsic::nvvm_rcp_rp_d:
2174 case Intrinsic::nvvm_rcp_rp_f:
2175 case Intrinsic::nvvm_rcp_rp_ftz_f:
2176 case Intrinsic::nvvm_rcp_rz_d:
2177 case Intrinsic::nvvm_rcp_rz_f:
2178 case Intrinsic::nvvm_rcp_rz_ftz_f:
2179
2180 case Intrinsic::nvvm_round_d:
2181 case Intrinsic::nvvm_round_f:
2182 case Intrinsic::nvvm_round_ftz_f:
2183
2184 case Intrinsic::nvvm_saturate_d:
2185 case Intrinsic::nvvm_saturate_f:
2186 case Intrinsic::nvvm_saturate_ftz_f:
2187
2188 case Intrinsic::nvvm_sqrt_f:
2189 case Intrinsic::nvvm_sqrt_rn_d:
2190 case Intrinsic::nvvm_sqrt_rn_f:
2191 case Intrinsic::nvvm_sqrt_rn_ftz_f:
2192 return !IsStrictFP;
2193
2194 // NVVM fadd/fmul/fma intrinsics with explicit rounding modes
2195 case Intrinsic::nvvm_fadd:
2196 case Intrinsic::nvvm_fadd_ftz:
2197 case Intrinsic::nvvm_fma:
2198 case Intrinsic::nvvm_fma_ftz:
2199 case Intrinsic::nvvm_fmul:
2200 case Intrinsic::nvvm_fmul_ftz:
2201
2202 // NVVM div intrinsics with explicit rounding modes
2203 case Intrinsic::nvvm_div_rm_d:
2204 case Intrinsic::nvvm_div_rn_d:
2205 case Intrinsic::nvvm_div_rp_d:
2206 case Intrinsic::nvvm_div_rz_d:
2207 case Intrinsic::nvvm_div_rm_f:
2208 case Intrinsic::nvvm_div_rn_f:
2209 case Intrinsic::nvvm_div_rp_f:
2210 case Intrinsic::nvvm_div_rz_f:
2211 case Intrinsic::nvvm_div_rm_ftz_f:
2212 case Intrinsic::nvvm_div_rn_ftz_f:
2213 case Intrinsic::nvvm_div_rp_ftz_f:
2214 case Intrinsic::nvvm_div_rz_ftz_f:
2215
2216 // Sign operations are actually bitwise operations, they do not raise
2217 // exceptions even for SNANs.
2218 case Intrinsic::fabs:
2219 case Intrinsic::copysign:
2220 case Intrinsic::is_fpclass:
2221 // Non-constrained variants of rounding operations means default FP
2222 // environment, they can be folded in any case.
2223 case Intrinsic::ceil:
2224 case Intrinsic::floor:
2225 case Intrinsic::round:
2226 case Intrinsic::roundeven:
2227 case Intrinsic::trunc:
2228 case Intrinsic::nearbyint:
2229 case Intrinsic::rint:
2230 case Intrinsic::canonicalize:
2231
2232 // Constrained intrinsics can be folded if FP environment is known
2233 // to compiler.
2234 case Intrinsic::experimental_constrained_fma:
2235 case Intrinsic::experimental_constrained_fmuladd:
2236 case Intrinsic::experimental_constrained_fadd:
2237 case Intrinsic::experimental_constrained_fsub:
2238 case Intrinsic::experimental_constrained_fmul:
2239 case Intrinsic::experimental_constrained_fdiv:
2240 case Intrinsic::experimental_constrained_frem:
2241 case Intrinsic::experimental_constrained_ceil:
2242 case Intrinsic::experimental_constrained_floor:
2243 case Intrinsic::experimental_constrained_round:
2244 case Intrinsic::experimental_constrained_roundeven:
2245 case Intrinsic::experimental_constrained_trunc:
2246 case Intrinsic::experimental_constrained_nearbyint:
2247 case Intrinsic::experimental_constrained_rint:
2248 case Intrinsic::experimental_constrained_fcmp:
2249 case Intrinsic::experimental_constrained_fcmps:
2250
2251 case Intrinsic::experimental_cttz_elts:
2252 return true;
2253 default:
2254 return false;
2255 }
2256}
2257
2258/// Given a function's return type and its operands, determine if any of them of
2259/// of floating-point type.
2261 return RetTy->isFloatingPointTy() || any_of(Ops, [](Value *V) {
2262 return V->getType()->isFloatingPointTy();
2263 });
2264}
2265
2267 const TargetLibraryInfo *TLI) {
2268 if (Call->isNoBuiltin())
2269 return false;
2270 if (Call->getFunctionType() != F->getFunctionType())
2271 return false;
2272
2273 // Allow FP calls (both libcalls and intrinsics) to avoid being folded.
2274 // This can be useful for GPU targets or in cross-compilation scenarios
2275 // when the exact target FP behaviour is required, and the host compiler's
2276 // behaviour may be slightly different from the device's run-time behaviour.
2279 F->getReturnType(),
2280 ArrayRef<Value *>((Value *const *)(F->arg_begin()), F->arg_size())))
2281 return false;
2282
2283 if (F->getIntrinsicID() != Intrinsic::not_intrinsic)
2284 return canConstantFoldIntrinsic(F->getIntrinsicID(), Call->isStrictFP());
2285
2286 if (!TLI || Call->isStrictFP())
2287 return false;
2288
2289 LibFunc Func = TLI->getLibFunc(*F);
2290 if (Func == NotLibFunc)
2291 return false;
2292
2293 switch (Func) {
2294 case LibFunc_acos:
2295 case LibFunc_acosf:
2296 case LibFunc_acos_finite:
2297 case LibFunc_acosf_finite:
2298 case LibFunc_asin:
2299 case LibFunc_asinf:
2300 case LibFunc_asin_finite:
2301 case LibFunc_asinf_finite:
2302 case LibFunc_atan:
2303 case LibFunc_atanf:
2304 case LibFunc_atan2:
2305 case LibFunc_atan2f:
2306 case LibFunc_atan2_finite:
2307 case LibFunc_atan2f_finite:
2308 case LibFunc_ceil:
2309 case LibFunc_ceilf:
2310 case LibFunc_cosh:
2311 case LibFunc_coshf:
2312 case LibFunc_cosh_finite:
2313 case LibFunc_coshf_finite:
2314 case LibFunc_cos:
2315 case LibFunc_cosf:
2316 case LibFunc_erf:
2317 case LibFunc_erff:
2318 case LibFunc_exp:
2319 case LibFunc_expf:
2320 case LibFunc_exp_finite:
2321 case LibFunc_expf_finite:
2322 case LibFunc_exp2:
2323 case LibFunc_exp2f:
2324 case LibFunc_exp2_finite:
2325 case LibFunc_exp2f_finite:
2326 case LibFunc_fabs:
2327 case LibFunc_fabsf:
2328 case LibFunc_floor:
2329 case LibFunc_floorf:
2330 case LibFunc_fmod:
2331 case LibFunc_fmodf:
2332 case LibFunc_ilogb:
2333 case LibFunc_ilogbf:
2334 case LibFunc_log:
2335 case LibFunc_logf:
2336 case LibFunc_log_finite:
2337 case LibFunc_logf_finite:
2338 case LibFunc_logb:
2339 case LibFunc_logbf:
2340 case LibFunc_logl:
2341 case LibFunc_log2:
2342 case LibFunc_log2f:
2343 case LibFunc_log2_finite:
2344 case LibFunc_log2f_finite:
2345 case LibFunc_log10:
2346 case LibFunc_log10f:
2347 case LibFunc_log10_finite:
2348 case LibFunc_log10f_finite:
2349 case LibFunc_log1p:
2350 case LibFunc_log1pf:
2351 case LibFunc_nearbyint:
2352 case LibFunc_nearbyintf:
2353 case LibFunc_nextafter:
2354 case LibFunc_nextafterf:
2355 case LibFunc_nexttoward:
2356 case LibFunc_nexttowardf:
2357 case LibFunc_pow:
2358 case LibFunc_powf:
2359 case LibFunc_pow_finite:
2360 case LibFunc_powf_finite:
2361 case LibFunc_remainder:
2362 case LibFunc_remainderf:
2363 case LibFunc_rint:
2364 case LibFunc_rintf:
2365 case LibFunc_round:
2366 case LibFunc_roundf:
2367 case LibFunc_roundeven:
2368 case LibFunc_roundevenf:
2369 case LibFunc_sin:
2370 case LibFunc_sinf:
2371 case LibFunc_sinh:
2372 case LibFunc_sinhf:
2373 case LibFunc_sinh_finite:
2374 case LibFunc_sinhf_finite:
2375 case LibFunc_sqrt:
2376 case LibFunc_sqrtf:
2377 case LibFunc_tan:
2378 case LibFunc_tanf:
2379 case LibFunc_tanh:
2380 case LibFunc_tanhf:
2381 case LibFunc_trunc:
2382 case LibFunc_truncf:
2383 return true;
2384 default:
2385 return false;
2386 }
2387}
2388
2389namespace {
2390
2391Constant *GetConstantFoldFPValue(double V, Type *Ty) {
2392 if (Ty->isHalfTy() || Ty->isFloatTy() || Ty->isBFloatTy()) {
2393 APFloat APF(V);
2394 bool unused;
2395 APF.convert(Ty->getFltSemantics(), APFloat::rmNearestTiesToEven, &unused);
2396 return ConstantFP::get(Ty->getContext(), APF);
2397 }
2398 if (Ty->isDoubleTy())
2399 return ConstantFP::get(Ty->getContext(), APFloat(V));
2400 llvm_unreachable("Can only constant fold half/float/double/bfloat");
2401}
2402
2403#if defined(HAS_IEE754_FLOAT128) && defined(HAS_LOGF128)
2404Constant *GetConstantFoldFPValue128(float128 V, Type *Ty) {
2405 if (Ty->isFP128Ty())
2406 return ConstantFP::get(Ty, V);
2407 llvm_unreachable("Can only constant fold fp128");
2408}
2409#endif
2410
2411/// Clear the floating-point exception state.
2412inline void llvm_fenv_clearexcept() {
2413#if defined(FE_ALL_EXCEPT)
2414 feclearexcept(FE_ALL_EXCEPT);
2415#endif
2416 errno = 0;
2417}
2418
2419/// Test if a floating-point exception was raised.
2420inline bool llvm_fenv_testexcept() {
2421 int errno_val = errno;
2422 if (errno_val == ERANGE || errno_val == EDOM)
2423 return true;
2424#if defined(FE_ALL_EXCEPT) && defined(FE_INEXACT)
2425 if (fetestexcept(FE_ALL_EXCEPT & ~FE_INEXACT))
2426 return true;
2427#endif
2428 return false;
2429}
2430
2431static APFloat FTZPreserveSign(const APFloat &V) {
2432 if (V.isDenormal())
2433 return APFloat::getZero(V.getSemantics(), V.isNegative());
2434 return V;
2435}
2436
2437static APFloat FlushToPositiveZero(const APFloat &V) {
2438 if (V.isDenormal())
2439 return APFloat::getZero(V.getSemantics(), false);
2440 return V;
2441}
2442
2443static APFloat FlushWithDenormKind(const APFloat &V,
2444 DenormalMode::DenormalModeKind DenormKind) {
2447 switch (DenormKind) {
2449 return V;
2451 return FTZPreserveSign(V);
2453 return FlushToPositiveZero(V);
2454 default:
2455 llvm_unreachable("Invalid denormal mode!");
2456 }
2457}
2458
2459#if defined(HAS_IEE754_FLOAT128) && defined(HAS_LOGF128)
2460Constant *ConstantFoldFP128(float128 (*NativeFP)(float128), const APFloat &V,
2461 Type *Ty) {
2462 llvm_fenv_clearexcept();
2463 float128 Result = NativeFP(V.convertToQuad());
2464 if (llvm_fenv_testexcept()) {
2465 llvm_fenv_clearexcept();
2466 return nullptr;
2467 }
2468
2469 return GetConstantFoldFPValue128(Result, Ty);
2470}
2471#endif
2472
2473Constant *ConstantFoldBinaryFP(double (*NativeFP)(double, double),
2474 const APFloat &V, const APFloat &W, Type *Ty) {
2475 llvm_fenv_clearexcept();
2476 double Result = NativeFP(V.convertToDouble(), W.convertToDouble());
2477 if (llvm_fenv_testexcept()) {
2478 llvm_fenv_clearexcept();
2479 return nullptr;
2480 }
2481
2482 return GetConstantFoldFPValue(Result, Ty);
2483}
2484
2485Constant *constantFoldVectorReduce(Intrinsic::ID IID, Constant *Op) {
2486 auto *OpVT = cast<VectorType>(Op->getType());
2487
2488 // This is the same as the underlying binops - poison propagates.
2489 if (Op->containsPoisonElement())
2490 return PoisonValue::get(OpVT->getElementType());
2491
2492 // Shortcut non-accumulating reductions.
2493 if (Constant *SplatVal = Op->getSplatValue()) {
2494 switch (IID) {
2495 case Intrinsic::vector_reduce_and:
2496 case Intrinsic::vector_reduce_or:
2497 case Intrinsic::vector_reduce_smin:
2498 case Intrinsic::vector_reduce_smax:
2499 case Intrinsic::vector_reduce_umin:
2500 case Intrinsic::vector_reduce_umax:
2501 return SplatVal;
2502 case Intrinsic::vector_reduce_add:
2503 if (SplatVal->isNullValue())
2504 return SplatVal;
2505 break;
2506 case Intrinsic::vector_reduce_mul:
2507 if (SplatVal->isNullValue() || SplatVal->isOneValue())
2508 return SplatVal;
2509 break;
2510 case Intrinsic::vector_reduce_xor:
2511 if (SplatVal->isNullValue())
2512 return SplatVal;
2513 if (OpVT->getElementCount().isKnownMultipleOf(2))
2514 return Constant::getNullValue(OpVT->getElementType());
2515 break;
2516 }
2517 }
2518
2520 if (!VT)
2521 return nullptr;
2522
2523 auto *EltC = dyn_cast_or_null<ConstantInt>(Op->getAggregateElement(0U));
2524 if (!EltC)
2525 return nullptr;
2526
2527 APInt Acc = EltC->getValue();
2528 for (unsigned I = 1, E = VT->getNumElements(); I != E; I++) {
2529 if (!(EltC = dyn_cast_or_null<ConstantInt>(Op->getAggregateElement(I))))
2530 return nullptr;
2531 const APInt &X = EltC->getValue();
2532 switch (IID) {
2533 case Intrinsic::vector_reduce_add:
2534 Acc = Acc + X;
2535 break;
2536 case Intrinsic::vector_reduce_mul:
2537 Acc = Acc * X;
2538 break;
2539 case Intrinsic::vector_reduce_and:
2540 Acc = Acc & X;
2541 break;
2542 case Intrinsic::vector_reduce_or:
2543 Acc = Acc | X;
2544 break;
2545 case Intrinsic::vector_reduce_xor:
2546 Acc = Acc ^ X;
2547 break;
2548 case Intrinsic::vector_reduce_smin:
2549 Acc = APIntOps::smin(Acc, X);
2550 break;
2551 case Intrinsic::vector_reduce_smax:
2552 Acc = APIntOps::smax(Acc, X);
2553 break;
2554 case Intrinsic::vector_reduce_umin:
2555 Acc = APIntOps::umin(Acc, X);
2556 break;
2557 case Intrinsic::vector_reduce_umax:
2558 Acc = APIntOps::umax(Acc, X);
2559 break;
2560 }
2561 }
2562
2563 return ConstantInt::get(Op->getContext(), Acc);
2564}
2565
2566/// Fold a vector partial reduction add using the deterministic grouping
2567/// chosen by TargetLowering::expandPartialReduceMLA. Although the
2568/// LangRef leaves the grouping unspecified, input element I is accumulated
2569/// into result lane I % NumAccElts, with each accumulator element seeding
2570/// its corresponding result lane. Returns nullptr if any element cannot be
2571/// folded.
2572static Constant *constantFoldVectorPartialReduceAdd(Constant *Acc,
2573 Constant *Input,
2574 const DataLayout &DL) {
2575 auto *AccTy = cast<FixedVectorType>(Acc->getType());
2576 // A fixed result type does not guarantee a fixed input type.
2577 auto *InputTy = dyn_cast<FixedVectorType>(Input->getType());
2578 if (!InputTy)
2579 return nullptr;
2580
2581 unsigned NumAccElts = AccTy->getNumElements();
2582 unsigned NumInputElts = InputTy->getNumElements();
2583
2584 SmallVector<Constant *> ResultElts(NumAccElts);
2585 for (unsigned I = 0; I < NumAccElts; ++I) {
2586 ResultElts[I] = Acc->getAggregateElement(I);
2587 if (!ResultElts[I])
2588 return nullptr;
2589 }
2590
2591 for (unsigned I = 0; I < NumInputElts; ++I) {
2592 Constant *InputElt = Input->getAggregateElement(I);
2593 if (!InputElt)
2594 return nullptr;
2595
2596 unsigned ResultIdx = I % NumAccElts;
2598 Instruction::Add, ResultElts[ResultIdx], InputElt, DL);
2599 if (!Folded)
2600 return nullptr;
2601
2602 ResultElts[ResultIdx] = Folded;
2603 }
2604
2605 return ConstantVector::get(ResultElts);
2606}
2607
2608/// Attempt to fold an SSE floating point to integer conversion of a constant
2609/// floating point. If roundTowardZero is false, the default IEEE rounding is
2610/// used (toward nearest, ties to even). This matches the behavior of the
2611/// non-truncating SSE instructions in the default rounding mode. The desired
2612/// integer type Ty is used to select how many bits are available for the
2613/// result. Returns null if the conversion cannot be performed, otherwise
2614/// returns the Constant value resulting from the conversion.
2615Constant *ConstantFoldSSEConvertToInt(const APFloat &Val, bool roundTowardZero,
2616 Type *Ty, bool IsSigned) {
2617 // All of these conversion intrinsics form an integer of at most 64bits.
2618 unsigned ResultWidth = Ty->getIntegerBitWidth();
2619 assert(ResultWidth <= 64 &&
2620 "Can only constant fold conversions to 64 and 32 bit ints");
2621
2622 uint64_t UIntVal;
2623 bool isExact = false;
2627 Val.convertToInteger(MutableArrayRef(UIntVal), ResultWidth,
2628 IsSigned, mode, &isExact);
2629 if (status != APFloat::opOK &&
2630 (!roundTowardZero || status != APFloat::opInexact))
2631 return nullptr;
2632 return ConstantInt::get(Ty, UIntVal, IsSigned);
2633}
2634
2635double getValueAsDouble(ConstantFP *Op) {
2636 Type *Ty = Op->getType();
2637
2638 if (Ty->isBFloatTy() || Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy())
2639 return Op->getValueAPF().convertToDouble();
2640
2641 bool unused;
2642 APFloat APF = Op->getValueAPF();
2644 return APF.convertToDouble();
2645}
2646
2647static bool getConstIntOrUndef(Value *Op, const APInt *&C) {
2648 if (auto *CI = dyn_cast<ConstantInt>(Op)) {
2649 C = &CI->getValue();
2650 return true;
2651 }
2652 if (isa<UndefValue>(Op)) {
2653 C = nullptr;
2654 return true;
2655 }
2656 return false;
2657}
2658
2659/// Checks if the given intrinsic call, which evaluates to constant, is allowed
2660/// to be folded.
2661///
2662/// \param CI Constrained intrinsic call.
2663/// \param St Exception flags raised during constant evaluation.
2664static bool mayFoldConstrained(ConstrainedFPIntrinsic *CI,
2665 APFloat::opStatus St) {
2666 std::optional<RoundingMode> ORM = CI->getRoundingMode();
2667 std::optional<fp::ExceptionBehavior> EB = CI->getExceptionBehavior();
2668
2669 // If the operation does not change exception status flags, it is safe
2670 // to fold.
2671 if (St == APFloat::opStatus::opOK)
2672 return true;
2673
2674 // If evaluation raised FP exception, the result can depend on rounding
2675 // mode. If the latter is unknown, folding is not possible.
2676 if (ORM == RoundingMode::Dynamic)
2677 return false;
2678
2679 // If FP exceptions are ignored, fold the call, even if such exception is
2680 // raised.
2681 if (EB && *EB != fp::ExceptionBehavior::ebStrict)
2682 return true;
2683
2684 // Leave the calculation for runtime so that exception flags be correctly set
2685 // in hardware.
2686 return false;
2687}
2688
2689/// Returns the rounding mode that should be used for constant evaluation.
2690static RoundingMode
2691getEvaluationRoundingMode(const ConstrainedFPIntrinsic *CI) {
2692 std::optional<RoundingMode> ORM = CI->getRoundingMode();
2693 if (!ORM || *ORM == RoundingMode::Dynamic)
2694 // Even if the rounding mode is unknown, try evaluating the operation.
2695 // If it does not raise inexact exception, rounding was not applied,
2696 // so the result is exact and does not depend on rounding mode. Whether
2697 // other FP exceptions are raised, it does not depend on rounding mode.
2699 return *ORM;
2700}
2701
2702/// Try to constant fold llvm.canonicalize for the given caller and value.
2703static Constant *constantFoldCanonicalize(const Type *Ty, const APFloat &Src,
2704 const Function *CtxF = nullptr) {
2705 // Zero, positive and negative, is always OK to fold.
2706 if (Src.isZero()) {
2707 // Get a fresh 0, since ppc_fp128 does have non-canonical zeros.
2708 return ConstantFP::get(
2709 Ty->getContext(),
2710 APFloat::getZero(Src.getSemantics(), Src.isNegative()));
2711 }
2712
2713 if (!Ty->isIEEELikeFPTy())
2714 return nullptr;
2715
2716 // Zero is always canonical and the sign must be preserved.
2717 //
2718 // Denorms and nans may have special encodings, but it should be OK to fold a
2719 // totally average number.
2720 if (Src.isNormal() || Src.isInfinity())
2721 return ConstantFP::get(Ty->getContext(), Src);
2722
2723 if (Src.isDenormal() && CtxF) {
2724 DenormalMode DenormMode = CtxF->getDenormalMode(Src.getSemantics());
2725
2726 if (DenormMode == DenormalMode::getIEEE())
2727 return ConstantFP::get(Ty->getContext(), Src);
2728
2729 if (DenormMode.Input == DenormalMode::Dynamic)
2730 return nullptr;
2731
2732 // If we know if either input or output is flushed, we can fold.
2733 if ((DenormMode.Input == DenormalMode::Dynamic &&
2734 DenormMode.Output == DenormalMode::IEEE) ||
2735 (DenormMode.Input == DenormalMode::IEEE &&
2736 DenormMode.Output == DenormalMode::Dynamic))
2737 return nullptr;
2738
2739 bool IsPositive =
2740 (!Src.isNegative() || DenormMode.Input == DenormalMode::PositiveZero ||
2741 (DenormMode.Output == DenormalMode::PositiveZero &&
2742 DenormMode.Input == DenormalMode::IEEE));
2743
2744 return ConstantFP::get(Ty->getContext(),
2745 APFloat::getZero(Src.getSemantics(), !IsPositive));
2746 }
2747
2748 return nullptr;
2749}
2750
2751static Constant *ConstantFoldScalarCall1(StringRef Name,
2752 Intrinsic::ID IntrinsicID, Type *Ty,
2754 const TargetLibraryInfo *TLI = nullptr,
2755 const CallBase *Call = nullptr) {
2756 assert(Operands.size() == 1 && "Wrong number of operands.");
2757
2758 if (IntrinsicID == Intrinsic::is_constant) {
2759 // We know we have a "Constant" argument. But we want to only
2760 // return true for manifest constants, not those that depend on
2761 // constants with unknowable values, e.g. GlobalValue or BlockAddress.
2762 if (Operands[0]->isManifestConstant())
2763 return ConstantInt::getTrue(Ty->getContext());
2764 return nullptr;
2765 }
2766
2767 if (isa<UndefValue>(Operands[0])) {
2768 // cosine(arg) is between -1 and 1. cosine(invalid arg) is NaN.
2769 // ctpop() is between 0 and bitwidth, pick 0 for undef.
2770 // fptoui.sat and fptosi.sat can always fold to zero (for a zero input).
2771 if (IntrinsicID == Intrinsic::cos ||
2772 IntrinsicID == Intrinsic::ctpop ||
2773 IntrinsicID == Intrinsic::fptoui_sat ||
2774 IntrinsicID == Intrinsic::fptosi_sat ||
2775 IntrinsicID == Intrinsic::canonicalize)
2776 return Constant::getNullValue(Ty);
2777 if (IntrinsicID == Intrinsic::bswap ||
2778 IntrinsicID == Intrinsic::bitreverse ||
2779 IntrinsicID == Intrinsic::launder_invariant_group)
2780 return Operands[0];
2781 }
2782
2784 // launder(null) == null iff in addrspace 0
2785 if (IntrinsicID == Intrinsic::launder_invariant_group) {
2786 // If instruction is not yet put in a basic block (e.g. when cloning
2787 // a function during inlining), Call's caller may not be available.
2788 // So check Call's BB first before querying Call->getCaller.
2789 const Function *Caller =
2790 Call && Call->getParent() ? Call->getCaller() : nullptr;
2791 if (Caller &&
2793 Caller, Operands[0]->getType()->getPointerAddressSpace())) {
2794 return Operands[0];
2795 }
2796 return nullptr;
2797 }
2798 }
2799
2800 if (auto *Op = dyn_cast<ConstantFP>(Operands[0])) {
2801 APFloat U = Op->getValueAPF();
2802
2803 if (IntrinsicID == Intrinsic::wasm_trunc_signed ||
2804 IntrinsicID == Intrinsic::wasm_trunc_unsigned) {
2805 bool Signed = IntrinsicID == Intrinsic::wasm_trunc_signed;
2806
2807 if (U.isNaN())
2808 return nullptr;
2809
2810 unsigned Width = Ty->getIntegerBitWidth();
2811 APSInt Int(Width, !Signed);
2812 bool IsExact = false;
2814 U.convertToInteger(Int, APFloat::rmTowardZero, &IsExact);
2815
2817 return ConstantInt::get(Ty, Int);
2818
2819 return nullptr;
2820 }
2821
2822 if (IntrinsicID == Intrinsic::fptoui_sat ||
2823 IntrinsicID == Intrinsic::fptosi_sat) {
2824 // convertToInteger() already has the desired saturation semantics.
2825 APSInt Int(Ty->getIntegerBitWidth(),
2826 IntrinsicID == Intrinsic::fptoui_sat);
2827 bool IsExact;
2828 U.convertToInteger(Int, APFloat::rmTowardZero, &IsExact);
2829 return ConstantInt::get(Ty, Int);
2830 }
2831
2832 if (IntrinsicID == Intrinsic::canonicalize) {
2833 const Function *CtxF =
2834 Call && Call->getParent() ? Call->getFunction() : nullptr;
2835 return constantFoldCanonicalize(Ty, U, CtxF);
2836 }
2837
2838#if defined(HAS_IEE754_FLOAT128) && defined(HAS_LOGF128)
2839 if (Ty->isFP128Ty()) {
2840 if (IntrinsicID == Intrinsic::log) {
2841 float128 Result = logf128(Op->getValueAPF().convertToQuad());
2842 return GetConstantFoldFPValue128(Result, Ty);
2843 }
2844
2845 if (TLI && TLI->getLibFunc(Name) == LibFunc_logl &&
2846 TLI->has(LibFunc_logl))
2847 return ConstantFoldFP128(logf128, Op->getValueAPF(), Ty);
2848 }
2849#endif
2850
2851 if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy() &&
2852 !Ty->isIntegerTy() && !Ty->isBFloatTy())
2853 return nullptr;
2854
2855 // Use internal versions of these intrinsics.
2856
2857 if (IntrinsicID == Intrinsic::nearbyint || IntrinsicID == Intrinsic::rint ||
2858 IntrinsicID == Intrinsic::roundeven) {
2859 U.roundToIntegral(APFloat::rmNearestTiesToEven);
2860 return ConstantFP::get(Ty, U);
2861 }
2862
2863 if (IntrinsicID == Intrinsic::round) {
2864 U.roundToIntegral(APFloat::rmNearestTiesToAway);
2865 return ConstantFP::get(Ty, U);
2866 }
2867
2868 if (IntrinsicID == Intrinsic::roundeven) {
2869 U.roundToIntegral(APFloat::rmNearestTiesToEven);
2870 return ConstantFP::get(Ty, U);
2871 }
2872
2873 if (IntrinsicID == Intrinsic::ceil) {
2874 U.roundToIntegral(APFloat::rmTowardPositive);
2875 return ConstantFP::get(Ty, U);
2876 }
2877
2878 if (IntrinsicID == Intrinsic::floor) {
2879 U.roundToIntegral(APFloat::rmTowardNegative);
2880 return ConstantFP::get(Ty, U);
2881 }
2882
2883 if (IntrinsicID == Intrinsic::trunc) {
2884 U.roundToIntegral(APFloat::rmTowardZero);
2885 return ConstantFP::get(Ty, U);
2886 }
2887
2888 if (IntrinsicID == Intrinsic::fabs) {
2889 U.clearSign();
2890 return ConstantFP::get(Ty, U);
2891 }
2892
2893 if (IntrinsicID == Intrinsic::amdgcn_fract) {
2894 // The v_fract instruction behaves like the OpenCL spec, which defines
2895 // fract(x) as fmin(x - floor(x), 0x1.fffffep-1f): "The min() operator is
2896 // there to prevent fract(-small) from returning 1.0. It returns the
2897 // largest positive floating-point number less than 1.0."
2898 APFloat FloorU(U);
2899 FloorU.roundToIntegral(APFloat::rmTowardNegative);
2900 APFloat FractU(U - FloorU);
2901 APFloat AlmostOne(U.getSemantics(), 1);
2902 AlmostOne.next(/*nextDown*/ true);
2903 return ConstantFP::get(Ty, minimum(FractU, AlmostOne));
2904 }
2905
2906 // Rounding operations (floor, trunc, ceil, round and nearbyint) do not
2907 // raise FP exceptions, unless the argument is signaling NaN.
2908
2910 std::optional<APFloat::roundingMode> RM;
2911 switch (IntrinsicID) {
2912 default:
2913 break;
2914 case Intrinsic::experimental_constrained_nearbyint:
2915 case Intrinsic::experimental_constrained_rint: {
2916 RM = CI->getRoundingMode();
2917 if (!RM || *RM == RoundingMode::Dynamic)
2918 return nullptr;
2919 break;
2920 }
2921 case Intrinsic::experimental_constrained_round:
2923 break;
2924 case Intrinsic::experimental_constrained_ceil:
2926 break;
2927 case Intrinsic::experimental_constrained_floor:
2929 break;
2930 case Intrinsic::experimental_constrained_trunc:
2932 break;
2933 }
2934 if (RM) {
2935 if (U.isFinite()) {
2936 APFloat::opStatus St = U.roundToIntegral(*RM);
2937 if (IntrinsicID == Intrinsic::experimental_constrained_rint &&
2938 St == APFloat::opInexact) {
2939 std::optional<fp::ExceptionBehavior> EB =
2941 if (EB == fp::ebStrict)
2942 return nullptr;
2943 }
2944 } else if (U.isSignaling()) {
2945 std::optional<fp::ExceptionBehavior> EB = CI->getExceptionBehavior();
2946 if (EB && *EB != fp::ebIgnore)
2947 return nullptr;
2948 U = APFloat::getQNaN(U.getSemantics());
2949 }
2950 return ConstantFP::get(Ty, U);
2951 }
2952 }
2953
2954 // NVVM float/double to signed/unsigned int32/int64 conversions:
2955 switch (IntrinsicID) {
2956 // f2i
2957 case Intrinsic::nvvm_f2i_rm:
2958 case Intrinsic::nvvm_f2i_rn:
2959 case Intrinsic::nvvm_f2i_rp:
2960 case Intrinsic::nvvm_f2i_rz:
2961 case Intrinsic::nvvm_f2i_rm_ftz:
2962 case Intrinsic::nvvm_f2i_rn_ftz:
2963 case Intrinsic::nvvm_f2i_rp_ftz:
2964 case Intrinsic::nvvm_f2i_rz_ftz:
2965 // f2ui
2966 case Intrinsic::nvvm_f2ui_rm:
2967 case Intrinsic::nvvm_f2ui_rn:
2968 case Intrinsic::nvvm_f2ui_rp:
2969 case Intrinsic::nvvm_f2ui_rz:
2970 case Intrinsic::nvvm_f2ui_rm_ftz:
2971 case Intrinsic::nvvm_f2ui_rn_ftz:
2972 case Intrinsic::nvvm_f2ui_rp_ftz:
2973 case Intrinsic::nvvm_f2ui_rz_ftz:
2974 // d2i
2975 case Intrinsic::nvvm_d2i_rm:
2976 case Intrinsic::nvvm_d2i_rn:
2977 case Intrinsic::nvvm_d2i_rp:
2978 case Intrinsic::nvvm_d2i_rz:
2979 // d2ui
2980 case Intrinsic::nvvm_d2ui_rm:
2981 case Intrinsic::nvvm_d2ui_rn:
2982 case Intrinsic::nvvm_d2ui_rp:
2983 case Intrinsic::nvvm_d2ui_rz:
2984 // f2ll
2985 case Intrinsic::nvvm_f2ll_rm:
2986 case Intrinsic::nvvm_f2ll_rn:
2987 case Intrinsic::nvvm_f2ll_rp:
2988 case Intrinsic::nvvm_f2ll_rz:
2989 case Intrinsic::nvvm_f2ll_rm_ftz:
2990 case Intrinsic::nvvm_f2ll_rn_ftz:
2991 case Intrinsic::nvvm_f2ll_rp_ftz:
2992 case Intrinsic::nvvm_f2ll_rz_ftz:
2993 // f2ull
2994 case Intrinsic::nvvm_f2ull_rm:
2995 case Intrinsic::nvvm_f2ull_rn:
2996 case Intrinsic::nvvm_f2ull_rp:
2997 case Intrinsic::nvvm_f2ull_rz:
2998 case Intrinsic::nvvm_f2ull_rm_ftz:
2999 case Intrinsic::nvvm_f2ull_rn_ftz:
3000 case Intrinsic::nvvm_f2ull_rp_ftz:
3001 case Intrinsic::nvvm_f2ull_rz_ftz:
3002 // d2ll
3003 case Intrinsic::nvvm_d2ll_rm:
3004 case Intrinsic::nvvm_d2ll_rn:
3005 case Intrinsic::nvvm_d2ll_rp:
3006 case Intrinsic::nvvm_d2ll_rz:
3007 // d2ull
3008 case Intrinsic::nvvm_d2ull_rm:
3009 case Intrinsic::nvvm_d2ull_rn:
3010 case Intrinsic::nvvm_d2ull_rp:
3011 case Intrinsic::nvvm_d2ull_rz: {
3012 // In float-to-integer conversion, NaN inputs are converted to 0.
3013 if (U.isNaN()) {
3014 // In float-to-integer conversion, NaN inputs are converted to 0
3015 // when the source and destination bitwidths are both less than 64.
3016 if (nvvm::FPToIntegerIntrinsicNaNZero(IntrinsicID))
3017 return ConstantInt::get(Ty, 0);
3018
3019 // Otherwise, the most significant bit is set.
3020 unsigned BitWidth = Ty->getIntegerBitWidth();
3021 uint64_t Val = 1ULL << (BitWidth - 1);
3022 return ConstantInt::get(Ty, APInt(BitWidth, Val, /*IsSigned=*/false));
3023 }
3024
3025 APFloat::roundingMode RMode =
3027 bool IsFTZ = nvvm::FPToIntegerIntrinsicShouldFTZ(IntrinsicID);
3028 bool IsSigned = nvvm::FPToIntegerIntrinsicResultIsSigned(IntrinsicID);
3029
3030 APSInt ResInt(Ty->getIntegerBitWidth(), !IsSigned);
3031 auto FloatToRound = IsFTZ ? FTZPreserveSign(U) : U;
3032
3033 // Return max/min value for integers if the result is +/-inf or
3034 // is too large to fit in the result's integer bitwidth.
3035 bool IsExact = false;
3036 FloatToRound.convertToInteger(ResInt, RMode, &IsExact);
3037 return ConstantInt::get(Ty, ResInt);
3038 }
3039 }
3040
3041 /// We only fold functions with finite arguments. Folding NaN and inf is
3042 /// likely to be aborted with an exception anyway, and some host libms
3043 /// have known errors raising exceptions.
3044 if (!U.isFinite())
3045 return nullptr;
3046
3047 /// Currently APFloat versions of these functions do not exist, so we use
3048 /// the host native double versions. Float versions are not called
3049 /// directly but for all these it is true (float)(f((double)arg)) ==
3050 /// f(arg). Long double not supported yet.
3051 const APFloat &APF = Op->getValueAPF();
3052
3053 switch (IntrinsicID) {
3054 default: break;
3055 case Intrinsic::log:
3056 if (U.isZero())
3057 return ConstantFP::getInfinity(Ty, true);
3058 if (U.isNegative())
3059 return ConstantFP::getNaN(Ty);
3060 if (U.isOne())
3061 return ConstantFP::getZero(Ty);
3062 return ConstantFoldFP(log, APF, Ty);
3063 case Intrinsic::log2:
3064 if (U.isZero())
3065 return ConstantFP::getInfinity(Ty, true);
3066 if (U.isNegative())
3067 return ConstantFP::getNaN(Ty);
3068 if (U.isOne())
3069 return ConstantFP::getZero(Ty);
3070 // TODO: What about hosts that lack a C99 library?
3071 return ConstantFoldFP(log2, APF, Ty);
3072 case Intrinsic::log10:
3073 if (U.isZero())
3074 return ConstantFP::getInfinity(Ty, true);
3075 if (U.isNegative())
3076 return ConstantFP::getNaN(Ty);
3077 if (U.isOne())
3078 return ConstantFP::getZero(Ty);
3079 // TODO: What about hosts that lack a C99 library?
3080 return ConstantFoldFP(log10, APF, Ty);
3081 case Intrinsic::exp:
3082 return ConstantFoldFP(exp, APF, Ty);
3083 case Intrinsic::exp2:
3084 // Fold exp2(x) as pow(2, x), in case the host lacks a C99 library.
3085 return ConstantFoldBinaryFP(pow, APFloat(2.0), APF, Ty);
3086 case Intrinsic::exp10:
3087 // Fold exp10(x) as pow(10, x), in case the host lacks a C99 library.
3088 return ConstantFoldBinaryFP(pow, APFloat(10.0), APF, Ty);
3089 case Intrinsic::sin:
3090 return ConstantFoldFP(sin, APF, Ty);
3091 case Intrinsic::cos:
3092 return ConstantFoldFP(cos, APF, Ty);
3093 case Intrinsic::sinh:
3094 return ConstantFoldFP(sinh, APF, Ty);
3095 case Intrinsic::cosh:
3096 return ConstantFoldFP(cosh, APF, Ty);
3097 case Intrinsic::atan:
3098 // Implement optional behavior from C's Annex F for +/-0.0.
3099 if (U.isZero())
3100 return ConstantFP::get(Ty, U);
3101 return ConstantFoldFP(atan, APF, Ty);
3102 case Intrinsic::sqrt:
3103 return ConstantFoldFP(sqrt, APF, Ty);
3104
3105 // NVVM Intrinsics:
3106 case Intrinsic::nvvm_ceil_ftz_f:
3107 case Intrinsic::nvvm_ceil_f:
3108 case Intrinsic::nvvm_ceil_d:
3109 return ConstantFoldFP(
3110 ceil, APF, Ty,
3112 nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID)));
3113
3114 case Intrinsic::nvvm_fabs_ftz:
3115 case Intrinsic::nvvm_fabs:
3116 return ConstantFoldFP(
3117 fabs, APF, Ty,
3119 nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID)));
3120
3121 case Intrinsic::nvvm_floor_ftz_f:
3122 case Intrinsic::nvvm_floor_f:
3123 case Intrinsic::nvvm_floor_d:
3124 return ConstantFoldFP(
3125 floor, APF, Ty,
3127 nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID)));
3128
3129 case Intrinsic::nvvm_rcp_rm_ftz_f:
3130 case Intrinsic::nvvm_rcp_rn_ftz_f:
3131 case Intrinsic::nvvm_rcp_rp_ftz_f:
3132 case Intrinsic::nvvm_rcp_rz_ftz_f:
3133 case Intrinsic::nvvm_rcp_rm_d:
3134 case Intrinsic::nvvm_rcp_rm_f:
3135 case Intrinsic::nvvm_rcp_rn_d:
3136 case Intrinsic::nvvm_rcp_rn_f:
3137 case Intrinsic::nvvm_rcp_rp_d:
3138 case Intrinsic::nvvm_rcp_rp_f:
3139 case Intrinsic::nvvm_rcp_rz_d:
3140 case Intrinsic::nvvm_rcp_rz_f: {
3141 APFloat::roundingMode RoundMode = nvvm::GetRCPRoundingMode(IntrinsicID);
3142 bool IsFTZ = nvvm::RCPShouldFTZ(IntrinsicID);
3143
3144 auto Denominator = IsFTZ ? FTZPreserveSign(APF) : APF;
3146 APFloat::opStatus Status = Res.divide(Denominator, RoundMode);
3147
3149 if (IsFTZ)
3150 Res = FTZPreserveSign(Res);
3151 return ConstantFP::get(Ty, Res);
3152 }
3153 return nullptr;
3154 }
3155
3156 case Intrinsic::nvvm_round_ftz_f:
3157 case Intrinsic::nvvm_round_f:
3158 case Intrinsic::nvvm_round_d: {
3159 // nvvm_round is lowered to PTX cvt.rni, which will round to nearest
3160 // integer, choosing even integer if source is equidistant between two
3161 // integers, so the semantics are closer to "rint" rather than "round".
3162 bool IsFTZ = nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID);
3163 auto V = IsFTZ ? FTZPreserveSign(APF) : APF;
3164 V.roundToIntegral(APFloat::rmNearestTiesToEven);
3165 return ConstantFP::get(Ty, V);
3166 }
3167
3168 case Intrinsic::nvvm_saturate_ftz_f:
3169 case Intrinsic::nvvm_saturate_d:
3170 case Intrinsic::nvvm_saturate_f: {
3171 bool IsFTZ = nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID);
3172 auto V = IsFTZ ? FTZPreserveSign(APF) : APF;
3173 if (V.isNegative() || V.isZero() || V.isNaN())
3174 return ConstantFP::getZero(Ty);
3176 if (V > One)
3177 return ConstantFP::get(Ty, One);
3178 return ConstantFP::get(Ty, APF);
3179 }
3180
3181 case Intrinsic::nvvm_sqrt_rn_ftz_f:
3182 case Intrinsic::nvvm_sqrt_f:
3183 case Intrinsic::nvvm_sqrt_rn_d:
3184 case Intrinsic::nvvm_sqrt_rn_f:
3185 if (APF.isNegative())
3186 return nullptr;
3187 return ConstantFoldFP(
3188 sqrt, APF, Ty,
3190 nvvm::UnaryMathIntrinsicShouldFTZ(IntrinsicID)));
3191
3192 // AMDGCN Intrinsics:
3193 case Intrinsic::amdgcn_cos:
3194 case Intrinsic::amdgcn_sin: {
3195 double V = getValueAsDouble(Op);
3196 if (V < -256.0 || V > 256.0)
3197 // The gfx8 and gfx9 architectures handle arguments outside the range
3198 // [-256, 256] differently. This should be a rare case so bail out
3199 // rather than trying to handle the difference.
3200 return nullptr;
3201 bool IsCos = IntrinsicID == Intrinsic::amdgcn_cos;
3202 double V4 = V * 4.0;
3203 if (V4 == floor(V4)) {
3204 // Force exact results for quarter-integer inputs.
3205 const double SinVals[4] = { 0.0, 1.0, 0.0, -1.0 };
3206 V = SinVals[((int)V4 + (IsCos ? 1 : 0)) & 3];
3207 } else {
3208 if (IsCos)
3209 V = cos(V * 2.0 * numbers::pi);
3210 else
3211 V = sin(V * 2.0 * numbers::pi);
3212 }
3213 return GetConstantFoldFPValue(V, Ty);
3214 }
3215 }
3216
3217 if (!TLI)
3218 return nullptr;
3219
3220 LibFunc Func = TLI->getLibFunc(Name);
3221 if (Func == NotLibFunc)
3222 return nullptr;
3223
3224 switch (Func) {
3225 default:
3226 break;
3227 case LibFunc_acos:
3228 case LibFunc_acosf:
3229 case LibFunc_acos_finite:
3230 case LibFunc_acosf_finite:
3231 if (TLI->has(Func))
3232 return ConstantFoldFP(acos, APF, Ty);
3233 break;
3234 case LibFunc_asin:
3235 case LibFunc_asinf:
3236 case LibFunc_asin_finite:
3237 case LibFunc_asinf_finite:
3238 if (TLI->has(Func))
3239 return ConstantFoldFP(asin, APF, Ty);
3240 break;
3241 case LibFunc_atan:
3242 case LibFunc_atanf:
3243 // Implement optional behavior from C's Annex F for +/-0.0.
3244 if (U.isZero())
3245 return ConstantFP::get(Ty, U);
3246 if (TLI->has(Func))
3247 return ConstantFoldFP(atan, APF, Ty);
3248 break;
3249 case LibFunc_ceil:
3250 case LibFunc_ceilf:
3251 if (TLI->has(Func)) {
3252 U.roundToIntegral(APFloat::rmTowardPositive);
3253 return ConstantFP::get(Ty, U);
3254 }
3255 break;
3256 case LibFunc_cos:
3257 case LibFunc_cosf:
3258 if (TLI->has(Func))
3259 return ConstantFoldFP(cos, APF, Ty);
3260 break;
3261 case LibFunc_cosh:
3262 case LibFunc_coshf:
3263 case LibFunc_cosh_finite:
3264 case LibFunc_coshf_finite:
3265 if (TLI->has(Func))
3266 return ConstantFoldFP(cosh, APF, Ty);
3267 break;
3268 case LibFunc_exp:
3269 case LibFunc_expf:
3270 case LibFunc_exp_finite:
3271 case LibFunc_expf_finite:
3272 if (TLI->has(Func))
3273 return ConstantFoldFP(exp, APF, Ty);
3274 break;
3275 case LibFunc_exp2:
3276 case LibFunc_exp2f:
3277 case LibFunc_exp2_finite:
3278 case LibFunc_exp2f_finite:
3279 if (TLI->has(Func))
3280 // Fold exp2(x) as pow(2, x), in case the host lacks a C99 library.
3281 return ConstantFoldBinaryFP(pow, APFloat(2.0), APF, Ty);
3282 break;
3283 case LibFunc_fabs:
3284 case LibFunc_fabsf:
3285 if (TLI->has(Func)) {
3286 U.clearSign();
3287 return ConstantFP::get(Ty, U);
3288 }
3289 break;
3290 case LibFunc_floor:
3291 case LibFunc_floorf:
3292 if (TLI->has(Func)) {
3293 U.roundToIntegral(APFloat::rmTowardNegative);
3294 return ConstantFP::get(Ty, U);
3295 }
3296 break;
3297 case LibFunc_log:
3298 case LibFunc_logf:
3299 case LibFunc_log_finite:
3300 case LibFunc_logf_finite:
3301 if (!APF.isNegative() && !APF.isZero() && TLI->has(Func))
3302 return ConstantFoldFP(log, APF, Ty);
3303 break;
3304 case LibFunc_log2:
3305 case LibFunc_log2f:
3306 case LibFunc_log2_finite:
3307 case LibFunc_log2f_finite:
3308 if (!APF.isNegative() && !APF.isZero() && TLI->has(Func))
3309 // TODO: What about hosts that lack a C99 library?
3310 return ConstantFoldFP(log2, APF, Ty);
3311 break;
3312 case LibFunc_log10:
3313 case LibFunc_log10f:
3314 case LibFunc_log10_finite:
3315 case LibFunc_log10f_finite:
3316 if (!APF.isNegative() && !APF.isZero() && TLI->has(Func))
3317 // TODO: What about hosts that lack a C99 library?
3318 return ConstantFoldFP(log10, APF, Ty);
3319 break;
3320 case LibFunc_ilogb:
3321 case LibFunc_ilogbf:
3322 if (!APF.isZero() && TLI->has(Func))
3323 return ConstantInt::get(Ty, ilogb(APF), true);
3324 break;
3325 case LibFunc_logb:
3326 case LibFunc_logbf:
3327 if (!APF.isZero() && TLI->has(Func))
3328 return ConstantFoldFP(logb, APF, Ty);
3329 break;
3330 case LibFunc_log1p:
3331 case LibFunc_log1pf:
3332 // Implement optional behavior from C's Annex F for +/-0.0.
3333 if (U.isZero())
3334 return ConstantFP::get(Ty, U);
3335 if (APF > APFloat::getOne(APF.getSemantics(), true) && TLI->has(Func))
3336 return ConstantFoldFP(log1p, APF, Ty);
3337 break;
3338 case LibFunc_logl:
3339 return nullptr;
3340 case LibFunc_erf:
3341 case LibFunc_erff:
3342 if (TLI->has(Func))
3343 return ConstantFoldFP(erf, APF, Ty);
3344 break;
3345 case LibFunc_nearbyint:
3346 case LibFunc_nearbyintf:
3347 case LibFunc_rint:
3348 case LibFunc_rintf:
3349 case LibFunc_roundeven:
3350 case LibFunc_roundevenf:
3351 if (TLI->has(Func)) {
3352 U.roundToIntegral(APFloat::rmNearestTiesToEven);
3353 return ConstantFP::get(Ty, U);
3354 }
3355 break;
3356 case LibFunc_round:
3357 case LibFunc_roundf:
3358 if (TLI->has(Func)) {
3359 U.roundToIntegral(APFloat::rmNearestTiesToAway);
3360 return ConstantFP::get(Ty, U);
3361 }
3362 break;
3363 case LibFunc_sin:
3364 case LibFunc_sinf:
3365 if (TLI->has(Func))
3366 return ConstantFoldFP(sin, APF, Ty);
3367 break;
3368 case LibFunc_sinh:
3369 case LibFunc_sinhf:
3370 case LibFunc_sinh_finite:
3371 case LibFunc_sinhf_finite:
3372 if (TLI->has(Func))
3373 return ConstantFoldFP(sinh, APF, Ty);
3374 break;
3375 case LibFunc_sqrt:
3376 case LibFunc_sqrtf:
3377 if (!APF.isNegative() && TLI->has(Func))
3378 return ConstantFoldFP(sqrt, APF, Ty);
3379 break;
3380 case LibFunc_tan:
3381 case LibFunc_tanf:
3382 if (TLI->has(Func))
3383 return ConstantFoldFP(tan, APF, Ty);
3384 break;
3385 case LibFunc_tanh:
3386 case LibFunc_tanhf:
3387 if (TLI->has(Func))
3388 return ConstantFoldFP(tanh, APF, Ty);
3389 break;
3390 case LibFunc_trunc:
3391 case LibFunc_truncf:
3392 if (TLI->has(Func)) {
3393 U.roundToIntegral(APFloat::rmTowardZero);
3394 return ConstantFP::get(Ty, U);
3395 }
3396 break;
3397 }
3398 return nullptr;
3399 }
3400
3401 if (auto *Op = dyn_cast<ConstantInt>(Operands[0])) {
3402 switch (IntrinsicID) {
3403 case Intrinsic::bswap:
3404 return ConstantInt::get(Ty->getContext(), Op->getValue().byteSwap());
3405 case Intrinsic::ctpop:
3406 return ConstantInt::get(Ty, Op->getValue().popcount());
3407 case Intrinsic::bitreverse:
3408 return ConstantInt::get(Ty->getContext(), Op->getValue().reverseBits());
3409 case Intrinsic::amdgcn_s_wqm: {
3410 uint64_t Val = Op->getZExtValue();
3411 Val |= (Val & 0x5555555555555555ULL) << 1 |
3412 ((Val >> 1) & 0x5555555555555555ULL);
3413 Val |= (Val & 0x3333333333333333ULL) << 2 |
3414 ((Val >> 2) & 0x3333333333333333ULL);
3415 return ConstantInt::get(Ty, Val);
3416 }
3417
3418 case Intrinsic::amdgcn_s_quadmask: {
3419 uint64_t Val = Op->getZExtValue();
3420 uint64_t QuadMask = 0;
3421 for (unsigned I = 0; I < Op->getBitWidth() / 4; ++I, Val >>= 4) {
3422 if (!(Val & 0xF))
3423 continue;
3424
3425 QuadMask |= (1ULL << I);
3426 }
3427 return ConstantInt::get(Ty, QuadMask);
3428 }
3429
3430 case Intrinsic::amdgcn_s_bitreplicate: {
3431 uint64_t Val = Op->getZExtValue();
3432 Val = (Val & 0x000000000000FFFFULL) | (Val & 0x00000000FFFF0000ULL) << 16;
3433 Val = (Val & 0x000000FF000000FFULL) | (Val & 0x0000FF000000FF00ULL) << 8;
3434 Val = (Val & 0x000F000F000F000FULL) | (Val & 0x00F000F000F000F0ULL) << 4;
3435 Val = (Val & 0x0303030303030303ULL) | (Val & 0x0C0C0C0C0C0C0C0CULL) << 2;
3436 Val = (Val & 0x1111111111111111ULL) | (Val & 0x2222222222222222ULL) << 1;
3437 Val = Val | Val << 1;
3438 return ConstantInt::get(Ty, Val);
3439 }
3440 }
3441 }
3442
3443 if (Operands[0]->getType()->isVectorTy()) {
3444 auto *Op = cast<Constant>(Operands[0]);
3445 switch (IntrinsicID) {
3446 default: break;
3447 case Intrinsic::vector_reduce_add:
3448 case Intrinsic::vector_reduce_mul:
3449 case Intrinsic::vector_reduce_and:
3450 case Intrinsic::vector_reduce_or:
3451 case Intrinsic::vector_reduce_xor:
3452 case Intrinsic::vector_reduce_smin:
3453 case Intrinsic::vector_reduce_smax:
3454 case Intrinsic::vector_reduce_umin:
3455 case Intrinsic::vector_reduce_umax:
3456 if (Constant *C = constantFoldVectorReduce(IntrinsicID, Operands[0]))
3457 return C;
3458 break;
3459 case Intrinsic::x86_sse_cvtss2si:
3460 case Intrinsic::x86_sse_cvtss2si64:
3461 case Intrinsic::x86_sse2_cvtsd2si:
3462 case Intrinsic::x86_sse2_cvtsd2si64:
3463 if (ConstantFP *FPOp =
3464 dyn_cast_or_null<ConstantFP>(Op->getAggregateElement(0U)))
3465 return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(),
3466 /*roundTowardZero=*/false, Ty,
3467 /*IsSigned*/true);
3468 break;
3469 case Intrinsic::x86_sse_cvttss2si:
3470 case Intrinsic::x86_sse_cvttss2si64:
3471 case Intrinsic::x86_sse2_cvttsd2si:
3472 case Intrinsic::x86_sse2_cvttsd2si64:
3473 if (ConstantFP *FPOp =
3474 dyn_cast_or_null<ConstantFP>(Op->getAggregateElement(0U)))
3475 return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(),
3476 /*roundTowardZero=*/true, Ty,
3477 /*IsSigned*/true);
3478 break;
3479
3480 case Intrinsic::wasm_anytrue:
3481 return Op->isNullValue() ? ConstantInt::get(Ty, 0)
3482 : ConstantInt::get(Ty, 1);
3483
3484 case Intrinsic::wasm_alltrue:
3485 // Check each element individually
3486 unsigned E = cast<FixedVectorType>(Op->getType())->getNumElements();
3487 for (unsigned I = 0; I != E; ++I) {
3488 Constant *Elt = Op->getAggregateElement(I);
3489 // Return false as soon as we find a non-true element.
3490 if (Elt && Elt->isNullValue())
3491 return ConstantInt::get(Ty, 0);
3492 // Bail as soon as we find an element we cannot prove to be true.
3493 if (!Elt || !isa<ConstantInt>(Elt))
3494 return nullptr;
3495 }
3496
3497 return ConstantInt::get(Ty, 1);
3498 }
3499 }
3500
3501 return nullptr;
3502}
3503
3504static Constant *evaluateCompare(const APFloat &Op1, const APFloat &Op2,
3508 FCmpInst::Predicate Cond = FCmp->getPredicate();
3509 if (FCmp->isSignaling()) {
3510 if (Op1.isNaN() || Op2.isNaN())
3512 } else {
3513 if (Op1.isSignaling() || Op2.isSignaling())
3515 }
3516 bool Result = FCmpInst::compare(Op1, Op2, Cond);
3517 if (mayFoldConstrained(const_cast<ConstrainedFPCmpIntrinsic *>(FCmp), St))
3518 return ConstantInt::get(Call->getType()->getScalarType(), Result);
3519 return nullptr;
3520}
3521
3522static Constant *ConstantFoldNextToward(const APFloat &Op0, const APFloat &Op1,
3523 const Type *RetTy) {
3524 assert(RetTy != nullptr);
3525 bool LosesInfo;
3526
3527 if (Op1.isSignaling())
3528 return nullptr;
3529 if (Op1.isNaN()) {
3530 APFloat Ret(Op1);
3531 Ret.convert(RetTy->getFltSemantics(), detail::rmNearestTiesToEven,
3532 &LosesInfo);
3533 return ConstantFP::get(RetTy->getContext(), Ret);
3534 }
3535
3536 // Recall that the second argument of nexttoward is always a long double,
3537 // so we may need to promote the first argument for comparisons to be valid.
3538 APFloat PromotedOp0(Op0);
3539 PromotedOp0.convert(Op1.getSemantics(), detail::rmNearestTiesToEven,
3540 &LosesInfo);
3541 assert(!LosesInfo && "Unexpected lossy promotion");
3542 const APFloat::cmpResult Result = PromotedOp0.compare(Op1);
3543
3544 // When equal, the standard says we must return the second argument.
3545 // This allows nice behavior such as nexttoward(0.0, -0.0) = -0.0 and
3546 // nexttoward(-0.0, 0.0) = 0.0
3547 if (Result == detail::cmpEqual) {
3548 APFloat Ret(Op1);
3549 Ret.convert(RetTy->getFltSemantics(), detail::rmNearestTiesToEven,
3550 &LosesInfo);
3551 return ConstantFP::get(RetTy->getContext(), Ret);
3552 }
3553
3554 APFloat Next(Op0);
3555 Next.next(/*nextDown=*/Result == APFloat::cmpGreaterThan);
3556 if (Next.isZero() || Next.isDenormal() || Next.isSignaling())
3557 return nullptr;
3558 return ConstantFP::get(RetTy->getContext(), Next);
3559}
3560
3561static Constant *ConstantFoldLibCall2(StringRef Name, Type *Ty,
3563 const TargetLibraryInfo *TLI = nullptr) {
3564 if (!TLI)
3565 return nullptr;
3566
3567 LibFunc Func = TLI->getLibFunc(Name);
3568 if (Func == NotLibFunc)
3569 return nullptr;
3570
3571 const auto *Op1 = dyn_cast<ConstantFP>(Operands[0]);
3572 if (!Op1)
3573 return nullptr;
3574
3575 const auto *Op2 = dyn_cast<ConstantFP>(Operands[1]);
3576 if (!Op2)
3577 return nullptr;
3578
3579 const APFloat &Op1V = Op1->getValueAPF();
3580 const APFloat &Op2V = Op2->getValueAPF();
3581
3582 switch (Func) {
3583 default:
3584 break;
3585 case LibFunc_pow:
3586 case LibFunc_powf:
3587 case LibFunc_pow_finite:
3588 case LibFunc_powf_finite:
3589 if (TLI->has(Func))
3590 return ConstantFoldBinaryFP(pow, Op1V, Op2V, Ty);
3591 break;
3592 case LibFunc_fmod:
3593 case LibFunc_fmodf:
3594 if (TLI->has(Func)) {
3595 APFloat V = Op1->getValueAPF();
3596 if (APFloat::opStatus::opOK == V.mod(Op2->getValueAPF()))
3597 return ConstantFP::get(Ty, V);
3598 }
3599 break;
3600 case LibFunc_remainder:
3601 case LibFunc_remainderf:
3602 if (TLI->has(Func)) {
3603 APFloat V = Op1->getValueAPF();
3604 if (APFloat::opStatus::opOK == V.remainder(Op2->getValueAPF()))
3605 return ConstantFP::get(Ty, V);
3606 }
3607 break;
3608 case LibFunc_atan2:
3609 case LibFunc_atan2f:
3610 // atan2(+/-0.0, +/-0.0) is known to raise an exception on some libm
3611 // (Solaris), so we do not assume a known result for that.
3612 if (Op1V.isZero() && Op2V.isZero())
3613 return nullptr;
3614 [[fallthrough]];
3615 case LibFunc_atan2_finite:
3616 case LibFunc_atan2f_finite:
3617 if (TLI->has(Func))
3618 return ConstantFoldBinaryFP(atan2, Op1V, Op2V, Ty);
3619 break;
3620 case LibFunc_nextafter:
3621 case LibFunc_nextafterf:
3622 case LibFunc_nexttoward:
3623 case LibFunc_nexttowardf:
3624 if (TLI->has(Func))
3625 return ConstantFoldNextToward(Op1V, Op2V, Ty);
3626 break;
3627 }
3628
3629 return nullptr;
3630}
3631
3632static Constant *ConstantFoldCRC32(Type *Ty, const APInt *CrcArg,
3633 const APInt *DataArg, unsigned DataBytes,
3634 uint32_t Poly) {
3635 if (!CrcArg || !DataArg)
3636 return nullptr;
3637 uint32_t Crc = CrcArg->getZExtValue();
3638 uint64_t Data = DataArg->getZExtValue();
3639 uint32_t Result = calculateReflectedCRC32(Crc, Data, DataBytes, Poly);
3640 return ConstantInt::get(Ty, Result);
3641}
3642
3643static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
3645 const CallBase *Call = nullptr) {
3646 assert(Operands.size() == 2 && "Wrong number of operands.");
3647
3648 if (Ty->isFloatingPointTy()) {
3649 // TODO: We should have undef handling for all of the FP intrinsics that
3650 // are attempted to be folded in this function.
3651 bool IsOp0Undef = isa<UndefValue>(Operands[0]);
3652 bool IsOp1Undef = isa<UndefValue>(Operands[1]);
3653 switch (IntrinsicID) {
3654 case Intrinsic::maxnum:
3655 case Intrinsic::minnum:
3656 case Intrinsic::maximum:
3657 case Intrinsic::minimum:
3658 case Intrinsic::maximumnum:
3659 case Intrinsic::minimumnum:
3660 case Intrinsic::nvvm_fmax_d:
3661 case Intrinsic::nvvm_fmin_d:
3662 // If one argument is undef, return the other argument.
3663 if (IsOp0Undef)
3664 return Operands[1];
3665 if (IsOp1Undef)
3666 return Operands[0];
3667 break;
3668
3669 case Intrinsic::nvvm_fmax_f:
3670 case Intrinsic::nvvm_fmax_ftz_f:
3671 case Intrinsic::nvvm_fmax_ftz_nan_f:
3672 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
3673 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
3674 case Intrinsic::nvvm_fmax_nan_f:
3675 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
3676 case Intrinsic::nvvm_fmax_xorsign_abs_f:
3677
3678 case Intrinsic::nvvm_fmin_f:
3679 case Intrinsic::nvvm_fmin_ftz_f:
3680 case Intrinsic::nvvm_fmin_ftz_nan_f:
3681 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
3682 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
3683 case Intrinsic::nvvm_fmin_nan_f:
3684 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
3685 case Intrinsic::nvvm_fmin_xorsign_abs_f:
3686 // If one arg is undef, the other arg can be returned only if it is
3687 // constant, as we may need to flush it to sign-preserving zero or
3688 // canonicalize the NaN.
3689 if (!IsOp0Undef && !IsOp1Undef)
3690 break;
3691 if (auto *Op = dyn_cast<ConstantFP>(Operands[IsOp0Undef ? 1 : 0])) {
3692 if (Op->isNaN()) {
3693 APInt NVCanonicalNaN(32, 0x7fffffff);
3694 return ConstantFP::get(
3695 Ty, APFloat(Ty->getFltSemantics(), NVCanonicalNaN));
3696 }
3697 if (nvvm::FMinFMaxShouldFTZ(IntrinsicID))
3698 return ConstantFP::get(Ty, FTZPreserveSign(Op->getValueAPF()));
3699 else
3700 return Op;
3701 }
3702 break;
3703 }
3704 }
3705
3706 if (const auto *Op1 = dyn_cast<ConstantFP>(Operands[0])) {
3707 const APFloat &Op1V = Op1->getValueAPF();
3708
3709 if (const auto *Op2 = dyn_cast<ConstantFP>(Operands[1])) {
3710 if (Op2->getType() != Op1->getType())
3711 return nullptr;
3712 const APFloat &Op2V = Op2->getValueAPF();
3713
3714 if (const auto *ConstrIntr =
3716 RoundingMode RM = getEvaluationRoundingMode(ConstrIntr);
3717 APFloat Res = Op1V;
3719 switch (IntrinsicID) {
3720 default:
3721 return nullptr;
3722 case Intrinsic::experimental_constrained_fadd:
3723 St = Res.add(Op2V, RM);
3724 break;
3725 case Intrinsic::experimental_constrained_fsub:
3726 St = Res.subtract(Op2V, RM);
3727 break;
3728 case Intrinsic::experimental_constrained_fmul:
3729 St = Res.multiply(Op2V, RM);
3730 break;
3731 case Intrinsic::experimental_constrained_fdiv:
3732 St = Res.divide(Op2V, RM);
3733 break;
3734 case Intrinsic::experimental_constrained_frem:
3735 St = Res.mod(Op2V);
3736 break;
3737 case Intrinsic::experimental_constrained_fcmp:
3738 case Intrinsic::experimental_constrained_fcmps:
3739 return evaluateCompare(Op1V, Op2V, ConstrIntr);
3740 }
3741 if (mayFoldConstrained(const_cast<ConstrainedFPIntrinsic *>(ConstrIntr),
3742 St))
3743 return ConstantFP::get(Ty, Res);
3744 return nullptr;
3745 }
3746
3747 switch (IntrinsicID) {
3748 default:
3749 break;
3750 case Intrinsic::copysign:
3751 return ConstantFP::get(Ty, APFloat::copySign(Op1V, Op2V));
3752 case Intrinsic::minnum:
3753 return ConstantFP::get(Ty, minnum(Op1V, Op2V));
3754 case Intrinsic::maxnum:
3755 return ConstantFP::get(Ty, maxnum(Op1V, Op2V));
3756 case Intrinsic::minimum:
3757 return ConstantFP::get(Ty, minimum(Op1V, Op2V));
3758 case Intrinsic::maximum:
3759 return ConstantFP::get(Ty, maximum(Op1V, Op2V));
3760 case Intrinsic::minimumnum:
3761 return ConstantFP::get(Ty, minimumnum(Op1V, Op2V));
3762 case Intrinsic::maximumnum:
3763 return ConstantFP::get(Ty, maximumnum(Op1V, Op2V));
3764
3765 case Intrinsic::nvvm_fmax_d:
3766 case Intrinsic::nvvm_fmax_f:
3767 case Intrinsic::nvvm_fmax_ftz_f:
3768 case Intrinsic::nvvm_fmax_ftz_nan_f:
3769 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
3770 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
3771 case Intrinsic::nvvm_fmax_nan_f:
3772 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
3773 case Intrinsic::nvvm_fmax_xorsign_abs_f:
3774
3775 case Intrinsic::nvvm_fmin_d:
3776 case Intrinsic::nvvm_fmin_f:
3777 case Intrinsic::nvvm_fmin_ftz_f:
3778 case Intrinsic::nvvm_fmin_ftz_nan_f:
3779 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
3780 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
3781 case Intrinsic::nvvm_fmin_nan_f:
3782 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
3783 case Intrinsic::nvvm_fmin_xorsign_abs_f: {
3784
3785 bool ShouldCanonicalizeNaNs = !(IntrinsicID == Intrinsic::nvvm_fmax_d ||
3786 IntrinsicID == Intrinsic::nvvm_fmin_d);
3787 bool IsFTZ = nvvm::FMinFMaxShouldFTZ(IntrinsicID);
3788 bool IsNaNPropagating = nvvm::FMinFMaxPropagatesNaNs(IntrinsicID);
3789 bool IsXorSignAbs = nvvm::FMinFMaxIsXorSignAbs(IntrinsicID);
3790
3791 APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
3792 APFloat B = IsFTZ ? FTZPreserveSign(Op2V) : Op2V;
3793
3794 bool XorSign = false;
3795 if (IsXorSignAbs) {
3796 XorSign = A.isNegative() ^ B.isNegative();
3797 A = abs(A);
3798 B = abs(B);
3799 }
3800
3801 bool IsFMax = false;
3802 switch (IntrinsicID) {
3803 case Intrinsic::nvvm_fmax_d:
3804 case Intrinsic::nvvm_fmax_f:
3805 case Intrinsic::nvvm_fmax_ftz_f:
3806 case Intrinsic::nvvm_fmax_ftz_nan_f:
3807 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
3808 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
3809 case Intrinsic::nvvm_fmax_nan_f:
3810 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
3811 case Intrinsic::nvvm_fmax_xorsign_abs_f:
3812 IsFMax = true;
3813 break;
3814 }
3815 APFloat Res =
3816 IsFMax ? (IsNaNPropagating ? maximum(A, B) : maximumnum(A, B))
3817 : (IsNaNPropagating ? minimum(A, B) : minimumnum(A, B));
3818
3819 if (ShouldCanonicalizeNaNs && Res.isNaN()) {
3820 APFloat NVCanonicalNaN(Res.getSemantics(), APInt(32, 0x7fffffff));
3821 return ConstantFP::get(Ty, NVCanonicalNaN);
3822 }
3823
3824 if (IsXorSignAbs && XorSign != Res.isNegative())
3825 Res.changeSign();
3826
3827 return ConstantFP::get(Ty, Res);
3828 }
3829
3830 case Intrinsic::nvvm_div_rm_f:
3831 case Intrinsic::nvvm_div_rn_f:
3832 case Intrinsic::nvvm_div_rp_f:
3833 case Intrinsic::nvvm_div_rz_f:
3834 case Intrinsic::nvvm_div_rm_d:
3835 case Intrinsic::nvvm_div_rn_d:
3836 case Intrinsic::nvvm_div_rp_d:
3837 case Intrinsic::nvvm_div_rz_d:
3838 case Intrinsic::nvvm_div_rm_ftz_f:
3839 case Intrinsic::nvvm_div_rn_ftz_f:
3840 case Intrinsic::nvvm_div_rp_ftz_f:
3841 case Intrinsic::nvvm_div_rz_ftz_f: {
3842 bool IsFTZ = nvvm::FDivShouldFTZ(IntrinsicID);
3843 APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
3844 APFloat B = IsFTZ ? FTZPreserveSign(Op2V) : Op2V;
3845 APFloat::roundingMode RoundMode =
3846 nvvm::GetFDivRoundingMode(IntrinsicID);
3847
3848 APFloat Res = A;
3849 APFloat::opStatus Status = Res.divide(B, RoundMode);
3850 if (!Res.isNaN() &&
3852 Res = IsFTZ ? FTZPreserveSign(Res) : Res;
3853 return ConstantFP::get(Ty, Res);
3854 }
3855 return nullptr;
3856 }
3857 }
3858
3859 if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy())
3860 return nullptr;
3861
3862 switch (IntrinsicID) {
3863 default:
3864 break;
3865 case Intrinsic::pow:
3866 return ConstantFoldBinaryFP(pow, Op1V, Op2V, Ty);
3867 case Intrinsic::amdgcn_fmul_legacy:
3868 // The legacy behaviour is that multiplying +/- 0.0 by anything, even
3869 // NaN or infinity, gives +0.0.
3870 if (Op1V.isZero() || Op2V.isZero())
3871 return ConstantFP::getZero(Ty);
3872 return ConstantFP::get(Ty, Op1V * Op2V);
3873 }
3874
3875 } else if (auto *Op2C = dyn_cast<ConstantInt>(Operands[1])) {
3876 switch (IntrinsicID) {
3877 case Intrinsic::ldexp: {
3878 // APFloat::scalbn takes the exponent as `int`. Clamp wider integer
3879 // exponents into [INT_MIN, INT_MAX] so values still saturate the
3880 // result to +/-inf or +/-0.
3881 APInt Exp = Op2C->getValue();
3882 Exp = Exp.getBitWidth() < 32 ? Exp.sext(32) : Exp.truncSSat(32);
3883 return ConstantFP::get(
3884 Ty->getContext(),
3885 scalbn(Op1V, Exp.getSExtValue(), APFloat::rmNearestTiesToEven));
3886 }
3887 case Intrinsic::is_fpclass: {
3888 FPClassTest Mask = static_cast<FPClassTest>(Op2C->getZExtValue());
3889 bool Result =
3890 ((Mask & fcSNan) && Op1V.isNaN() && Op1V.isSignaling()) ||
3891 ((Mask & fcQNan) && Op1V.isNaN() && !Op1V.isSignaling()) ||
3892 ((Mask & fcNegInf) && Op1V.isNegInfinity()) ||
3893 ((Mask & fcNegNormal) && Op1V.isNormal() && Op1V.isNegative()) ||
3894 ((Mask & fcNegSubnormal) && Op1V.isDenormal() && Op1V.isNegative()) ||
3895 ((Mask & fcNegZero) && Op1V.isZero() && Op1V.isNegative()) ||
3896 ((Mask & fcPosZero) && Op1V.isZero() && !Op1V.isNegative()) ||
3897 ((Mask & fcPosSubnormal) && Op1V.isDenormal() && !Op1V.isNegative()) ||
3898 ((Mask & fcPosNormal) && Op1V.isNormal() && !Op1V.isNegative()) ||
3899 ((Mask & fcPosInf) && Op1V.isPosInfinity());
3900 return ConstantInt::get(Ty, Result);
3901 }
3902 case Intrinsic::powi: {
3903 // Square-and-multiply using the operand's own semantics, matching
3904 // the multiply sequence ExpandPowI builds in SelectionDAG.
3905 int Exp = static_cast<int>(Op2C->getSExtValue());
3906 unsigned UExp = static_cast<unsigned>(Exp);
3907 if (Exp < 0)
3908 UExp = -UExp;
3909 const fltSemantics &Semantics = Op1V.getSemantics();
3910 APFloat Res = APFloat::getOne(Semantics);
3911 APFloat CurSquare = Op1V;
3912 while (UExp) {
3913 if (UExp & 1)
3914 Res = Res * CurSquare;
3915 CurSquare = CurSquare * CurSquare;
3916 UExp >>= 1;
3917 }
3918 if (Exp < 0)
3919 Res = APFloat::getOne(Semantics) / Res;
3920 return ConstantFP::get(Ty, Res);
3921 }
3922 default:
3923 break;
3924 }
3925 }
3926 return nullptr;
3927 }
3928
3929 if (Operands[0]->getType()->isIntegerTy() &&
3930 Operands[1]->getType()->isIntegerTy()) {
3931 const APInt *C0, *C1;
3932 if (!getConstIntOrUndef(Operands[0], C0) ||
3933 !getConstIntOrUndef(Operands[1], C1))
3934 return nullptr;
3935
3936 switch (IntrinsicID) {
3937 default: break;
3938 case Intrinsic::smax:
3939 case Intrinsic::smin:
3940 case Intrinsic::umax:
3941 case Intrinsic::umin:
3942 if (!C0 || !C1)
3943 return MinMaxIntrinsic::getSaturationPoint(IntrinsicID, Ty);
3944 return ConstantInt::get(
3945 Ty, ICmpInst::compare(*C0, *C1,
3946 MinMaxIntrinsic::getPredicate(IntrinsicID))
3947 ? *C0
3948 : *C1);
3949
3950 case Intrinsic::scmp:
3951 case Intrinsic::ucmp:
3952 if (!C0 || !C1)
3953 return ConstantInt::get(Ty, 0);
3954
3955 int Res;
3956 if (IntrinsicID == Intrinsic::scmp)
3957 Res = C0->sgt(*C1) ? 1 : C0->slt(*C1) ? -1 : 0;
3958 else
3959 Res = C0->ugt(*C1) ? 1 : C0->ult(*C1) ? -1 : 0;
3960 return ConstantInt::get(Ty, Res, /*IsSigned=*/true);
3961
3962 case Intrinsic::usub_with_overflow:
3963 case Intrinsic::ssub_with_overflow:
3964 // X - undef -> { 0, false }
3965 // undef - X -> { 0, false }
3966 if (!C0 || !C1)
3967 return Constant::getNullValue(Ty);
3968 [[fallthrough]];
3969 case Intrinsic::uadd_with_overflow:
3970 case Intrinsic::sadd_with_overflow:
3971 // X + undef -> { -1, false }
3972 // undef + x -> { -1, false }
3973 if (!C0 || !C1) {
3974 return ConstantStruct::get(
3975 cast<StructType>(Ty),
3976 {Constant::getAllOnesValue(Ty->getStructElementType(0)),
3977 Constant::getNullValue(Ty->getStructElementType(1))});
3978 }
3979 [[fallthrough]];
3980 case Intrinsic::smul_with_overflow:
3981 case Intrinsic::umul_with_overflow: {
3982 // undef * X -> { 0, false }
3983 // X * undef -> { 0, false }
3984 if (!C0 || !C1)
3985 return Constant::getNullValue(Ty);
3986
3987 APInt Res;
3988 bool Overflow;
3989 switch (IntrinsicID) {
3990 default: llvm_unreachable("Invalid case");
3991 case Intrinsic::sadd_with_overflow:
3992 Res = C0->sadd_ov(*C1, Overflow);
3993 break;
3994 case Intrinsic::uadd_with_overflow:
3995 Res = C0->uadd_ov(*C1, Overflow);
3996 break;
3997 case Intrinsic::ssub_with_overflow:
3998 Res = C0->ssub_ov(*C1, Overflow);
3999 break;
4000 case Intrinsic::usub_with_overflow:
4001 Res = C0->usub_ov(*C1, Overflow);
4002 break;
4003 case Intrinsic::smul_with_overflow:
4004 Res = C0->smul_ov(*C1, Overflow);
4005 break;
4006 case Intrinsic::umul_with_overflow:
4007 Res = C0->umul_ov(*C1, Overflow);
4008 break;
4009 }
4010 Constant *Ops[] = {
4011 ConstantInt::get(Ty->getContext(), Res),
4012 ConstantInt::get(Type::getInt1Ty(Ty->getContext()), Overflow)
4013 };
4015 }
4016 case Intrinsic::uadd_sat:
4017 case Intrinsic::sadd_sat:
4018 if (!C0 || !C1)
4019 return Constant::getAllOnesValue(Ty);
4020 if (IntrinsicID == Intrinsic::uadd_sat)
4021 return ConstantInt::get(Ty, C0->uadd_sat(*C1));
4022 else
4023 return ConstantInt::get(Ty, C0->sadd_sat(*C1));
4024 case Intrinsic::usub_sat:
4025 case Intrinsic::ssub_sat:
4026 if (!C0 || !C1)
4027 return Constant::getNullValue(Ty);
4028 if (IntrinsicID == Intrinsic::usub_sat)
4029 return ConstantInt::get(Ty, C0->usub_sat(*C1));
4030 else
4031 return ConstantInt::get(Ty, C0->ssub_sat(*C1));
4032 case Intrinsic::ushl_sat:
4033 case Intrinsic::sshl_sat:
4034 if (!C0 || !C1)
4035 return nullptr;
4036 if (C1->uge(C1->getBitWidth()))
4037 return PoisonValue::get(Ty);
4038 if (IntrinsicID == Intrinsic::ushl_sat)
4039 return ConstantInt::get(Ty, C0->ushl_sat(*C1));
4040 return ConstantInt::get(Ty, C0->sshl_sat(*C1));
4041 case Intrinsic::cttz:
4042 case Intrinsic::ctlz:
4043 assert(C1 && "Must be constant int");
4044
4045 // cttz(0, 1) and ctlz(0, 1) are poison.
4046 if (C1->isOne() && (!C0 || C0->isZero()))
4047 return PoisonValue::get(Ty);
4048 if (!C0)
4049 return Constant::getNullValue(Ty);
4050 if (IntrinsicID == Intrinsic::cttz)
4051 return ConstantInt::get(Ty, C0->countr_zero());
4052 else
4053 return ConstantInt::get(Ty, C0->countl_zero());
4054
4055 case Intrinsic::abs:
4056 assert(C1 && "Must be constant int");
4057 assert((C1->isOne() || C1->isZero()) && "Must be 0 or 1");
4058
4059 // Undef or minimum val operand with poison min --> poison
4060 if (C1->isOne() && (!C0 || C0->isMinSignedValue()))
4061 return PoisonValue::get(Ty);
4062
4063 // Undef operand with no poison min --> 0 (sign bit must be clear)
4064 if (!C0)
4065 return Constant::getNullValue(Ty);
4066
4067 return ConstantInt::get(Ty, C0->abs());
4068 case Intrinsic::clmul:
4069 if (!C0 || !C1)
4070 return Constant::getNullValue(Ty);
4071 return ConstantInt::get(Ty, APIntOps::clmul(*C0, *C1));
4072 case Intrinsic::pdep:
4073 if (!C0 || !C1)
4074 return Constant::getNullValue(Ty);
4075 return ConstantInt::get(Ty, APIntOps::pdep(*C0, *C1));
4076 case Intrinsic::pext:
4077 if (!C0 || !C1)
4078 return Constant::getNullValue(Ty);
4079 return ConstantInt::get(Ty, APIntOps::pext(*C0, *C1));
4080 case Intrinsic::smulh:
4081 if (!C0 || !C1)
4082 return Constant::getNullValue(Ty);
4083 return ConstantInt::get(Ty, APIntOps::mulhs(*C0, *C1));
4084 case Intrinsic::umulh:
4085 if (!C0 || !C1)
4086 return Constant::getNullValue(Ty);
4087 return ConstantInt::get(Ty, APIntOps::mulhu(*C0, *C1));
4088 case Intrinsic::amdgcn_wave_reduce_add:
4089 case Intrinsic::amdgcn_wave_reduce_sub:
4090 case Intrinsic::amdgcn_wave_reduce_xor: {
4091 if (C0 && C0->isZero())
4092 return Constant::getNullValue(Ty);
4093 return nullptr;
4094 }
4095 case Intrinsic::amdgcn_wave_reduce_umin:
4096 case Intrinsic::amdgcn_wave_reduce_umax:
4097 case Intrinsic::amdgcn_wave_reduce_max:
4098 case Intrinsic::amdgcn_wave_reduce_min:
4099 case Intrinsic::amdgcn_wave_reduce_and:
4100 case Intrinsic::amdgcn_wave_reduce_or:
4101 return Operands[0];
4102 case Intrinsic::aarch64_crc32b:
4103 case Intrinsic::arm_crc32b:
4104 return ConstantFoldCRC32(Ty, C0, C1, 1, 0xEDB88320);
4105 case Intrinsic::aarch64_crc32h:
4106 case Intrinsic::arm_crc32h:
4107 return ConstantFoldCRC32(Ty, C0, C1, 2, 0xEDB88320);
4108 case Intrinsic::aarch64_crc32w:
4109 case Intrinsic::arm_crc32w:
4110 return ConstantFoldCRC32(Ty, C0, C1, 4, 0xEDB88320);
4111 case Intrinsic::aarch64_crc32x:
4112 return ConstantFoldCRC32(Ty, C0, C1, 8, 0xEDB88320);
4113 case Intrinsic::aarch64_crc32cb:
4114 case Intrinsic::arm_crc32cb:
4115 case Intrinsic::x86_sse42_crc32_32_8:
4116 return ConstantFoldCRC32(Ty, C0, C1, 1, 0x82F63B78);
4117 case Intrinsic::aarch64_crc32ch:
4118 case Intrinsic::arm_crc32ch:
4119 case Intrinsic::x86_sse42_crc32_32_16:
4120 return ConstantFoldCRC32(Ty, C0, C1, 2, 0x82F63B78);
4121 case Intrinsic::aarch64_crc32cw:
4122 case Intrinsic::arm_crc32cw:
4123 case Intrinsic::x86_sse42_crc32_32_32:
4124 return ConstantFoldCRC32(Ty, C0, C1, 4, 0x82F63B78);
4125 case Intrinsic::aarch64_crc32cx:
4126 case Intrinsic::x86_sse42_crc32_64_64:
4127 return ConstantFoldCRC32(Ty, C0, C1, 8, 0x82F63B78);
4128 }
4129
4130 return nullptr;
4131 }
4132
4133 // Support ConstantVector in case we have an Undef in the top.
4134 if ((isa<ConstantVector>(Operands[0]) ||
4136 // Check for default rounding mode.
4137 // FIXME: Support other rounding modes?
4139 cast<ConstantInt>(Operands[1])->getValue() == 4) {
4140 auto *Op = cast<Constant>(Operands[0]);
4141 switch (IntrinsicID) {
4142 default: break;
4143 case Intrinsic::x86_avx512_vcvtss2si32:
4144 case Intrinsic::x86_avx512_vcvtss2si64:
4145 case Intrinsic::x86_avx512_vcvtsd2si32:
4146 case Intrinsic::x86_avx512_vcvtsd2si64:
4147 if (ConstantFP *FPOp =
4148 dyn_cast_or_null<ConstantFP>(Op->getAggregateElement(0U)))
4149 return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(),
4150 /*roundTowardZero=*/false, Ty,
4151 /*IsSigned*/true);
4152 break;
4153 case Intrinsic::x86_avx512_vcvtss2usi32:
4154 case Intrinsic::x86_avx512_vcvtss2usi64:
4155 case Intrinsic::x86_avx512_vcvtsd2usi32:
4156 case Intrinsic::x86_avx512_vcvtsd2usi64:
4157 if (ConstantFP *FPOp =
4158 dyn_cast_or_null<ConstantFP>(Op->getAggregateElement(0U)))
4159 return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(),
4160 /*roundTowardZero=*/false, Ty,
4161 /*IsSigned*/false);
4162 break;
4163 case Intrinsic::x86_avx512_cvttss2si:
4164 case Intrinsic::x86_avx512_cvttss2si64:
4165 case Intrinsic::x86_avx512_cvttsd2si:
4166 case Intrinsic::x86_avx512_cvttsd2si64:
4167 if (ConstantFP *FPOp =
4168 dyn_cast_or_null<ConstantFP>(Op->getAggregateElement(0U)))
4169 return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(),
4170 /*roundTowardZero=*/true, Ty,
4171 /*IsSigned*/true);
4172 break;
4173 case Intrinsic::x86_avx512_cvttss2usi:
4174 case Intrinsic::x86_avx512_cvttss2usi64:
4175 case Intrinsic::x86_avx512_cvttsd2usi:
4176 case Intrinsic::x86_avx512_cvttsd2usi64:
4177 if (ConstantFP *FPOp =
4178 dyn_cast_or_null<ConstantFP>(Op->getAggregateElement(0U)))
4179 return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(),
4180 /*roundTowardZero=*/true, Ty,
4181 /*IsSigned*/false);
4182 break;
4183 }
4184 }
4185
4186 if (IntrinsicID == Intrinsic::experimental_cttz_elts) {
4187 auto *FVTy = dyn_cast<FixedVectorType>(Operands[0]->getType());
4188 bool ZeroIsPoison = cast<ConstantInt>(Operands[1])->isOne();
4189 if (!FVTy)
4190 return nullptr;
4191 unsigned Width = Ty->getIntegerBitWidth();
4192 if (APInt::getMaxValue(Width).ult(FVTy->getNumElements()) ||
4193 Operands[0]->containsPoisonElement())
4194 return PoisonValue::get(Ty);
4195 for (unsigned I = 0; I < FVTy->getNumElements(); ++I) {
4196 Constant *Elt = Operands[0]->getAggregateElement(I);
4197 if (!Elt)
4198 return nullptr;
4199 if (isa<UndefValue>(Elt) || Elt->isNullValue())
4200 continue;
4201 return ConstantInt::get(Ty, I);
4202 }
4203 if (ZeroIsPoison)
4204 return PoisonValue::get(Ty);
4205 return ConstantInt::get(Ty, FVTy->getNumElements());
4206 }
4207 return nullptr;
4208}
4209
4210static APFloat ConstantFoldAMDGCNCubeIntrinsic(Intrinsic::ID IntrinsicID,
4211 const APFloat &S0,
4212 const APFloat &S1,
4213 const APFloat &S2) {
4214 unsigned ID;
4215 const fltSemantics &Sem = S0.getSemantics();
4216 APFloat MA(Sem), SC(Sem), TC(Sem);
4217 if (abs(S2) >= abs(S0) && abs(S2) >= abs(S1)) {
4218 if (S2.isNegative() && S2.isNonZero() && !S2.isNaN()) {
4219 // S2 < 0
4220 ID = 5;
4221 SC = -S0;
4222 } else {
4223 ID = 4;
4224 SC = S0;
4225 }
4226 MA = S2;
4227 TC = -S1;
4228 } else if (abs(S1) >= abs(S0)) {
4229 if (S1.isNegative() && S1.isNonZero() && !S1.isNaN()) {
4230 // S1 < 0
4231 ID = 3;
4232 TC = -S2;
4233 } else {
4234 ID = 2;
4235 TC = S2;
4236 }
4237 MA = S1;
4238 SC = S0;
4239 } else {
4240 if (S0.isNegative() && S0.isNonZero() && !S0.isNaN()) {
4241 // S0 < 0
4242 ID = 1;
4243 SC = S2;
4244 } else {
4245 ID = 0;
4246 SC = -S2;
4247 }
4248 MA = S0;
4249 TC = -S1;
4250 }
4251 switch (IntrinsicID) {
4252 default:
4253 llvm_unreachable("unhandled amdgcn cube intrinsic");
4254 case Intrinsic::amdgcn_cubeid:
4255 return APFloat(Sem, ID);
4256 case Intrinsic::amdgcn_cubema:
4257 return MA + MA;
4258 case Intrinsic::amdgcn_cubesc:
4259 return SC;
4260 case Intrinsic::amdgcn_cubetc:
4261 return TC;
4262 }
4263}
4264
4265static Constant *ConstantFoldAMDGCNPermIntrinsic(ArrayRef<Constant *> Operands,
4266 Type *Ty) {
4267 const APInt *C0, *C1, *C2;
4268 if (!getConstIntOrUndef(Operands[0], C0) ||
4269 !getConstIntOrUndef(Operands[1], C1) ||
4270 !getConstIntOrUndef(Operands[2], C2))
4271 return nullptr;
4272
4273 if (!C2)
4274 return UndefValue::get(Ty);
4275
4276 APInt Val(32, 0);
4277 unsigned NumUndefBytes = 0;
4278 for (unsigned I = 0; I < 32; I += 8) {
4279 unsigned Sel = C2->extractBitsAsZExtValue(8, I);
4280 unsigned B = 0;
4281
4282 if (Sel >= 13)
4283 B = 0xff;
4284 else if (Sel == 12)
4285 B = 0x00;
4286 else {
4287 const APInt *Src = ((Sel & 10) == 10 || (Sel & 12) == 4) ? C0 : C1;
4288 if (!Src)
4289 ++NumUndefBytes;
4290 else if (Sel < 8)
4291 B = Src->extractBitsAsZExtValue(8, (Sel & 3) * 8);
4292 else
4293 B = Src->extractBitsAsZExtValue(1, (Sel & 1) ? 31 : 15) * 0xff;
4294 }
4295
4296 Val.insertBits(B, I, 8);
4297 }
4298
4299 if (NumUndefBytes == 4)
4300 return UndefValue::get(Ty);
4301
4302 return ConstantInt::get(Ty, Val);
4303}
4304
4305static Constant *ConstantFoldScalarCall3(StringRef Name,
4306 Intrinsic::ID IntrinsicID, Type *Ty,
4308 const TargetLibraryInfo *TLI = nullptr,
4309 const CallBase *Call = nullptr) {
4310 assert(Operands.size() == 3 && "Wrong number of operands.");
4311
4312 if (const auto *Op1 = dyn_cast<ConstantFP>(Operands[0])) {
4313 if (const auto *Op2 = dyn_cast<ConstantFP>(Operands[1])) {
4314 if (const auto *Op3 = dyn_cast<ConstantFP>(Operands[2])) {
4315 const APFloat &C1 = Op1->getValueAPF();
4316 const APFloat &C2 = Op2->getValueAPF();
4317 const APFloat &C3 = Op3->getValueAPF();
4318
4319 if (const auto *ConstrIntr =
4321 RoundingMode RM = getEvaluationRoundingMode(ConstrIntr);
4322 APFloat Res = C1;
4324 switch (IntrinsicID) {
4325 default:
4326 return nullptr;
4327 case Intrinsic::experimental_constrained_fma:
4328 case Intrinsic::experimental_constrained_fmuladd:
4329 St = Res.fusedMultiplyAdd(C2, C3, RM);
4330 break;
4331 }
4332 if (mayFoldConstrained(
4333 const_cast<ConstrainedFPIntrinsic *>(ConstrIntr), St))
4334 return ConstantFP::get(Ty, Res);
4335 return nullptr;
4336 }
4337
4338 switch (IntrinsicID) {
4339 default: break;
4340 case Intrinsic::amdgcn_fma_legacy: {
4341 // The legacy behaviour is that multiplying +/- 0.0 by anything, even
4342 // NaN or infinity, gives +0.0.
4343 if (C1.isZero() || C2.isZero()) {
4344 // It's tempting to just return C3 here, but that would give the
4345 // wrong result if C3 was -0.0.
4346 return ConstantFP::get(Ty, APFloat(0.0f) + C3);
4347 }
4348 [[fallthrough]];
4349 }
4350 case Intrinsic::fma:
4351 case Intrinsic::fmuladd: {
4352 APFloat V = C1;
4353 V.fusedMultiplyAdd(C2, C3, APFloat::rmNearestTiesToEven);
4354 return ConstantFP::get(Ty, V);
4355 }
4356
4357 case Intrinsic::amdgcn_cubeid:
4358 case Intrinsic::amdgcn_cubema:
4359 case Intrinsic::amdgcn_cubesc:
4360 case Intrinsic::amdgcn_cubetc: {
4361 APFloat V = ConstantFoldAMDGCNCubeIntrinsic(IntrinsicID, C1, C2, C3);
4362 return ConstantFP::get(Ty, V);
4363 }
4364 }
4365 }
4366
4367 // TODO: Add constant folding for the _sat variants.
4368 const bool IsFAdd = IntrinsicID == Intrinsic::nvvm_fadd ||
4369 IntrinsicID == Intrinsic::nvvm_fadd_ftz;
4370 const bool IsFMul = IntrinsicID == Intrinsic::nvvm_fmul ||
4371 IntrinsicID == Intrinsic::nvvm_fmul_ftz;
4372 if (IsFAdd || IsFMul) {
4373 bool IsFTZ = IntrinsicID == Intrinsic::nvvm_fadd_ftz ||
4374 IntrinsicID == Intrinsic::nvvm_fmul_ftz;
4375 APFloat A =
4376 IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
4377 APFloat B =
4378 IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
4379
4380 APFloat::roundingMode RoundMode =
4382
4383 APFloat Res = A;
4385 IsFAdd ? Res.add(B, RoundMode) : Res.multiply(B, RoundMode);
4386
4387 if (!Res.isNaN() &&
4389 Res = IsFTZ ? FTZPreserveSign(Res) : Res;
4390 return ConstantFP::get(Ty, Res);
4391 }
4392 return nullptr;
4393 }
4394 }
4395 }
4396
4397 if (IntrinsicID == Intrinsic::smul_fix ||
4398 IntrinsicID == Intrinsic::smul_fix_sat) {
4399 const APInt *C0, *C1;
4400 if (!getConstIntOrUndef(Operands[0], C0) ||
4401 !getConstIntOrUndef(Operands[1], C1))
4402 return nullptr;
4403
4404 // undef * C -> 0
4405 // C * undef -> 0
4406 if (!C0 || !C1)
4407 return Constant::getNullValue(Ty);
4408
4409 // This code performs rounding towards negative infinity in case the result
4410 // cannot be represented exactly for the given scale. Targets that do care
4411 // about rounding should use a target hook for specifying how rounding
4412 // should be done, and provide their own folding to be consistent with
4413 // rounding. This is the same approach as used by
4414 // DAGTypeLegalizer::ExpandIntRes_MULFIX.
4415 unsigned Scale = cast<ConstantInt>(Operands[2])->getZExtValue();
4416 unsigned Width = C0->getBitWidth();
4417 assert(Scale < Width && "Illegal scale.");
4418 unsigned ExtendedWidth = Width * 2;
4419 APInt Product =
4420 (C0->sext(ExtendedWidth) * C1->sext(ExtendedWidth)).ashr(Scale);
4421 if (IntrinsicID == Intrinsic::smul_fix_sat) {
4422 APInt Max = APInt::getSignedMaxValue(Width).sext(ExtendedWidth);
4423 APInt Min = APInt::getSignedMinValue(Width).sext(ExtendedWidth);
4424 Product = APIntOps::smin(Product, Max);
4425 Product = APIntOps::smax(Product, Min);
4426 }
4427 return ConstantInt::get(Ty->getContext(), Product.sextOrTrunc(Width));
4428 }
4429
4430 if (IntrinsicID == Intrinsic::fshl || IntrinsicID == Intrinsic::fshr) {
4431 const APInt *C0, *C1, *C2;
4432 if (!getConstIntOrUndef(Operands[0], C0) ||
4433 !getConstIntOrUndef(Operands[1], C1) ||
4434 !getConstIntOrUndef(Operands[2], C2))
4435 return nullptr;
4436
4437 bool IsRight = IntrinsicID == Intrinsic::fshr;
4438 if (!C2)
4439 return Operands[IsRight ? 1 : 0];
4440 if (!C0 && !C1)
4441 return UndefValue::get(Ty);
4442
4443 // The shift amount is interpreted as modulo the bitwidth. If the shift
4444 // amount is effectively 0, avoid UB due to oversized inverse shift below.
4445 unsigned BitWidth = C2->getBitWidth();
4446 unsigned ShAmt = C2->urem(BitWidth);
4447 if (!ShAmt)
4448 return Operands[IsRight ? 1 : 0];
4449
4450 // (C0 << ShlAmt) | (C1 >> LshrAmt)
4451 unsigned LshrAmt = IsRight ? ShAmt : BitWidth - ShAmt;
4452 unsigned ShlAmt = !IsRight ? ShAmt : BitWidth - ShAmt;
4453 if (!C0)
4454 return ConstantInt::get(Ty, C1->lshr(LshrAmt));
4455 if (!C1)
4456 return ConstantInt::get(Ty, C0->shl(ShlAmt));
4457 return ConstantInt::get(Ty, C0->shl(ShlAmt) | C1->lshr(LshrAmt));
4458 }
4459
4460 if (IntrinsicID == Intrinsic::amdgcn_perm)
4461 return ConstantFoldAMDGCNPermIntrinsic(Operands, Ty);
4462
4463 return nullptr;
4464}
4465
4466static Constant *ConstantFoldScalarCall4(StringRef Name,
4467 Intrinsic::ID IntrinsicID, Type *Ty,
4469 const TargetLibraryInfo *TLI = nullptr,
4470 const CallBase *Call = nullptr) {
4471 assert(Operands.size() == 4 && "Wrong number of operands.");
4472
4473 // TODO: Add constant folding for the _sat and _relu variants.
4474 if (IntrinsicID == Intrinsic::nvvm_fma ||
4475 IntrinsicID == Intrinsic::nvvm_fma_ftz) {
4476 const auto *Op1 = dyn_cast<ConstantFP>(Operands[0]);
4477 const auto *Op2 = dyn_cast<ConstantFP>(Operands[1]);
4478 const auto *Op3 = dyn_cast<ConstantFP>(Operands[2]);
4479 if (!Op1 || !Op2 || !Op3)
4480 return nullptr;
4481
4482 bool IsFTZ = nvvm::FPArithShouldFTZ(IntrinsicID);
4483 APFloat A =
4484 IsFTZ ? FTZPreserveSign(Op1->getValueAPF()) : Op1->getValueAPF();
4485 APFloat B =
4486 IsFTZ ? FTZPreserveSign(Op2->getValueAPF()) : Op2->getValueAPF();
4487 APFloat C =
4488 IsFTZ ? FTZPreserveSign(Op3->getValueAPF()) : Op3->getValueAPF();
4489
4490 APFloat::roundingMode RoundMode =
4492
4493 APFloat Res = A;
4494 APFloat::opStatus Status = Res.fusedMultiplyAdd(B, C, RoundMode);
4495
4496 if (!Res.isNaN() &&
4498 Res = IsFTZ ? FTZPreserveSign(Res) : Res;
4499 return ConstantFP::get(Ty, Res);
4500 }
4501 return nullptr;
4502 }
4503
4504 return nullptr;
4505}
4506
4507static Constant *ConstantFoldScalarCall(StringRef Name,
4508 Intrinsic::ID IntrinsicID, Type *Ty,
4510 const TargetLibraryInfo *TLI = nullptr,
4511 const CallBase *Call = nullptr) {
4512 if (IntrinsicID != Intrinsic::not_intrinsic &&
4514 intrinsicPropagatesPoison(IntrinsicID))
4515 return PoisonValue::get(Ty);
4516
4517 if (Operands.size() == 1)
4518 return ConstantFoldScalarCall1(Name, IntrinsicID, Ty, Operands, TLI, Call);
4519
4520 if (Operands.size() == 2) {
4521 if (Constant *FoldedLibCall =
4522 ConstantFoldLibCall2(Name, Ty, Operands, TLI)) {
4523 return FoldedLibCall;
4524 }
4525 return ConstantFoldIntrinsicCall2(IntrinsicID, Ty, Operands, Call);
4526 }
4527
4528 if (Operands.size() == 3)
4529 return ConstantFoldScalarCall3(Name, IntrinsicID, Ty, Operands, TLI, Call);
4530
4531 if (Operands.size() == 4)
4532 return ConstantFoldScalarCall4(Name, IntrinsicID, Ty, Operands, TLI, Call);
4533
4534 return nullptr;
4535}
4536
4537static Constant *ConstantFoldFixedVectorCall(
4538 StringRef Name, Intrinsic::ID IntrinsicID, FixedVectorType *FVTy,
4540 const TargetLibraryInfo *TLI = nullptr, const CallBase *Call = nullptr) {
4543 Type *Ty = FVTy->getElementType();
4544
4545 switch (IntrinsicID) {
4546 case Intrinsic::masked_load: {
4547 auto *SrcPtr = Operands[0];
4548 auto *Mask = Operands[1];
4549 auto *Passthru = Operands[2];
4550
4551 Constant *VecData = ConstantFoldLoadFromConstPtr(SrcPtr, FVTy, DL);
4552
4553 SmallVector<Constant *, 32> NewElements;
4554 for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
4555 auto *MaskElt = Mask->getAggregateElement(I);
4556 if (!MaskElt)
4557 break;
4558 auto *PassthruElt = Passthru->getAggregateElement(I);
4559 auto *VecElt = VecData ? VecData->getAggregateElement(I) : nullptr;
4560 if (isa<UndefValue>(MaskElt)) {
4561 if (PassthruElt)
4562 NewElements.push_back(PassthruElt);
4563 else if (VecElt)
4564 NewElements.push_back(VecElt);
4565 else
4566 return nullptr;
4567 }
4568 if (MaskElt->isNullValue()) {
4569 if (!PassthruElt)
4570 return nullptr;
4571 NewElements.push_back(PassthruElt);
4572 } else if (MaskElt->isOneValue()) {
4573 if (!VecElt)
4574 return nullptr;
4575 NewElements.push_back(VecElt);
4576 } else {
4577 return nullptr;
4578 }
4579 }
4580 if (NewElements.size() != FVTy->getNumElements())
4581 return nullptr;
4582 return ConstantVector::get(NewElements);
4583 }
4584 case Intrinsic::arm_mve_vctp8:
4585 case Intrinsic::arm_mve_vctp16:
4586 case Intrinsic::arm_mve_vctp32:
4587 case Intrinsic::arm_mve_vctp64: {
4588 if (auto *Op = dyn_cast<ConstantInt>(Operands[0])) {
4589 unsigned Lanes = FVTy->getNumElements();
4590 uint64_t Limit = Op->getZExtValue();
4591
4593 for (unsigned i = 0; i < Lanes; i++) {
4594 if (i < Limit)
4596 else
4598 }
4599 return ConstantVector::get(NCs);
4600 }
4601 return nullptr;
4602 }
4603 case Intrinsic::get_active_lane_mask: {
4604 auto *Op0 = dyn_cast<ConstantInt>(Operands[0]);
4605 auto *Op1 = dyn_cast<ConstantInt>(Operands[1]);
4606 if (Op0 && Op1) {
4607 unsigned Lanes = FVTy->getNumElements();
4608 APInt Base = Op0->getValue();
4609 APInt Limit = Op1->getValue();
4610
4612 for (unsigned I = 0; I < Lanes; I++) {
4613 bool Overflow;
4614 if (Base.uadd_ov(APInt(Base.getBitWidth(), I), Overflow).ult(Limit) &&
4615 !Overflow)
4617 else
4619 }
4620 return ConstantVector::get(NCs);
4621 }
4622 return nullptr;
4623 }
4624 case Intrinsic::mask_beforefirst: {
4627 Constant *Current = True;
4628 for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
4629 Constant *Elt = Operands[0]->getAggregateElement(I);
4631 return PoisonValue::get(FVTy);
4632 auto *CI = dyn_cast_or_null<ConstantInt>(Elt);
4633 if (!CI)
4634 return nullptr;
4635 if (CI->isOne())
4636 Current = False;
4637 Result[I] = Current;
4638 }
4639 return ConstantVector::get(Result);
4640 }
4641 case Intrinsic::vector_extract: {
4643 Constant *Vec = Operands[0];
4644 if (!Idx || !isa<FixedVectorType>(Vec->getType()))
4645 return nullptr;
4646
4647 unsigned NumElements = FVTy->getNumElements();
4648 unsigned VecNumElements =
4649 cast<FixedVectorType>(Vec->getType())->getNumElements();
4650 unsigned StartingIndex = Idx->getZExtValue();
4651
4652 // Extracting entire vector is nop
4653 if (NumElements == VecNumElements && StartingIndex == 0)
4654 return Vec;
4655
4656 for (unsigned I = StartingIndex, E = StartingIndex + NumElements; I < E;
4657 ++I) {
4658 Constant *Elt = Vec->getAggregateElement(I);
4659 if (!Elt)
4660 return nullptr;
4661 Result[I - StartingIndex] = Elt;
4662 }
4663
4664 return ConstantVector::get(Result);
4665 }
4666 case Intrinsic::vector_insert: {
4667 Constant *Vec = Operands[0];
4668 Constant *SubVec = Operands[1];
4670 if (!Idx || !isa<FixedVectorType>(Vec->getType()))
4671 return nullptr;
4672
4673 unsigned SubVecNumElements =
4674 cast<FixedVectorType>(SubVec->getType())->getNumElements();
4675 unsigned VecNumElements =
4676 cast<FixedVectorType>(Vec->getType())->getNumElements();
4677 unsigned IdxN = Idx->getZExtValue();
4678 // Replacing entire vector with a subvec is nop
4679 if (SubVecNumElements == VecNumElements && IdxN == 0)
4680 return SubVec;
4681
4682 for (unsigned I = 0; I < VecNumElements; ++I) {
4683 Constant *Elt;
4684 if (I < IdxN + SubVecNumElements)
4685 Elt = SubVec->getAggregateElement(I - IdxN);
4686 else
4687 Elt = Vec->getAggregateElement(I);
4688 if (!Elt)
4689 return nullptr;
4690 Result[I] = Elt;
4691 }
4692 return ConstantVector::get(Result);
4693 }
4694 case Intrinsic::vector_interleave2:
4695 case Intrinsic::vector_interleave3:
4696 case Intrinsic::vector_interleave4:
4697 case Intrinsic::vector_interleave5:
4698 case Intrinsic::vector_interleave6:
4699 case Intrinsic::vector_interleave7:
4700 case Intrinsic::vector_interleave8: {
4701 unsigned NumElements =
4702 cast<FixedVectorType>(Operands[0]->getType())->getNumElements();
4703 unsigned NumOperands = Operands.size();
4704 for (unsigned I = 0; I < NumElements; ++I) {
4705 for (unsigned J = 0; J < NumOperands; ++J) {
4706 Constant *Elt = Operands[J]->getAggregateElement(I);
4707 if (!Elt)
4708 return nullptr;
4709 Result[NumOperands * I + J] = Elt;
4710 }
4711 }
4712 return ConstantVector::get(Result);
4713 }
4714 case Intrinsic::vector_partial_reduce_add:
4715 return constantFoldVectorPartialReduceAdd(Operands[0], Operands[1], DL);
4716 case Intrinsic::wasm_dot: {
4717 unsigned NumElements =
4718 cast<FixedVectorType>(Operands[0]->getType())->getNumElements();
4719
4720 assert(NumElements == 8 && Result.size() == 4 &&
4721 "wasm dot takes i16x8 and produces i32x4");
4722 assert(Ty->isIntegerTy());
4723 int32_t MulVector[8];
4724
4725 for (unsigned I = 0; I < NumElements; ++I) {
4726 ConstantInt *Elt0 =
4727 dyn_cast<ConstantInt>(Operands[0]->getAggregateElement(I));
4728 ConstantInt *Elt1 =
4729 dyn_cast<ConstantInt>(Operands[1]->getAggregateElement(I));
4730
4731 if (!Elt0 || !Elt1)
4732 return nullptr;
4733
4734 MulVector[I] = Elt0->getSExtValue() * Elt1->getSExtValue();
4735 }
4736 for (unsigned I = 0; I < Result.size(); I++) {
4737 int64_t IAdd = (int64_t)MulVector[I * 2] + (int64_t)MulVector[I * 2 + 1];
4738 Result[I] = ConstantInt::getSigned(Ty, IAdd, /*ImplicitTrunc=*/true);
4739 }
4740
4741 return ConstantVector::get(Result);
4742 }
4743 case Intrinsic::nvvm_fadd:
4744 case Intrinsic::nvvm_fadd_ftz:
4745 case Intrinsic::nvvm_fma:
4746 case Intrinsic::nvvm_fma_ftz:
4747 case Intrinsic::nvvm_fmul:
4748 case Intrinsic::nvvm_fmul_ftz:
4749 // The rounding mode operand is a scalar, so the lane-wise folding below
4750 // does not apply.
4751 // TODO: Fold these by passing the rounding mode through to every lane.
4752 return nullptr;
4753 default:
4754 break;
4755 }
4756
4757 for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
4758 // Gather a column of constants.
4759 for (unsigned J = 0, JE = Operands.size(); J != JE; ++J) {
4760 // Some intrinsics use a scalar type for certain arguments.
4761 if (isVectorIntrinsicWithScalarOpAtArg(IntrinsicID, J, /*TTI=*/nullptr)) {
4762 Lane[J] = Operands[J];
4763 continue;
4764 }
4765
4766 Constant *Agg = Operands[J]->getAggregateElement(I);
4767 if (!Agg)
4768 return nullptr;
4769
4770 Lane[J] = Agg;
4771 }
4772
4773 // Use the regular scalar folding to simplify this column.
4774 Constant *Folded =
4775 ConstantFoldScalarCall(Name, IntrinsicID, Ty, Lane, TLI, Call);
4776 if (!Folded)
4777 return nullptr;
4778 Result[I] = Folded;
4779 }
4780
4781 return ConstantVector::get(Result);
4782}
4783
4784static Constant *ConstantFoldScalableVectorCall(
4785 StringRef Name, Intrinsic::ID IntrinsicID, ScalableVectorType *SVTy,
4787 const TargetLibraryInfo *TLI, const CallBase *Call) {
4788 switch (IntrinsicID) {
4789 case Intrinsic::aarch64_sve_convert_from_svbool: {
4790 Constant *Src = Operands[0];
4791 if (!Src->isNullValue())
4792 break;
4793
4794 return ConstantInt::getFalse(SVTy);
4795 }
4796 case Intrinsic::get_active_lane_mask: {
4797 auto *Op0 = dyn_cast<ConstantInt>(Operands[0]);
4798 auto *Op1 = dyn_cast<ConstantInt>(Operands[1]);
4799 if (Op0 && Op1 && Op0->getValue().uge(Op1->getValue()))
4800 return ConstantVector::getNullValue(SVTy);
4801 break;
4802 }
4803 case Intrinsic::mask_beforefirst: {
4804 Constant *SplatVal = Operands[0]->getSplatValue();
4805 if (isa_and_nonnull<PoisonValue>(SplatVal))
4806 return PoisonValue::get(SVTy);
4807 if (auto *CI = dyn_cast_if_present<ConstantInt>(SplatVal))
4808 return ConstantInt::getBool(SVTy, CI->isZero());
4809 break;
4810 }
4811 case Intrinsic::vector_interleave2:
4812 case Intrinsic::vector_interleave3:
4813 case Intrinsic::vector_interleave4:
4814 case Intrinsic::vector_interleave5:
4815 case Intrinsic::vector_interleave6:
4816 case Intrinsic::vector_interleave7:
4817 case Intrinsic::vector_interleave8: {
4818 Constant *SplatVal = Operands[0]->getSplatValue();
4819 if (!SplatVal)
4820 return nullptr;
4821
4823 return nullptr;
4824
4825 return ConstantVector::getSplat(SVTy->getElementCount(), SplatVal);
4826 }
4827 default:
4828 break;
4829 }
4830
4831 // If trivially vectorizable, try folding it via the scalar call if all
4832 // operands are splats.
4833
4834 // TODO: ConstantFoldFixedVectorCall should probably check this too?
4835 if (!isTriviallyVectorizable(IntrinsicID))
4836 return nullptr;
4837
4839 for (auto [I, Op] : enumerate(Operands)) {
4840 if (isVectorIntrinsicWithScalarOpAtArg(IntrinsicID, I, /*TTI=*/nullptr)) {
4841 SplatOps.push_back(Op);
4842 continue;
4843 }
4844 Constant *Splat = Op->getSplatValue();
4845 if (!Splat)
4846 return nullptr;
4847 SplatOps.push_back(Splat);
4848 }
4849 Constant *Folded = ConstantFoldScalarCall(
4850 Name, IntrinsicID, SVTy->getElementType(), SplatOps, TLI, Call);
4851 if (!Folded)
4852 return nullptr;
4854}
4855
4856static std::pair<Constant *, Constant *>
4857ConstantFoldScalarFrexpCall(Constant *Op, Type *IntTy) {
4858 auto *ConstFP = dyn_cast<ConstantFP>(Op);
4859 if (!ConstFP)
4860 return {};
4861
4862 const APFloat &U = ConstFP->getValueAPF();
4863 int FrexpExp;
4864 APFloat FrexpMant = frexp(U, FrexpExp, APFloat::rmNearestTiesToEven);
4865 Constant *Result0 = ConstantFP::get(ConstFP->getType(), FrexpMant);
4866
4867 // The exponent is an "unspecified value" for inf/nan. We use zero to avoid
4868 // using undef.
4869 Constant *Result1 = FrexpMant.isFinite()
4870 ? ConstantInt::getSigned(IntTy, FrexpExp)
4871 : ConstantInt::getNullValue(IntTy);
4872 return {Result0, Result1};
4873}
4874
4875/// Handle intrinsics that return tuples, which may be tuples of vectors.
4876static Constant *
4877ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
4879 const DataLayout &DL, const TargetLibraryInfo *TLI,
4880 const CallBase *Call) {
4881
4882 switch (IntrinsicID) {
4883 case Intrinsic::frexp: {
4884 Type *Ty0 = StTy->getContainedType(0);
4885 Type *Ty1 = StTy->getContainedType(1)->getScalarType();
4886
4887 if (auto *FVTy0 = dyn_cast<FixedVectorType>(Ty0)) {
4888 SmallVector<Constant *, 4> Results0(FVTy0->getNumElements());
4889 SmallVector<Constant *, 4> Results1(FVTy0->getNumElements());
4890
4891 for (unsigned I = 0, E = FVTy0->getNumElements(); I != E; ++I) {
4892 Constant *Lane = Operands[0]->getAggregateElement(I);
4893 std::tie(Results0[I], Results1[I]) =
4894 ConstantFoldScalarFrexpCall(Lane, Ty1);
4895 if (!Results0[I])
4896 return nullptr;
4897 }
4898
4899 return ConstantStruct::get(StTy, ConstantVector::get(Results0),
4900 ConstantVector::get(Results1));
4901 }
4902
4903 auto [Result0, Result1] = ConstantFoldScalarFrexpCall(Operands[0], Ty1);
4904 if (!Result0)
4905 return nullptr;
4906 return ConstantStruct::get(StTy, Result0, Result1);
4907 }
4908 case Intrinsic::sincos: {
4909 Type *Ty = StTy->getContainedType(0);
4910 Type *TyScalar = Ty->getScalarType();
4911
4912 auto ConstantFoldScalarSincosCall =
4913 [&](Constant *Op) -> std::pair<Constant *, Constant *> {
4914 Constant *SinResult =
4915 ConstantFoldScalarCall(Name, Intrinsic::sin, TyScalar, Op, TLI, Call);
4916 Constant *CosResult =
4917 ConstantFoldScalarCall(Name, Intrinsic::cos, TyScalar, Op, TLI, Call);
4918 return std::make_pair(SinResult, CosResult);
4919 };
4920
4921 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty)) {
4922 SmallVector<Constant *> SinResults(FVTy->getNumElements());
4923 SmallVector<Constant *> CosResults(FVTy->getNumElements());
4924
4925 for (unsigned I = 0, E = FVTy->getNumElements(); I != E; ++I) {
4926 Constant *Lane = Operands[0]->getAggregateElement(I);
4927 std::tie(SinResults[I], CosResults[I]) =
4928 ConstantFoldScalarSincosCall(Lane);
4929 if (!SinResults[I] || !CosResults[I])
4930 return nullptr;
4931 }
4932
4933 return ConstantStruct::get(StTy, ConstantVector::get(SinResults),
4934 ConstantVector::get(CosResults));
4935 }
4936
4937 if (!Ty->isFloatingPointTy())
4938 return nullptr;
4939
4940 auto [SinResult, CosResult] = ConstantFoldScalarSincosCall(Operands[0]);
4941 if (!SinResult || !CosResult)
4942 return nullptr;
4943 return ConstantStruct::get(StTy, SinResult, CosResult);
4944 }
4945 case Intrinsic::vector_deinterleave2:
4946 case Intrinsic::vector_deinterleave3:
4947 case Intrinsic::vector_deinterleave4:
4948 case Intrinsic::vector_deinterleave5:
4949 case Intrinsic::vector_deinterleave6:
4950 case Intrinsic::vector_deinterleave7:
4951 case Intrinsic::vector_deinterleave8: {
4952 unsigned NumResults = StTy->getNumElements();
4953 auto *Vec = Operands[0];
4954 auto *VecTy = cast<VectorType>(Vec->getType());
4955
4956 ElementCount ResultEC =
4957 VecTy->getElementCount().divideCoefficientBy(NumResults);
4958
4959 if (auto *EltC = Vec->getSplatValue()) {
4960 auto *ResultVec = ConstantVector::getSplat(ResultEC, EltC);
4961 SmallVector<Constant *, 8> Results(NumResults, ResultVec);
4962 return ConstantStruct::get(StTy, Results);
4963 }
4964
4965 if (!ResultEC.isFixed())
4966 return nullptr;
4967
4968 unsigned NumElements = ResultEC.getFixedValue();
4970 SmallVector<Constant *> Elements(NumElements);
4971 for (unsigned I = 0; I != NumResults; ++I) {
4972 for (unsigned J = 0; J != NumElements; ++J) {
4973 Constant *Elt = Vec->getAggregateElement(J * NumResults + I);
4974 if (!Elt)
4975 return nullptr;
4976 Elements[J] = Elt;
4977 }
4978 Results[I] = ConstantVector::get(Elements);
4979 }
4980 return ConstantStruct::get(StTy, Results);
4981 }
4982 default:
4983 // TODO: Constant folding of vector intrinsics that fall through here does
4984 // not work (e.g. overflow intrinsics)
4985 return ConstantFoldScalarCall(Name, IntrinsicID, StTy, Operands, TLI, Call);
4986 }
4987
4988 return nullptr;
4989}
4990
4991} // end anonymous namespace
4992
4993Constant *llvm::ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
4994 Type *Ty, DenormalMode DenormMode) {
4995 assert(DenormMode.isValid() && "invalid denormal mode");
4998 return nullptr;
4999
5000 llvm_fenv_clearexcept();
5001 APFloat Input = FlushWithDenormKind(V, DenormMode.Input);
5002 double Result = NativeFP(Input.convertToDouble());
5003 if (llvm_fenv_testexcept()) {
5004 llvm_fenv_clearexcept();
5005 return nullptr;
5006 }
5007
5008 Constant *Output = GetConstantFoldFPValue(Result, Ty);
5010 return Output;
5011 auto *CFP = cast<ConstantFP>(Output);
5012 APFloat Res = FlushWithDenormKind(CFP->getValueAPF(), DenormMode.Output);
5013 return ConstantFP::get(Ty->getContext(), Res);
5014}
5015
5018 const DataLayout &DL,
5019 const Function *CtxF) {
5020 // In the absence of CtxF, assume strictfp conservatively.
5021 if (!canConstantFoldIntrinsic(ID, CtxF ? CtxF->isStrictFP() : true) ||
5024 Ty, ArrayRef<Value *>((Value *const *)Ops.data(), Ops.size()))))
5025 return nullptr;
5026 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty))
5027 return ConstantFoldFixedVectorCall("", ID, FVTy, Ops, DL);
5028 return ConstantFoldScalarCall("", ID, Ty, Ops);
5029}
5030
5033 const TargetLibraryInfo *TLI,
5034 bool AllowNonDeterministic) {
5035 if (Call->isNoBuiltin())
5036 return nullptr;
5037 if (!F->hasName())
5038 return nullptr;
5039
5040 // If this is not an intrinsic and not recognized as a library call, bail out.
5041 Intrinsic::ID IID = F->getIntrinsicID();
5042 if (IID == Intrinsic::not_intrinsic) {
5043 if (!TLI)
5044 return nullptr;
5045 if (TLI->getLibFunc(*F) == NotLibFunc)
5046 return nullptr;
5047 }
5048
5049 // Conservatively assume that floating-point libcalls may be
5050 // non-deterministic.
5051 Type *Ty = F->getReturnType();
5052 if (!AllowNonDeterministic && Ty->isFPOrFPVectorTy())
5053 return nullptr;
5054
5055 StringRef Name = F->getName();
5056 if (auto *FVTy = dyn_cast<FixedVectorType>(Ty))
5057 return ConstantFoldFixedVectorCall(
5058 Name, IID, FVTy, Operands, F->getDataLayout(), TLI, Call);
5059
5060 if (auto *SVTy = dyn_cast<ScalableVectorType>(Ty))
5061 return ConstantFoldScalableVectorCall(
5062 Name, IID, SVTy, Operands, F->getDataLayout(), TLI, Call);
5063
5064 if (auto *StTy = dyn_cast<StructType>(Ty))
5065 return ConstantFoldStructCall(Name, IID, StTy, Operands,
5066 F->getDataLayout(), TLI, Call);
5067
5068 // TODO: If this is a library function, we already discovered that above,
5069 // so we should pass the LibFunc, not the name (and it might be better
5070 // still to separate intrinsic handling from libcalls).
5071 return ConstantFoldScalarCall(Name, IID, Ty, Operands, TLI, Call);
5072}
5073
5075 const TargetLibraryInfo *TLI) {
5076 // FIXME: Refactor this code; this duplicates logic in LibCallsShrinkWrap
5077 // (and to some extent ConstantFoldScalarCall).
5078 if (Call->isNoBuiltin() || Call->isStrictFP())
5079 return false;
5080 Function *F = Call->getCalledFunction();
5081 if (!F)
5082 return false;
5083
5084 if (!TLI)
5085 return false;
5086
5087 LibFunc Func = TLI->getLibFunc(*F);
5088 if (Func == NotLibFunc)
5089 return false;
5090
5091 if (Call->arg_size() == 1) {
5092 if (ConstantFP *OpC = dyn_cast<ConstantFP>(Call->getArgOperand(0))) {
5093 const APFloat &Op = OpC->getValueAPF();
5094 switch (Func) {
5095 case LibFunc_logl:
5096 case LibFunc_log:
5097 case LibFunc_logf:
5098 case LibFunc_log2l:
5099 case LibFunc_log2:
5100 case LibFunc_log2f:
5101 case LibFunc_log10l:
5102 case LibFunc_log10:
5103 case LibFunc_log10f:
5104 return Op.isNaN() || (!Op.isZero() && !Op.isNegative());
5105
5106 case LibFunc_ilogb:
5107 return !Op.isNaN() && !Op.isZero() && !Op.isInfinity();
5108
5109 case LibFunc_expl:
5110 case LibFunc_exp:
5111 case LibFunc_expf:
5112 // FIXME: These boundaries are slightly conservative.
5113 if (OpC->getType()->isDoubleTy())
5114 return !(Op < APFloat(-745.0) || Op > APFloat(709.0));
5115 if (OpC->getType()->isFloatTy())
5116 return !(Op < APFloat(-103.0f) || Op > APFloat(88.0f));
5117 break;
5118
5119 case LibFunc_exp2l:
5120 case LibFunc_exp2:
5121 case LibFunc_exp2f:
5122 // FIXME: These boundaries are slightly conservative.
5123 if (OpC->getType()->isDoubleTy())
5124 return !(Op < APFloat(-1074.0) || Op > APFloat(1023.0));
5125 if (OpC->getType()->isFloatTy())
5126 return !(Op < APFloat(-149.0f) || Op > APFloat(127.0f));
5127 break;
5128
5129 case LibFunc_sinl:
5130 case LibFunc_sin:
5131 case LibFunc_sinf:
5132 case LibFunc_cosl:
5133 case LibFunc_cos:
5134 case LibFunc_cosf:
5135 return !Op.isInfinity();
5136
5137 case LibFunc_tanl:
5138 case LibFunc_tan:
5139 case LibFunc_tanf: {
5140 // FIXME: Stop using the host math library.
5141 // FIXME: The computation isn't done in the right precision.
5142 Type *Ty = OpC->getType();
5143 if (Ty->isDoubleTy() || Ty->isFloatTy() || Ty->isHalfTy())
5144 return ConstantFoldFP(tan, OpC->getValueAPF(), Ty) != nullptr;
5145 break;
5146 }
5147
5148 case LibFunc_atan:
5149 case LibFunc_atanf:
5150 case LibFunc_atanl:
5151 // Per POSIX, this MAY fail if Op is denormal. We choose not failing.
5152 return true;
5153
5154 case LibFunc_asinl:
5155 case LibFunc_asin:
5156 case LibFunc_asinf:
5157 case LibFunc_acosl:
5158 case LibFunc_acos:
5159 case LibFunc_acosf:
5160 return !(Op < APFloat::getOne(Op.getSemantics(), true) ||
5161 Op > APFloat::getOne(Op.getSemantics()));
5162
5163 case LibFunc_sinh:
5164 case LibFunc_cosh:
5165 case LibFunc_sinhf:
5166 case LibFunc_coshf:
5167 case LibFunc_sinhl:
5168 case LibFunc_coshl:
5169 // FIXME: These boundaries are slightly conservative.
5170 if (OpC->getType()->isDoubleTy())
5171 return !(Op < APFloat(-710.0) || Op > APFloat(710.0));
5172 if (OpC->getType()->isFloatTy())
5173 return !(Op < APFloat(-89.0f) || Op > APFloat(89.0f));
5174 break;
5175
5176 case LibFunc_sqrtl:
5177 case LibFunc_sqrt:
5178 case LibFunc_sqrtf:
5179 return Op.isNaN() || Op.isZero() || !Op.isNegative();
5180
5181 // FIXME: Add more functions: sqrt_finite, atanh, expm1, log1p,
5182 // maybe others?
5183 default:
5184 break;
5185 }
5186 }
5187 }
5188
5189 if (Call->arg_size() == 2) {
5190 ConstantFP *Op0C = dyn_cast<ConstantFP>(Call->getArgOperand(0));
5191 ConstantFP *Op1C = dyn_cast<ConstantFP>(Call->getArgOperand(1));
5192 if (Op0C && Op1C) {
5193 const APFloat &Op0 = Op0C->getValueAPF();
5194 const APFloat &Op1 = Op1C->getValueAPF();
5195
5196 switch (Func) {
5197 case LibFunc_powl:
5198 case LibFunc_pow:
5199 case LibFunc_powf: {
5200 // FIXME: Stop using the host math library.
5201 // FIXME: The computation isn't done in the right precision.
5202 Type *Ty = Op0C->getType();
5203 if (Ty->isDoubleTy() || Ty->isFloatTy() || Ty->isHalfTy()) {
5204 if (Ty == Op1C->getType())
5205 return ConstantFoldBinaryFP(pow, Op0, Op1, Ty) != nullptr;
5206 }
5207 break;
5208 }
5209
5210 case LibFunc_fmodl:
5211 case LibFunc_fmod:
5212 case LibFunc_fmodf:
5213 case LibFunc_remainderl:
5214 case LibFunc_remainder:
5215 case LibFunc_remainderf:
5216 return Op0.isNaN() || Op1.isNaN() ||
5217 (!Op0.isInfinity() && !Op1.isZero());
5218
5219 case LibFunc_atan2:
5220 case LibFunc_atan2f:
5221 case LibFunc_atan2l:
5222 // Although IEEE-754 says atan2(+/-0.0, +/-0.0) are well-defined, and
5223 // GLIBC and MSVC do not appear to raise an error on those, we
5224 // cannot rely on that behavior. POSIX and C11 say that a domain error
5225 // may occur, so allow for that possibility.
5226 return !Op0.isZero() || !Op1.isZero();
5227
5228 case LibFunc_nextafter:
5229 case LibFunc_nextafterf:
5230 case LibFunc_nextafterl:
5231 case LibFunc_nexttoward:
5232 case LibFunc_nexttowardf:
5233 case LibFunc_nexttowardl: {
5234 return ConstantFoldNextToward(Op0, Op1, F->getReturnType()) != nullptr;
5235 }
5236 default:
5237 break;
5238 }
5239 }
5240 }
5241
5242 return false;
5243}
5244
5246 unsigned CastOp, const DataLayout &DL,
5247 PreservedCastFlags *Flags) {
5248 switch (CastOp) {
5249 case Instruction::BitCast:
5250 // Bitcast is always lossless.
5251 return ConstantFoldCastOperand(Instruction::BitCast, C, InvCastTo, DL);
5252 case Instruction::Trunc: {
5253 auto *ZExtC = ConstantFoldCastOperand(Instruction::ZExt, C, InvCastTo, DL);
5254 if (Flags) {
5255 // Truncation back on ZExt value is always NUW.
5256 Flags->NUW = true;
5257 // Test positivity of C.
5258 auto *SExtC =
5259 ConstantFoldCastOperand(Instruction::SExt, C, InvCastTo, DL);
5260 Flags->NSW = ZExtC == SExtC;
5261 }
5262 return ZExtC;
5263 }
5264 case Instruction::SExt:
5265 case Instruction::ZExt: {
5266 auto *InvC = ConstantExpr::getTrunc(C, InvCastTo);
5267 auto *CastInvC = ConstantFoldCastOperand(CastOp, InvC, C->getType(), DL);
5268 // Must satisfy CastOp(InvC) == C.
5269 if (!CastInvC || CastInvC != C)
5270 return nullptr;
5271 if (Flags && CastOp == Instruction::ZExt) {
5272 auto *SExtInvC =
5273 ConstantFoldCastOperand(Instruction::SExt, InvC, C->getType(), DL);
5274 // Test positivity of InvC.
5275 Flags->NNeg = CastInvC == SExtInvC;
5276 }
5277 return InvC;
5278 }
5279 case Instruction::FPExt: {
5280 Constant *InvC =
5281 ConstantFoldCastOperand(Instruction::FPTrunc, C, InvCastTo, DL);
5282 if (InvC) {
5283 Constant *CastInvC =
5284 ConstantFoldCastOperand(CastOp, InvC, C->getType(), DL);
5285 if (CastInvC == C)
5286 return InvC;
5287 }
5288 return nullptr;
5289 }
5290 default:
5291 return nullptr;
5292 }
5293}
5294
5296 const DataLayout &DL,
5297 PreservedCastFlags *Flags) {
5298 return getLosslessInvCast(C, DestTy, Instruction::ZExt, DL, Flags);
5299}
5300
5302 const DataLayout &DL,
5303 PreservedCastFlags *Flags) {
5304 return getLosslessInvCast(C, DestTy, Instruction::SExt, DL, Flags);
5305}
5306
5307void TargetFolder::anchor() {}
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
constexpr LLT S1
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static Constant * FoldBitCast(Constant *V, Type *DestTy)
static ConstantFP * flushDenormalConstant(Type *Ty, const APFloat &APF, DenormalMode::DenormalModeKind Mode)
Constant * getConstantAtOffset(Constant *Base, APInt Offset, const DataLayout &DL)
If this Offset points exactly to the start of an aggregate element, return that element,...
static cl::opt< bool > DisableFPCallFolding("disable-fp-call-folding", cl::desc("Disable constant-folding of FP intrinsics and libcalls."), cl::init(false), cl::Hidden)
static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP)
Returns true if the intrinsic can be constant folded, given IsStrictFP.
static bool containsByteType(Type *Ty)
Return true if Ty is or contains a byte type.
static ConstantFP * flushDenormalConstantFP(ConstantFP *CFP, const Function *CtxF, bool IsOutput)
static bool anyTypeContainsFP(Type *RetTy, ArrayRef< Value * > Ops)
Given a function's return type and its operands, determine if any of them of of floating-point type.
static DenormalMode getInstrDenormalMode(const Function *CtxF, Type *Ty)
Return the denormal mode that can be assumed when executing a floating point operation at CtxI.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseMap class.
Hexagon Common GEP
amode Optimize addressing mode
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
static bool InRange(int64_t Value, unsigned short Shift, int LBound, int HBound)
This file contains the definitions of the enumerations and flags associated with NVVM Intrinsics,...
if(PassOpts->AAPipeline)
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
int64_t C1
int64_t C0
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
SI Fold Operands
This file contains some templates that are useful if you are working with the STL at all.
This file implements the SmallBitVector class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
The Input class is used to parse a yaml document into in-memory structs and vectors.
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
Definition APFloat.h:351
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
llvm::RoundingMode roundingMode
IEEE-754R 4.3: Rounding-direction attributes.
Definition APFloat.h:359
static const fltSemantics & IEEEdouble()
Definition APFloat.h:305
static constexpr roundingMode rmTowardNegative
Definition APFloat.h:364
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:361
static constexpr roundingMode rmTowardPositive
Definition APFloat.h:363
static constexpr roundingMode rmNearestTiesToAway
Definition APFloat.h:366
opStatus
IEEE-754R 7: Default exception handling.
Definition APFloat.h:377
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition APFloat.h:1224
opStatus divide(const APFloat &RHS, roundingMode RM)
Definition APFloat.h:1312
void copySign(const APFloat &RHS)
Definition APFloat.h:1406
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition APFloat.cpp:6045
opStatus subtract(const APFloat &RHS, roundingMode RM)
Definition APFloat.h:1294
bool isNegative() const
Definition APFloat.h:1583
LLVM_ABI double convertToDouble() const
Converts this APFloat to host double value.
Definition APFloat.cpp:6278
bool isPosInfinity() const
Definition APFloat.h:1596
bool isNormal() const
Definition APFloat.h:1587
bool isDenormal() const
Definition APFloat.h:1584
opStatus add(const APFloat &RHS, roundingMode RM)
Definition APFloat.h:1285
const fltSemantics & getSemantics() const
Definition APFloat.h:1591
bool isNonZero() const
Definition APFloat.h:1592
bool isFinite() const
Definition APFloat.h:1588
bool isNaN() const
Definition APFloat.h:1581
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
Definition APFloat.h:1192
opStatus multiply(const APFloat &RHS, roundingMode RM)
Definition APFloat.h:1303
bool isSignaling() const
Definition APFloat.h:1585
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
Definition APFloat.h:1339
bool isZero() const
Definition APFloat.h:1579
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition APFloat.h:1436
opStatus mod(const APFloat &RHS)
Definition APFloat.h:1330
bool isNegInfinity() const
Definition APFloat.h:1597
void changeSign()
Definition APFloat.h:1401
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition APFloat.h:1183
bool isInfinity() const
Definition APFloat.h:1580
Class for arbitrary precision integers.
Definition APInt.h:78
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
LLVM_ABI uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
Definition APInt.cpp:517
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition APInt.cpp:1078
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition APInt.h:202
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition APInt.cpp:1695
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition APInt.h:205
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
LLVM_ABI APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition APInt.cpp:1086
LLVM_ABI void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition APInt.cpp:393
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition APInt.h:829
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
Definition APInt.cpp:1030
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:196
LLVM_ABI APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition APInt.cpp:478
An arbitrary precision integer that knows its signedness.
Definition APSInt.h:24
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
static LLVM_ABI Instruction::CastOps getCastOpcode(const Value *Val, bool SrcIsSigned, Type *Ty, bool DstIsSigned)
Returns the opcode necessary to cast Val into Ty using usual casting rules.
static LLVM_ABI unsigned isEliminableCastPair(Instruction::CastOps firstOpcode, Instruction::CastOps secondOpcode, Type *SrcTy, Type *MidTy, Type *DstTy, const DataLayout *DL)
Determine how a pair of casts can be eliminated, if they can be at all.
static LLVM_ABI bool castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy)
This method can be used to determine if a cast from SrcTy to DstTy using Opcode op is valid or not.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
bool isSigned() const
Definition InstrTypes.h:993
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:890
static bool isFPPredicate(Predicate P)
Definition InstrTypes.h:833
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
Definition Constants.h:878
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getExtractElement(Constant *Vec, Constant *Idx, Type *OnlyIfReducedTy=nullptr)
static LLVM_ABI bool isDesirableCastOp(unsigned Opcode)
Whether creating a constant expression for this cast is desirable.
static LLVM_ABI Constant * getCast(unsigned ops, Constant *C, Type *Ty, bool OnlyIfReduced=false)
Convenience function for getting a Cast operation.
static LLVM_ABI Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getPtrAdd(Constant *Ptr, Constant *Offset, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReduced=nullptr)
Create a getelementptr i8, ptr, offset constant expression.
Definition Constants.h:1518
static LLVM_ABI Constant * getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx, Type *OnlyIfReducedTy=nullptr)
static LLVM_SUPPRESS_DEPRECATED_DECLARATIONS_PUSH Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
Definition Constants.h:1477
static LLVM_ABI Constant * getShuffleVector(Constant *V1, Constant *V2, ArrayRef< int > Mask, Type *OnlyIfReducedTy=nullptr)
static bool isSupportedGetElementPtr(const Type *SrcElemTy)
Whether creating a constant expression for this getelementptr type is supported.
Definition Constants.h:1624
static LLVM_ABI Constant * get(unsigned Opcode, Constant *C1, Constant *C2, unsigned Flags=0, Type *OnlyIfReducedTy=nullptr)
get - Return a binary or shift operator constant expression, folding if possible.
static LLVM_ABI bool isDesirableBinOp(unsigned Opcode)
Whether creating a constant expression for this binary operator is desirable.
static LLVM_ABI Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:420
const APFloat & getValueAPF() const
Definition Constants.h:463
static LLVM_ABI ConstantFP * getZero(Type *Ty, bool Negative=false)
static LLVM_ABI ConstantFP * getNaN(Type *Ty, bool Negative=false, uint64_t Payload=0)
static LLVM_ABI ConstantFP * getInfinity(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
Definition Constants.h:135
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition Constants.h:174
static LLVM_ABI ConstantInt * getBool(LLVMContext &Context, bool V)
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
Definition Constant.h:44
LLVM_ABI Constant * getSplatValue(bool AllowPoison=false) const
If all elements of the vector constant have the same value, return that value.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition Constant.h:65
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
LLVM_ABI Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Constrained floating point compare intrinsics.
This is the common base class for constrained floating point intrinsics.
LLVM_ABI std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
LLVM_ABI std::optional< RoundingMode > getRoundingMode() const
Wrapper for a function that represents a value that functionally represents the original function.
Definition Constants.h:1143
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
iterator end()
Definition DenseMap.h:686
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
static LLVM_ABI bool compare(const APFloat &LHS, const APFloat &RHS, FCmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
Definition Function.cpp:807
bool isStrictFP() const
Determine if the function has strict floating point sematics.
Definition Function.h:637
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags inBounds()
GEPNoWrapFlags withoutNoUnsignedSignedWrap() const
static GEPNoWrapFlags noUnsignedWrap()
bool hasNoUnsignedSignedWrap() const
bool isInBounds() const
LLVM_ABI bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset, function_ref< bool(Value &, APInt &)> ExternalAnalysis=nullptr) const
Accumulate the constant address offset of this GEP if possible.
Definition Operator.cpp:130
static LLVM_ABI Type * getIndexedType(Type *Ty, ArrayRef< Value * > IdxList)
Returns the result type of a getelementptr with the given source element type and indexes.
PointerType * getType() const
Global values are always pointers.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this global belongs to.
Definition Globals.cpp:205
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool isConstant() const
If the value is a global constant, its value is immutable throughout the runtime execution of the pro...
bool hasDefinitiveInitializer() const
hasDefinitiveInitializer - Whether the global variable has an initializer, and any other instances of...
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
Predicate getSignedPredicate() const
For example, EQ->EQ, SLE->SLE, UGT->SGT, etc.
bool isEquality() const
Return true if this predicate is either EQ or NE.
bool isCast() const
bool isBinaryOp() const
bool isUnaryOp() const
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
static APInt getSaturationPoint(Intrinsic::ID ID, unsigned numBits)
Min/max intrinsics are monotonic, they operate on a fixed-bitwidth values, so there is a certain thre...
static ICmpInst::Predicate getPredicate(Intrinsic::ID ID)
Returns the comparison predicate underlying the intrinsic.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Class to represent scalable SIMD vectors.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
SmallBitVector & set()
iterator_range< const_set_bits_iterator > set_bits() const
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition DataLayout.h:746
LLVM_ABI unsigned getElementContainingOffset(uint64_t FixedOffset) const
Given a valid byte offset into the structure, returns the structure index that contains it.
TypeSize getElementOffset(unsigned Idx) const
Definition DataLayout.h:777
Class to represent struct types.
unsigned getNumElements() const
Random access to the elements.
Provides information about what library functions are available for the current target.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:295
bool isByteTy() const
True if this is an instance of ByteType.
Definition Type.h:237
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
bool isSized() const
Return true if it makes sense to take the size of this type.
Definition Type.h:321
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
bool isByteOrByteVectorTy() const
Return true if this is a byte type or a vector of byte types.
Definition Type.h:243
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:293
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:291
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
bool isX86_AMXTy() const
Return true if this is X86 AMX.
Definition Type.h:202
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Definition Type.h:399
LLVM_ABI const fltSemantics & getFltSemantics() const
Definition Type.cpp:96
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
LLVM_ABI const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr, bool LookThroughIntToPtr=false) const
Accumulate the constant offset this value has compared to a base pointer.
LLVM_ABI uint64_t getPointerDereferenceableBytes(const DataLayout &DL, bool &CanBeNull, bool *CanBeFreed) const
Returns the number of bytes known to be dereferenceable for the pointer value.
Definition Value.cpp:913
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:172
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:253
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:238
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI APInt mulhu(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on zero-extended operands.
Definition APInt.cpp:3184
LLVM_ABI APInt pext(const APInt &Val, const APInt &Mask)
Perform a "compress" operation, also known as pext or bext.
Definition APInt.cpp:3264
LLVM_ABI APInt mulhs(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on sign-extended operands.
Definition APInt.cpp:3176
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
Definition APInt.h:2274
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
Definition APInt.h:2279
LLVM_ABI APInt clmul(const APInt &LHS, const APInt &RHS)
Perform a carry-less multiply, also known as XOR multiplication, and return low-bits.
Definition APInt.cpp:3244
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
Definition APInt.h:2284
LLVM_ABI APInt pdep(const APInt &Val, const APInt &Mask)
Perform an "expand" operation, also known as pdep or bdep.
Definition APInt.cpp:3274
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
Definition APInt.h:2289
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ CE
Windows NT (Windows on ARM)
Definition MCAsmInfo.h:51
initializer< Ty > init(const Ty &Val)
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:454
static constexpr cmpResult cmpEqual
Definition APFloat.h:462
@ ebStrict
This corresponds to "fpexcept.strict".
Definition FPEnv.h:42
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition FPEnv.h:40
constexpr double pi
APFloat::roundingMode GetRoundingModeFromImmArg(const Value *ImmArgVal)
DenormalMode GetNVVMDenormMode(bool ShouldFTZ)
bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID)
APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID)
bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID)
APFloat::roundingMode GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID)
bool RCPShouldFTZ(Intrinsic::ID IntrinsicID)
bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
bool FDivShouldFTZ(Intrinsic::ID IntrinsicID)
bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID)
bool FPArithShouldFTZ(Intrinsic::ID IntrinsicID)
bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID)
APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID)
bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID)
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
LLVM_ABI std::error_code status(const Twine &path, file_status &result, bool follow=true)
Get file status as if by POSIX stat().
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
LLVM_ABI Constant * ConstantFoldLoadThroughBitcast(Constant *C, Type *DestTy, const DataLayout &DL)
ConstantFoldLoadThroughBitcast - try to cast constant to destination type returning null if unsuccess...
static double log2(double V)
LLVM_ABI Constant * ConstantFoldSelectInstruction(Constant *Cond, Constant *V1, Constant *V2)
Attempt to constant fold a select instruction with the specified operands.
LLVM_ABI Constant * ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL, const Instruction *I, bool AllowNonDeterministic=true)
Attempt to constant fold a floating point binary operation with the specified operands,...
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
unsigned getPointerAddressSpace(const Type *T)
Definition SPIRVUtils.h:385
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI Constant * ConstantFoldInstruction(const Instruction *I, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldInstruction - Try to constant fold the specified instruction.
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition APFloat.h:1736
LLVM_ABI Constant * ConstantFoldCompareInstruction(CmpInst::Predicate Predicate, Constant *C1, Constant *C2)
LLVM_ABI Constant * ConstantFoldUnaryInstruction(unsigned Opcode, Constant *V)
LLVM_ABI bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, APInt &Offset, const DataLayout &DL, DSOLocalEquivalent **DSOEquiv=nullptr)
If this constant is a constant offset from a global, return the global and the constant.
LLVM_ABI bool isMathLibCallNoop(const CallBase *Call, const TargetLibraryInfo *TLI)
Check whether the given call has no side-effects.
LLVM_ABI Constant * ReadByteArrayFromGlobal(const GlobalVariable *GV, uint64_t Offset)
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
Definition APFloat.h:1816
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI Constant * ConstantFoldBitInsertOperands(Constant *Base, Constant *Val, Constant *Offset, const DataLayout &DL)
Attempt to constant fold a bitinsert instruction with the specified operands.
int ilogb(const APFloat &Arg)
Returns the exponent of the internal representation of the APFloat.
Definition APFloat.h:1707
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:676
LLVM_ABI Constant * ConstantFoldCall(const CallBase *Call, Function *F, ArrayRef< Constant * > Operands, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldCall - Attempt to constant fold a call to the specified function with the specified argum...
LLVM_ABI bool canConstantFoldCallTo(const CallBase *Call, const Function *F, const TargetLibraryInfo *TLI=nullptr)
canConstantFoldCallTo - Return true if its even possible to fold a call to the specified function.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
Definition APFloat.h:1728
LLVM_ABI Constant * ConstantFoldIntrinsic(Intrinsic::ID ID, ArrayRef< Constant * > Ops, Type *Ty, const DataLayout &DL, const Function *CtxF=nullptr)
LLVM_ABI Constant * ConstantFoldExtractValueInstruction(Constant *Agg, ArrayRef< unsigned > Idxs)
Attempt to constant fold an extractvalue instruction with the specified operands and indices.
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Function *CtxF=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
LLVM_ABI Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 maxNum semantics.
Definition APFloat.h:1771
LLVM_ABI Constant * ConstantFoldLoadFromUniformValue(Constant *C, Type *Ty, const DataLayout &DL)
If C is a uniform value where all bits are the same (either all zero, all ones, all undef or all pois...
LLVM_ABI Constant * ConstantFoldUnaryOpOperand(unsigned Opcode, Constant *Op, const DataLayout &DL)
Attempt to constant fold a unary operation with the specified operand.
LLVM_ABI Constant * getLosslessUnsignedTrunc(Constant *C, Type *DestTy, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
LLVM_READONLY LLVM_ABI std::optional< APFloat > exp(const APFloat &X, RoundingMode RM=APFloat::rmNearestTiesToEven, APFloat::opStatus *Status=nullptr)
Implement IEEE 754-2019 exp functions.
Definition APFloat.cpp:6438
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_READONLY APFloat minimumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimumNumber semantics.
Definition APFloat.h:1802
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Returns: X * 2^Exp for integral exponents.
Definition APFloat.h:1716
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
LLVM_ABI Constant * getLosslessSignedTrunc(Constant *C, Type *DestTy, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
LLVM_ABI Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
LLVM_ABI Constant * ConstantFoldLoadFromConst(Constant *C, Type *Ty, const APInt &Offset, const DataLayout &DL)
Extract value of C at the given Offset reinterpreted as Ty.
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI bool intrinsicPropagatesPoison(Intrinsic::ID IID)
Return whether this intrinsic propagates poison for all operands.
LLVM_ABI Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)
Attempt to constant fold a binary operation with the specified operands.
LLVM_ABI Constant * FlushFPConstant(Constant *Operand, const Function *CtxF, bool IsOutput)
Attempt to flush float point constant according to denormal mode set in the instruction's parent func...
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
LLVM_ABI Constant * ConstantFoldBitExtractOperands(Type *Ty, Constant *Src, Constant *Offset, const DataLayout &DL)
Attempt to constant fold a bitextract instruction with the specified result type and operands.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 minNum semantics.
Definition APFloat.h:1752
@ Sub
Subtraction of integers.
LLVM_ABI bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic has a scalar operand.
IntPtrTy
Definition InstrProf.h:82
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ NearestTiesToEven
roundTiesToEven.
@ Dynamic
Denotes mode unknown at compile time.
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
constexpr unsigned BitWidth
LLVM_ABI Constant * getLosslessInvCast(Constant *C, Type *InvCastTo, unsigned CastOp, const DataLayout &DL, PreservedCastFlags *Flags=nullptr)
Try to cast C to InvC losslessly, satisfying CastOp(InvC) equals C, or CastOp(InvC) is a refined valu...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
LLVM_ABI Constant * ConstantFoldCastInstruction(unsigned opcode, Constant *V, Type *DestTy)
LLVM_ABI Constant * ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val, ArrayRef< unsigned > Idxs)
Attempt to constant fold an insertvalue instruction with the specified operands and indices.
LLVM_ABI Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
constexpr uint32_t calculateReflectedCRC32(uint32_t Crc, uint64_t Data, unsigned DataBytes, uint32_t Poly)
Definition CRC.h:34
LLVM_ABI Constant * ConstantFoldInstOperands(const Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
Definition APFloat.h:1789
LLVM_READONLY APFloat maximumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximumNumber semantics.
Definition APFloat.h:1829
LLVM_ABI Constant * ConstantFoldIntegerCast(Constant *C, Type *DestTy, bool IsSigned, const DataLayout &DL)
Constant fold a zext, sext or trunc, depending on IsSigned and whether the DestTy is wider or narrowe...
LLVM_ABI bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
Definition Casting.h:866
LLVM_ABI Constant * ConstantFoldBinaryInstruction(unsigned Opcode, Constant *V1, Constant *V2)
LLVM_ABI Constant * ConstantFoldFP(double(*NativeFP)(double), const APFloat &V, Type *Ty, DenormalMode DenormMode=DenormalMode::getIEEE())
Evaluate NativeFP on V using the host libm, then convert the result to Ty.
Represent subnormal handling kind for floating point instruction inputs and outputs.
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
DenormalModeKind
Represent handled modes for denormal (aka subnormal) modes in the floating point environment.
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ PositiveZero
Denormals are flushed to positive zero.
@ Dynamic
Denormals have unknown treatment.
@ IEEE
IEEE-754 denormal numbers preserved.
constexpr bool isValid() const
DenormalModeKind Output
Denormal flushing mode for floating point instruction results in the default floating point environme...
static constexpr DenormalMode getDynamic()
static constexpr DenormalMode getIEEE()
bool isConstant() const
Returns true if we know the value of all bits.
Definition KnownBits.h:54
const APInt & getConstant() const
Returns the value when all bits have a known value.
Definition KnownBits.h:58