LLVM 22.0.0git
InstCombineCasts.cpp
Go to the documentation of this file.
1//===- InstCombineCasts.cpp -----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visit functions for cast operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/SetVector.h"
17#include "llvm/IR/DataLayout.h"
18#include "llvm/IR/DebugInfo.h"
20#include "llvm/IR/Value.h"
23#include <optional>
24
25using namespace llvm;
26using namespace PatternMatch;
27
28#define DEBUG_TYPE "instcombine"
29
30/// Given an expression that CanEvaluateTruncated or CanEvaluateSExtd returns
31/// true for, actually insert the code to evaluate the expression.
33 bool isSigned) {
36
37 // Otherwise, it must be an instruction.
39 Instruction *Res = nullptr;
40 unsigned Opc = I->getOpcode();
41 switch (Opc) {
42 case Instruction::Add:
43 case Instruction::Sub:
44 case Instruction::Mul:
45 case Instruction::And:
46 case Instruction::Or:
47 case Instruction::Xor:
48 case Instruction::AShr:
49 case Instruction::LShr:
50 case Instruction::Shl:
51 case Instruction::UDiv:
52 case Instruction::URem: {
53 Value *LHS = EvaluateInDifferentType(I->getOperand(0), Ty, isSigned);
54 Value *RHS = EvaluateInDifferentType(I->getOperand(1), Ty, isSigned);
56 if (Opc == Instruction::LShr || Opc == Instruction::AShr)
57 Res->setIsExact(I->isExact());
58 break;
59 }
60 case Instruction::Trunc:
61 case Instruction::ZExt:
62 case Instruction::SExt:
63 // If the source type of the cast is the type we're trying for then we can
64 // just return the source. There's no need to insert it because it is not
65 // new.
66 if (I->getOperand(0)->getType() == Ty)
67 return I->getOperand(0);
68
69 // Otherwise, must be the same type of cast, so just reinsert a new one.
70 // This also handles the case of zext(trunc(x)) -> zext(x).
71 Res = CastInst::CreateIntegerCast(I->getOperand(0), Ty,
72 Opc == Instruction::SExt);
73 break;
74 case Instruction::Select: {
75 Value *True = EvaluateInDifferentType(I->getOperand(1), Ty, isSigned);
76 Value *False = EvaluateInDifferentType(I->getOperand(2), Ty, isSigned);
77 Res = SelectInst::Create(I->getOperand(0), True, False);
78 break;
79 }
80 case Instruction::PHI: {
81 PHINode *OPN = cast<PHINode>(I);
83 for (unsigned i = 0, e = OPN->getNumIncomingValues(); i != e; ++i) {
84 Value *V =
86 NPN->addIncoming(V, OPN->getIncomingBlock(i));
87 }
88 Res = NPN;
89 break;
90 }
91 case Instruction::FPToUI:
92 case Instruction::FPToSI:
93 Res = CastInst::Create(
94 static_cast<Instruction::CastOps>(Opc), I->getOperand(0), Ty);
95 break;
96 case Instruction::Call:
98 switch (II->getIntrinsicID()) {
99 default:
100 llvm_unreachable("Unsupported call!");
101 case Intrinsic::vscale: {
103 I->getModule(), Intrinsic::vscale, {Ty});
104 Res = CallInst::Create(Fn->getFunctionType(), Fn);
105 break;
106 }
107 }
108 }
109 break;
110 case Instruction::ShuffleVector: {
111 auto *ScalarTy = cast<VectorType>(Ty)->getElementType();
112 auto *VTy = cast<VectorType>(I->getOperand(0)->getType());
113 auto *FixedTy = VectorType::get(ScalarTy, VTy->getElementCount());
114 Value *Op0 = EvaluateInDifferentType(I->getOperand(0), FixedTy, isSigned);
115 Value *Op1 = EvaluateInDifferentType(I->getOperand(1), FixedTy, isSigned);
116 Res = new ShuffleVectorInst(Op0, Op1,
117 cast<ShuffleVectorInst>(I)->getShuffleMask());
118 break;
119 }
120 default:
121 // TODO: Can handle more cases here.
122 llvm_unreachable("Unreachable!");
123 }
124
125 Res->takeName(I);
126 return InsertNewInstWith(Res, I->getIterator());
127}
128
130InstCombinerImpl::isEliminableCastPair(const CastInst *CI1,
131 const CastInst *CI2) {
132 Type *SrcTy = CI1->getSrcTy();
133 Type *MidTy = CI1->getDestTy();
134 Type *DstTy = CI2->getDestTy();
135
136 Instruction::CastOps firstOp = CI1->getOpcode();
137 Instruction::CastOps secondOp = CI2->getOpcode();
138 Type *SrcIntPtrTy =
139 SrcTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(SrcTy) : nullptr;
140 Type *DstIntPtrTy =
141 DstTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(DstTy) : nullptr;
142 unsigned Res = CastInst::isEliminableCastPair(firstOp, secondOp, SrcTy, MidTy,
143 DstTy, &DL);
144
145 // We don't want to form an inttoptr or ptrtoint that converts to an integer
146 // type that differs from the pointer size.
147 if ((Res == Instruction::IntToPtr && SrcTy != DstIntPtrTy) ||
148 (Res == Instruction::PtrToInt && DstTy != SrcIntPtrTy))
149 Res = 0;
150
151 return Instruction::CastOps(Res);
152}
153
154/// Implement the transforms common to all CastInst visitors.
156 Value *Src = CI.getOperand(0);
157 Type *Ty = CI.getType();
158
159 if (Value *Res =
160 simplifyCastInst(CI.getOpcode(), Src, Ty, SQ.getWithInstruction(&CI)))
161 return replaceInstUsesWith(CI, Res);
162
163 // Try to eliminate a cast of a cast.
164 if (auto *CSrc = dyn_cast<CastInst>(Src)) { // A->B->C cast
165 if (Instruction::CastOps NewOpc = isEliminableCastPair(CSrc, &CI)) {
166 // The first cast (CSrc) is eliminable so we need to fix up or replace
167 // the second cast (CI). CSrc will then have a good chance of being dead.
168 auto *Res = CastInst::Create(NewOpc, CSrc->getOperand(0), Ty);
169 // Point debug users of the dying cast to the new one.
170 if (CSrc->hasOneUse())
171 replaceAllDbgUsesWith(*CSrc, *Res, CI, DT);
172 return Res;
173 }
174 }
175
176 if (auto *Sel = dyn_cast<SelectInst>(Src)) {
177 // We are casting a select. Try to fold the cast into the select if the
178 // select does not have a compare instruction with matching operand types
179 // or the select is likely better done in a narrow type.
180 // Creating a select with operands that are different sizes than its
181 // condition may inhibit other folds and lead to worse codegen.
182 auto *Cmp = dyn_cast<CmpInst>(Sel->getCondition());
183 if (!Cmp || Cmp->getOperand(0)->getType() != Sel->getType() ||
184 (CI.getOpcode() == Instruction::Trunc &&
185 shouldChangeType(CI.getSrcTy(), CI.getType()))) {
186
187 // If it's a bitcast involving vectors, make sure it has the same number
188 // of elements on both sides.
189 if (CI.getOpcode() != Instruction::BitCast ||
191 if (Instruction *NV = FoldOpIntoSelect(CI, Sel)) {
192 replaceAllDbgUsesWith(*Sel, *NV, CI, DT);
193 return NV;
194 }
195 }
196 }
197 }
198
199 // If we are casting a PHI, then fold the cast into the PHI.
200 if (auto *PN = dyn_cast<PHINode>(Src)) {
201 // Don't do this if it would create a PHI node with an illegal type from a
202 // legal type.
203 if (!Src->getType()->isIntegerTy() || !CI.getType()->isIntegerTy() ||
204 shouldChangeType(CI.getSrcTy(), CI.getType()))
205 if (Instruction *NV = foldOpIntoPhi(CI, PN))
206 return NV;
207 }
208
209 // Canonicalize a unary shuffle after the cast if neither operation changes
210 // the size or element size of the input vector.
211 // TODO: We could allow size-changing ops if that doesn't harm codegen.
212 // cast (shuffle X, Mask) --> shuffle (cast X), Mask
213 Value *X;
214 ArrayRef<int> Mask;
215 if (match(Src, m_OneUse(m_Shuffle(m_Value(X), m_Undef(), m_Mask(Mask))))) {
216 // TODO: Allow scalable vectors?
217 auto *SrcTy = dyn_cast<FixedVectorType>(X->getType());
218 auto *DestTy = dyn_cast<FixedVectorType>(Ty);
219 if (SrcTy && DestTy &&
220 SrcTy->getNumElements() == DestTy->getNumElements() &&
221 SrcTy->getPrimitiveSizeInBits() == DestTy->getPrimitiveSizeInBits()) {
222 Value *CastX = Builder.CreateCast(CI.getOpcode(), X, DestTy);
223 return new ShuffleVectorInst(CastX, Mask);
224 }
225 }
226
227 return nullptr;
228}
229
230/// Constants and extensions/truncates from the destination type are always
231/// free to be evaluated in that type. This is a helper for canEvaluate*.
232static bool canAlwaysEvaluateInType(Value *V, Type *Ty) {
233 if (isa<Constant>(V))
234 return match(V, m_ImmConstant());
235
236 Value *X;
237 if ((match(V, m_ZExtOrSExt(m_Value(X))) || match(V, m_Trunc(m_Value(X)))) &&
238 X->getType() == Ty)
239 return true;
240
241 return false;
242}
243
244/// Filter out values that we can not evaluate in the destination type for free.
245/// This is a helper for canEvaluate*.
246static bool canNotEvaluateInType(Value *V, Type *Ty) {
247 if (!isa<Instruction>(V))
248 return true;
249 // We don't extend or shrink something that has multiple uses -- doing so
250 // would require duplicating the instruction which isn't profitable.
251 if (!V->hasOneUse())
252 return true;
253
254 return false;
255}
256
257/// Return true if we can evaluate the specified expression tree as type Ty
258/// instead of its larger type, and arrive with the same value.
259/// This is used by code that tries to eliminate truncates.
260///
261/// Ty will always be a type smaller than V. We should return true if trunc(V)
262/// can be computed by computing V in the smaller type. If V is an instruction,
263/// then trunc(inst(x,y)) can be computed as inst(trunc(x),trunc(y)), which only
264/// makes sense if x and y can be efficiently truncated.
265///
266/// This function works on both vectors and scalars.
267///
269 Instruction *CxtI) {
270 if (canAlwaysEvaluateInType(V, Ty))
271 return true;
272 if (canNotEvaluateInType(V, Ty))
273 return false;
274
275 auto *I = cast<Instruction>(V);
276 Type *OrigTy = V->getType();
277 switch (I->getOpcode()) {
278 case Instruction::Add:
279 case Instruction::Sub:
280 case Instruction::Mul:
281 case Instruction::And:
282 case Instruction::Or:
283 case Instruction::Xor:
284 // These operators can all arbitrarily be extended or truncated.
285 return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
286 canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
287
288 case Instruction::UDiv:
289 case Instruction::URem: {
290 // UDiv and URem can be truncated if all the truncated bits are zero.
291 uint32_t OrigBitWidth = OrigTy->getScalarSizeInBits();
292 uint32_t BitWidth = Ty->getScalarSizeInBits();
293 assert(BitWidth < OrigBitWidth && "Unexpected bitwidths!");
294 APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
295 // Do not preserve the original context instruction. Simplifying div/rem
296 // based on later context may introduce a trap.
297 if (IC.MaskedValueIsZero(I->getOperand(0), Mask, I) &&
298 IC.MaskedValueIsZero(I->getOperand(1), Mask, I)) {
299 return canEvaluateTruncated(I->getOperand(0), Ty, IC, I) &&
300 canEvaluateTruncated(I->getOperand(1), Ty, IC, I);
301 }
302 break;
303 }
304 case Instruction::Shl: {
305 // If we are truncating the result of this SHL, and if it's a shift of an
306 // inrange amount, we can always perform a SHL in a smaller type.
307 uint32_t BitWidth = Ty->getScalarSizeInBits();
308 KnownBits AmtKnownBits =
309 llvm::computeKnownBits(I->getOperand(1), IC.getDataLayout());
310 if (AmtKnownBits.getMaxValue().ult(BitWidth))
311 return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
312 canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
313 break;
314 }
315 case Instruction::LShr: {
316 // If this is a truncate of a logical shr, we can truncate it to a smaller
317 // lshr iff we know that the bits we would otherwise be shifting in are
318 // already zeros.
319 // TODO: It is enough to check that the bits we would be shifting in are
320 // zero - use AmtKnownBits.getMaxValue().
321 uint32_t OrigBitWidth = OrigTy->getScalarSizeInBits();
322 uint32_t BitWidth = Ty->getScalarSizeInBits();
323 KnownBits AmtKnownBits = IC.computeKnownBits(I->getOperand(1), CxtI);
324 APInt MaxShiftAmt = AmtKnownBits.getMaxValue();
325 APInt ShiftedBits = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
326 if (MaxShiftAmt.ult(BitWidth)) {
327 // If the only user is a trunc then we can narrow the shift if any new
328 // MSBs are not going to be used.
329 if (auto *Trunc = dyn_cast<TruncInst>(V->user_back())) {
330 auto DemandedBits = Trunc->getType()->getScalarSizeInBits();
331 if ((MaxShiftAmt + DemandedBits).ule(BitWidth))
332 return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
333 canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
334 }
335 if (IC.MaskedValueIsZero(I->getOperand(0), ShiftedBits, CxtI))
336 return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
337 canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
338 }
339 break;
340 }
341 case Instruction::AShr: {
342 // If this is a truncate of an arithmetic shr, we can truncate it to a
343 // smaller ashr iff we know that all the bits from the sign bit of the
344 // original type and the sign bit of the truncate type are similar.
345 // TODO: It is enough to check that the bits we would be shifting in are
346 // similar to sign bit of the truncate type.
347 uint32_t OrigBitWidth = OrigTy->getScalarSizeInBits();
348 uint32_t BitWidth = Ty->getScalarSizeInBits();
349 KnownBits AmtKnownBits =
350 llvm::computeKnownBits(I->getOperand(1), IC.getDataLayout());
351 unsigned ShiftedBits = OrigBitWidth - BitWidth;
352 if (AmtKnownBits.getMaxValue().ult(BitWidth) &&
353 ShiftedBits < IC.ComputeNumSignBits(I->getOperand(0), CxtI))
354 return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
355 canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
356 break;
357 }
358 case Instruction::Trunc:
359 // trunc(trunc(x)) -> trunc(x)
360 return true;
361 case Instruction::ZExt:
362 case Instruction::SExt:
363 // trunc(ext(x)) -> ext(x) if the source type is smaller than the new dest
364 // trunc(ext(x)) -> trunc(x) if the source type is larger than the new dest
365 return true;
366 case Instruction::Select: {
368 return canEvaluateTruncated(SI->getTrueValue(), Ty, IC, CxtI) &&
369 canEvaluateTruncated(SI->getFalseValue(), Ty, IC, CxtI);
370 }
371 case Instruction::PHI: {
372 // We can change a phi if we can change all operands. Note that we never
373 // get into trouble with cyclic PHIs here because we only consider
374 // instructions with a single use.
375 PHINode *PN = cast<PHINode>(I);
376 for (Value *IncValue : PN->incoming_values())
377 if (!canEvaluateTruncated(IncValue, Ty, IC, CxtI))
378 return false;
379 return true;
380 }
381 case Instruction::FPToUI:
382 case Instruction::FPToSI: {
383 // If the integer type can hold the max FP value, it is safe to cast
384 // directly to that type. Otherwise, we may create poison via overflow
385 // that did not exist in the original code.
386 Type *InputTy = I->getOperand(0)->getType()->getScalarType();
387 const fltSemantics &Semantics = InputTy->getFltSemantics();
388 uint32_t MinBitWidth =
390 I->getOpcode() == Instruction::FPToSI);
391 return Ty->getScalarSizeInBits() >= MinBitWidth;
392 }
393 case Instruction::ShuffleVector:
394 return canEvaluateTruncated(I->getOperand(0), Ty, IC, CxtI) &&
395 canEvaluateTruncated(I->getOperand(1), Ty, IC, CxtI);
396 default:
397 // TODO: Can handle more cases here.
398 break;
399 }
400
401 return false;
402}
403
404/// Given a vector that is bitcast to an integer, optionally logically
405/// right-shifted, and truncated, convert it to an extractelement.
406/// Example (big endian):
407/// trunc (lshr (bitcast <4 x i32> %X to i128), 32) to i32
408/// --->
409/// extractelement <4 x i32> %X, 1
411 InstCombinerImpl &IC) {
412 Value *TruncOp = Trunc.getOperand(0);
413 Type *DestType = Trunc.getType();
414 if (!TruncOp->hasOneUse() || !isa<IntegerType>(DestType))
415 return nullptr;
416
417 Value *VecInput = nullptr;
418 ConstantInt *ShiftVal = nullptr;
419 if (!match(TruncOp, m_CombineOr(m_BitCast(m_Value(VecInput)),
420 m_LShr(m_BitCast(m_Value(VecInput)),
421 m_ConstantInt(ShiftVal)))) ||
422 !isa<VectorType>(VecInput->getType()))
423 return nullptr;
424
425 VectorType *VecType = cast<VectorType>(VecInput->getType());
426 unsigned VecWidth = VecType->getPrimitiveSizeInBits();
427 unsigned DestWidth = DestType->getPrimitiveSizeInBits();
428 unsigned ShiftAmount = ShiftVal ? ShiftVal->getZExtValue() : 0;
429
430 if ((VecWidth % DestWidth != 0) || (ShiftAmount % DestWidth != 0))
431 return nullptr;
432
433 // If the element type of the vector doesn't match the result type,
434 // bitcast it to a vector type that we can extract from.
435 unsigned NumVecElts = VecWidth / DestWidth;
436 if (VecType->getElementType() != DestType) {
437 VecType = FixedVectorType::get(DestType, NumVecElts);
438 VecInput = IC.Builder.CreateBitCast(VecInput, VecType, "bc");
439 }
440
441 unsigned Elt = ShiftAmount / DestWidth;
442 if (IC.getDataLayout().isBigEndian())
443 Elt = NumVecElts - 1 - Elt;
444
445 return ExtractElementInst::Create(VecInput, IC.Builder.getInt32(Elt));
446}
447
448/// Whenever an element is extracted from a vector, optionally shifted down, and
449/// then truncated, canonicalize by converting it to a bitcast followed by an
450/// extractelement.
451///
452/// Examples (little endian):
453/// trunc (extractelement <4 x i64> %X, 0) to i32
454/// --->
455/// extractelement <8 x i32> (bitcast <4 x i64> %X to <8 x i32>), i32 0
456///
457/// trunc (lshr (extractelement <4 x i32> %X, 0), 8) to i8
458/// --->
459/// extractelement <16 x i8> (bitcast <4 x i32> %X to <16 x i8>), i32 1
461 InstCombinerImpl &IC) {
462 Value *Src = Trunc.getOperand(0);
463 Type *SrcType = Src->getType();
464 Type *DstType = Trunc.getType();
465
466 // Only attempt this if we have simple aliasing of the vector elements.
467 // A badly fit destination size would result in an invalid cast.
468 unsigned SrcBits = SrcType->getScalarSizeInBits();
469 unsigned DstBits = DstType->getScalarSizeInBits();
470 unsigned TruncRatio = SrcBits / DstBits;
471 if ((SrcBits % DstBits) != 0)
472 return nullptr;
473
474 Value *VecOp;
475 ConstantInt *Cst;
476 const APInt *ShiftAmount = nullptr;
477 if (!match(Src, m_OneUse(m_ExtractElt(m_Value(VecOp), m_ConstantInt(Cst)))) &&
478 !match(Src,
480 m_APInt(ShiftAmount)))))
481 return nullptr;
482
483 auto *VecOpTy = cast<VectorType>(VecOp->getType());
484 auto VecElts = VecOpTy->getElementCount();
485
486 uint64_t BitCastNumElts = VecElts.getKnownMinValue() * TruncRatio;
487 uint64_t VecOpIdx = Cst->getZExtValue();
488 uint64_t NewIdx = IC.getDataLayout().isBigEndian()
489 ? (VecOpIdx + 1) * TruncRatio - 1
490 : VecOpIdx * TruncRatio;
491
492 // Adjust index by the whole number of truncated elements.
493 if (ShiftAmount) {
494 // Check shift amount is in range and shifts a whole number of truncated
495 // elements.
496 if (ShiftAmount->uge(SrcBits) || ShiftAmount->urem(DstBits) != 0)
497 return nullptr;
498
499 uint64_t IdxOfs = ShiftAmount->udiv(DstBits).getZExtValue();
500 NewIdx = IC.getDataLayout().isBigEndian() ? (NewIdx - IdxOfs)
501 : (NewIdx + IdxOfs);
502 }
503
504 assert(BitCastNumElts <= std::numeric_limits<uint32_t>::max() &&
505 NewIdx <= std::numeric_limits<uint32_t>::max() && "overflow 32-bits");
506
507 auto *BitCastTo =
508 VectorType::get(DstType, BitCastNumElts, VecElts.isScalable());
509 Value *BitCast = IC.Builder.CreateBitCast(VecOp, BitCastTo);
510 return ExtractElementInst::Create(BitCast, IC.Builder.getInt32(NewIdx));
511}
512
513/// Funnel/Rotate left/right may occur in a wider type than necessary because of
514/// type promotion rules. Try to narrow the inputs and convert to funnel shift.
515Instruction *InstCombinerImpl::narrowFunnelShift(TruncInst &Trunc) {
516 assert((isa<VectorType>(Trunc.getSrcTy()) ||
517 shouldChangeType(Trunc.getSrcTy(), Trunc.getType())) &&
518 "Don't narrow to an illegal scalar type");
519
520 // Bail out on strange types. It is possible to handle some of these patterns
521 // even with non-power-of-2 sizes, but it is not a likely scenario.
522 Type *DestTy = Trunc.getType();
523 unsigned NarrowWidth = DestTy->getScalarSizeInBits();
524 unsigned WideWidth = Trunc.getSrcTy()->getScalarSizeInBits();
525 if (!isPowerOf2_32(NarrowWidth))
526 return nullptr;
527
528 // First, find an or'd pair of opposite shifts:
529 // trunc (or (lshr ShVal0, ShAmt0), (shl ShVal1, ShAmt1))
530 BinaryOperator *Or0, *Or1;
531 if (!match(Trunc.getOperand(0), m_OneUse(m_Or(m_BinOp(Or0), m_BinOp(Or1)))))
532 return nullptr;
533
534 Value *ShVal0, *ShVal1, *ShAmt0, *ShAmt1;
535 if (!match(Or0, m_OneUse(m_LogicalShift(m_Value(ShVal0), m_Value(ShAmt0)))) ||
536 !match(Or1, m_OneUse(m_LogicalShift(m_Value(ShVal1), m_Value(ShAmt1)))) ||
537 Or0->getOpcode() == Or1->getOpcode())
538 return nullptr;
539
540 // Canonicalize to or(shl(ShVal0, ShAmt0), lshr(ShVal1, ShAmt1)).
541 if (Or0->getOpcode() == BinaryOperator::LShr) {
542 std::swap(Or0, Or1);
543 std::swap(ShVal0, ShVal1);
544 std::swap(ShAmt0, ShAmt1);
545 }
546 assert(Or0->getOpcode() == BinaryOperator::Shl &&
547 Or1->getOpcode() == BinaryOperator::LShr &&
548 "Illegal or(shift,shift) pair");
549
550 // Match the shift amount operands for a funnel/rotate pattern. This always
551 // matches a subtraction on the R operand.
552 auto matchShiftAmount = [&](Value *L, Value *R, unsigned Width) -> Value * {
553 // The shift amounts may add up to the narrow bit width:
554 // (shl ShVal0, L) | (lshr ShVal1, Width - L)
555 // If this is a funnel shift (different operands are shifted), then the
556 // shift amount can not over-shift (create poison) in the narrow type.
557 unsigned MaxShiftAmountWidth = Log2_32(NarrowWidth);
558 APInt HiBitMask = ~APInt::getLowBitsSet(WideWidth, MaxShiftAmountWidth);
559 if (ShVal0 == ShVal1 || MaskedValueIsZero(L, HiBitMask))
560 if (match(R, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(L)))))
561 return L;
562
563 // The following patterns currently only work for rotation patterns.
564 // TODO: Add more general funnel-shift compatible patterns.
565 if (ShVal0 != ShVal1)
566 return nullptr;
567
568 // The shift amount may be masked with negation:
569 // (shl ShVal0, (X & (Width - 1))) | (lshr ShVal1, ((-X) & (Width - 1)))
570 Value *X;
571 unsigned Mask = Width - 1;
572 if (match(L, m_And(m_Value(X), m_SpecificInt(Mask))) &&
574 return X;
575
576 // Same as above, but the shift amount may be extended after masking:
577 if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) &&
579 return X;
580
581 return nullptr;
582 };
583
584 Value *ShAmt = matchShiftAmount(ShAmt0, ShAmt1, NarrowWidth);
585 bool IsFshl = true; // Sub on LSHR.
586 if (!ShAmt) {
587 ShAmt = matchShiftAmount(ShAmt1, ShAmt0, NarrowWidth);
588 IsFshl = false; // Sub on SHL.
589 }
590 if (!ShAmt)
591 return nullptr;
592
593 // The right-shifted value must have high zeros in the wide type (for example
594 // from 'zext', 'and' or 'shift'). High bits of the left-shifted value are
595 // truncated, so those do not matter.
596 APInt HiBitMask = APInt::getHighBitsSet(WideWidth, WideWidth - NarrowWidth);
597 if (!MaskedValueIsZero(ShVal1, HiBitMask, &Trunc))
598 return nullptr;
599
600 // Adjust the width of ShAmt for narrowed funnel shift operation:
601 // - Zero-extend if ShAmt is narrower than the destination type.
602 // - Truncate if ShAmt is wider, discarding non-significant high-order bits.
603 // This prepares ShAmt for llvm.fshl.i8(trunc(ShVal), trunc(ShVal),
604 // zext/trunc(ShAmt)).
605 Value *NarrowShAmt = Builder.CreateZExtOrTrunc(ShAmt, DestTy);
606
607 Value *X, *Y;
608 X = Y = Builder.CreateTrunc(ShVal0, DestTy);
609 if (ShVal0 != ShVal1)
610 Y = Builder.CreateTrunc(ShVal1, DestTy);
611 Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
612 Function *F =
613 Intrinsic::getOrInsertDeclaration(Trunc.getModule(), IID, DestTy);
614 return CallInst::Create(F, {X, Y, NarrowShAmt});
615}
616
617/// Try to narrow the width of math or bitwise logic instructions by pulling a
618/// truncate ahead of binary operators.
619Instruction *InstCombinerImpl::narrowBinOp(TruncInst &Trunc) {
620 Type *SrcTy = Trunc.getSrcTy();
621 Type *DestTy = Trunc.getType();
622 unsigned SrcWidth = SrcTy->getScalarSizeInBits();
623 unsigned DestWidth = DestTy->getScalarSizeInBits();
624
625 if (!isa<VectorType>(SrcTy) && !shouldChangeType(SrcTy, DestTy))
626 return nullptr;
627
628 BinaryOperator *BinOp;
629 if (!match(Trunc.getOperand(0), m_OneUse(m_BinOp(BinOp))))
630 return nullptr;
631
632 Value *BinOp0 = BinOp->getOperand(0);
633 Value *BinOp1 = BinOp->getOperand(1);
634 switch (BinOp->getOpcode()) {
635 case Instruction::And:
636 case Instruction::Or:
637 case Instruction::Xor:
638 case Instruction::Add:
639 case Instruction::Sub:
640 case Instruction::Mul: {
641 Constant *C;
642 if (match(BinOp0, m_Constant(C))) {
643 // trunc (binop C, X) --> binop (trunc C', X)
644 Constant *NarrowC = ConstantExpr::getTrunc(C, DestTy);
645 Value *TruncX = Builder.CreateTrunc(BinOp1, DestTy);
646 return BinaryOperator::Create(BinOp->getOpcode(), NarrowC, TruncX);
647 }
648 if (match(BinOp1, m_Constant(C))) {
649 // trunc (binop X, C) --> binop (trunc X, C')
650 Constant *NarrowC = ConstantExpr::getTrunc(C, DestTy);
651 Value *TruncX = Builder.CreateTrunc(BinOp0, DestTy);
652 return BinaryOperator::Create(BinOp->getOpcode(), TruncX, NarrowC);
653 }
654 Value *X;
655 if (match(BinOp0, m_ZExtOrSExt(m_Value(X))) && X->getType() == DestTy) {
656 // trunc (binop (ext X), Y) --> binop X, (trunc Y)
657 Value *NarrowOp1 = Builder.CreateTrunc(BinOp1, DestTy);
658 return BinaryOperator::Create(BinOp->getOpcode(), X, NarrowOp1);
659 }
660 if (match(BinOp1, m_ZExtOrSExt(m_Value(X))) && X->getType() == DestTy) {
661 // trunc (binop Y, (ext X)) --> binop (trunc Y), X
662 Value *NarrowOp0 = Builder.CreateTrunc(BinOp0, DestTy);
663 return BinaryOperator::Create(BinOp->getOpcode(), NarrowOp0, X);
664 }
665 break;
666 }
667 case Instruction::LShr:
668 case Instruction::AShr: {
669 // trunc (*shr (trunc A), C) --> trunc(*shr A, C)
670 Value *A;
671 Constant *C;
672 if (match(BinOp0, m_Trunc(m_Value(A))) && match(BinOp1, m_Constant(C))) {
673 unsigned MaxShiftAmt = SrcWidth - DestWidth;
674 // If the shift is small enough, all zero/sign bits created by the shift
675 // are removed by the trunc.
677 APInt(SrcWidth, MaxShiftAmt)))) {
678 auto *OldShift = cast<Instruction>(Trunc.getOperand(0));
679 bool IsExact = OldShift->isExact();
680 if (Constant *ShAmt = ConstantFoldIntegerCast(C, A->getType(),
681 /*IsSigned*/ true, DL)) {
682 ShAmt = Constant::mergeUndefsWith(ShAmt, C);
683 Value *Shift =
684 OldShift->getOpcode() == Instruction::AShr
685 ? Builder.CreateAShr(A, ShAmt, OldShift->getName(), IsExact)
686 : Builder.CreateLShr(A, ShAmt, OldShift->getName(), IsExact);
687 return CastInst::CreateTruncOrBitCast(Shift, DestTy);
688 }
689 }
690 }
691 break;
692 }
693 default: break;
694 }
695
696 if (Instruction *NarrowOr = narrowFunnelShift(Trunc))
697 return NarrowOr;
698
699 return nullptr;
700}
701
702/// Try to narrow the width of a splat shuffle. This could be generalized to any
703/// shuffle with a constant operand, but we limit the transform to avoid
704/// creating a shuffle type that targets may not be able to lower effectively.
706 InstCombiner::BuilderTy &Builder) {
707 auto *Shuf = dyn_cast<ShuffleVectorInst>(Trunc.getOperand(0));
708 if (Shuf && Shuf->hasOneUse() && match(Shuf->getOperand(1), m_Undef()) &&
709 all_equal(Shuf->getShuffleMask()) &&
710 ElementCount::isKnownGE(Shuf->getType()->getElementCount(),
711 cast<VectorType>(Shuf->getOperand(0)->getType())
712 ->getElementCount())) {
713 // trunc (shuf X, Undef, SplatMask) --> shuf (trunc X), Poison, SplatMask
714 // trunc (shuf X, Poison, SplatMask) --> shuf (trunc X), Poison, SplatMask
715 Type *NewTruncTy = Shuf->getOperand(0)->getType()->getWithNewType(
716 Trunc.getType()->getScalarType());
717 Value *NarrowOp = Builder.CreateTrunc(Shuf->getOperand(0), NewTruncTy);
718 return new ShuffleVectorInst(NarrowOp, Shuf->getShuffleMask());
719 }
720
721 return nullptr;
722}
723
724/// Try to narrow the width of an insert element. This could be generalized for
725/// any vector constant, but we limit the transform to insertion into undef to
726/// avoid potential backend problems from unsupported insertion widths. This
727/// could also be extended to handle the case of inserting a scalar constant
728/// into a vector variable.
730 InstCombiner::BuilderTy &Builder) {
731 Instruction::CastOps Opcode = Trunc.getOpcode();
732 assert((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) &&
733 "Unexpected instruction for shrinking");
734
735 auto *InsElt = dyn_cast<InsertElementInst>(Trunc.getOperand(0));
736 if (!InsElt || !InsElt->hasOneUse())
737 return nullptr;
738
739 Type *DestTy = Trunc.getType();
740 Type *DestScalarTy = DestTy->getScalarType();
741 Value *VecOp = InsElt->getOperand(0);
742 Value *ScalarOp = InsElt->getOperand(1);
743 Value *Index = InsElt->getOperand(2);
744
745 if (match(VecOp, m_Undef())) {
746 // trunc (inselt undef, X, Index) --> inselt undef, (trunc X), Index
747 // fptrunc (inselt undef, X, Index) --> inselt undef, (fptrunc X), Index
748 UndefValue *NarrowUndef = UndefValue::get(DestTy);
749 Value *NarrowOp = Builder.CreateCast(Opcode, ScalarOp, DestScalarTy);
750 return InsertElementInst::Create(NarrowUndef, NarrowOp, Index);
751 }
752
753 return nullptr;
754}
755
757 if (Instruction *Result = commonCastTransforms(Trunc))
758 return Result;
759
760 Value *Src = Trunc.getOperand(0);
761 Type *DestTy = Trunc.getType(), *SrcTy = Src->getType();
762 unsigned DestWidth = DestTy->getScalarSizeInBits();
763 unsigned SrcWidth = SrcTy->getScalarSizeInBits();
764
765 // Attempt to truncate the entire input expression tree to the destination
766 // type. Only do this if the dest type is a simple type, don't convert the
767 // expression tree to something weird like i93 unless the source is also
768 // strange.
769 if ((DestTy->isVectorTy() || shouldChangeType(SrcTy, DestTy)) &&
770 canEvaluateTruncated(Src, DestTy, *this, &Trunc)) {
771
772 // If this cast is a truncate, evaluting in a different type always
773 // eliminates the cast, so it is always a win.
775 dbgs() << "ICE: EvaluateInDifferentType converting expression type"
776 " to avoid cast: "
777 << Trunc << '\n');
778 Value *Res = EvaluateInDifferentType(Src, DestTy, false);
779 assert(Res->getType() == DestTy);
780 return replaceInstUsesWith(Trunc, Res);
781 }
782
783 // For integer types, check if we can shorten the entire input expression to
784 // DestWidth * 2, which won't allow removing the truncate, but reducing the
785 // width may enable further optimizations, e.g. allowing for larger
786 // vectorization factors.
787 if (auto *DestITy = dyn_cast<IntegerType>(DestTy)) {
788 if (DestWidth * 2 < SrcWidth) {
789 auto *NewDestTy = DestITy->getExtendedType();
790 if (shouldChangeType(SrcTy, NewDestTy) &&
791 canEvaluateTruncated(Src, NewDestTy, *this, &Trunc)) {
793 dbgs() << "ICE: EvaluateInDifferentType converting expression type"
794 " to reduce the width of operand of"
795 << Trunc << '\n');
796 Value *Res = EvaluateInDifferentType(Src, NewDestTy, false);
797 return new TruncInst(Res, DestTy);
798 }
799 }
800 }
801
802 // See if we can simplify any instructions used by the input whose sole
803 // purpose is to compute bits we don't care about.
805 return &Trunc;
806
807 if (DestWidth == 1) {
808 Value *Zero = Constant::getNullValue(SrcTy);
809
810 Value *X;
811 const APInt *C1;
812 Constant *C2;
813 if (match(Src, m_OneUse(m_Shr(m_Shl(m_Power2(C1), m_Value(X)),
814 m_ImmConstant(C2))))) {
815 // trunc ((C1 << X) >> C2) to i1 --> X == (C2-cttz(C1)), where C1 is pow2
816 Constant *Log2C1 = ConstantInt::get(SrcTy, C1->exactLogBase2());
817 Constant *CmpC = ConstantExpr::getSub(C2, Log2C1);
818 return new ICmpInst(ICmpInst::ICMP_EQ, X, CmpC);
819 }
820
821 if (match(Src, m_Shr(m_Value(X), m_SpecificInt(SrcWidth - 1)))) {
822 // trunc (ashr X, BW-1) to i1 --> icmp slt X, 0
823 // trunc (lshr X, BW-1) to i1 --> icmp slt X, 0
824 return new ICmpInst(ICmpInst::ICMP_SLT, X, Zero);
825 }
826
827 Constant *C;
828 if (match(Src, m_OneUse(m_LShr(m_Value(X), m_ImmConstant(C))))) {
829 // trunc (lshr X, C) to i1 --> icmp ne (and X, C'), 0
830 Constant *One = ConstantInt::get(SrcTy, APInt(SrcWidth, 1));
831 Value *MaskC = Builder.CreateShl(One, C);
832 Value *And = Builder.CreateAnd(X, MaskC);
833 return new ICmpInst(ICmpInst::ICMP_NE, And, Zero);
834 }
836 m_Deferred(X))))) {
837 // trunc (or (lshr X, C), X) to i1 --> icmp ne (and X, C'), 0
838 Constant *One = ConstantInt::get(SrcTy, APInt(SrcWidth, 1));
839 Value *MaskC = Builder.CreateShl(One, C);
840 Value *And = Builder.CreateAnd(X, Builder.CreateOr(MaskC, One));
841 return new ICmpInst(ICmpInst::ICMP_NE, And, Zero);
842 }
843
844 {
845 const APInt *C;
846 if (match(Src, m_Shl(m_APInt(C), m_Value(X))) && (*C)[0] == 1) {
847 // trunc (C << X) to i1 --> X == 0, where C is odd
848 return new ICmpInst(ICmpInst::Predicate::ICMP_EQ, X, Zero);
849 }
850 }
851
852 if (Trunc.hasNoUnsignedWrap() || Trunc.hasNoSignedWrap()) {
853 Value *X, *Y;
854 if (match(Src, m_Xor(m_Value(X), m_Value(Y))))
855 return new ICmpInst(ICmpInst::ICMP_NE, X, Y);
856 }
857 }
858
859 Value *A, *B;
860 Constant *C;
861 if (match(Src, m_LShr(m_SExt(m_Value(A)), m_Constant(C)))) {
862 unsigned AWidth = A->getType()->getScalarSizeInBits();
863 unsigned MaxShiftAmt = SrcWidth - std::max(DestWidth, AWidth);
864 auto *OldSh = cast<Instruction>(Src);
865 bool IsExact = OldSh->isExact();
866
867 // If the shift is small enough, all zero bits created by the shift are
868 // removed by the trunc.
870 APInt(SrcWidth, MaxShiftAmt)))) {
871 auto GetNewShAmt = [&](unsigned Width) {
872 Constant *MaxAmt = ConstantInt::get(SrcTy, Width - 1, false);
873 Constant *Cmp =
875 Constant *ShAmt = ConstantFoldSelectInstruction(Cmp, C, MaxAmt);
876 return ConstantFoldCastOperand(Instruction::Trunc, ShAmt, A->getType(),
877 DL);
878 };
879
880 // trunc (lshr (sext A), C) --> ashr A, C
881 if (A->getType() == DestTy) {
882 Constant *ShAmt = GetNewShAmt(DestWidth);
883 ShAmt = Constant::mergeUndefsWith(ShAmt, C);
884 return IsExact ? BinaryOperator::CreateExactAShr(A, ShAmt)
885 : BinaryOperator::CreateAShr(A, ShAmt);
886 }
887 // The types are mismatched, so create a cast after shifting:
888 // trunc (lshr (sext A), C) --> sext/trunc (ashr A, C)
889 if (Src->hasOneUse()) {
890 Constant *ShAmt = GetNewShAmt(AWidth);
891 Value *Shift = Builder.CreateAShr(A, ShAmt, "", IsExact);
892 return CastInst::CreateIntegerCast(Shift, DestTy, true);
893 }
894 }
895 // TODO: Mask high bits with 'and'.
896 }
897
898 if (Instruction *I = narrowBinOp(Trunc))
899 return I;
900
902 return I;
903
904 if (Instruction *I = shrinkInsertElt(Trunc, Builder))
905 return I;
906
907 if (Src->hasOneUse() &&
908 (isa<VectorType>(SrcTy) || shouldChangeType(SrcTy, DestTy))) {
909 // Transform "trunc (shl X, cst)" -> "shl (trunc X), cst" so long as the
910 // dest type is native and cst < dest size.
911 if (match(Src, m_Shl(m_Value(A), m_Constant(C))) &&
912 !match(A, m_Shr(m_Value(), m_Constant()))) {
913 // Skip shifts of shift by constants. It undoes a combine in
914 // FoldShiftByConstant and is the extend in reg pattern.
915 APInt Threshold = APInt(C->getType()->getScalarSizeInBits(), DestWidth);
916 if (match(C, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, Threshold))) {
917 Value *NewTrunc = Builder.CreateTrunc(A, DestTy, A->getName() + ".tr");
918 return BinaryOperator::Create(Instruction::Shl, NewTrunc,
919 ConstantExpr::getTrunc(C, DestTy));
920 }
921 }
922 }
923
924 if (Instruction *I = foldVecTruncToExtElt(Trunc, *this))
925 return I;
926
927 if (Instruction *I = foldVecExtTruncToExtElt(Trunc, *this))
928 return I;
929
930 // trunc (ctlz_i32(zext(A), B) --> add(ctlz_i16(A, B), C)
932 m_Value(B))))) {
933 unsigned AWidth = A->getType()->getScalarSizeInBits();
934 if (AWidth == DestWidth && AWidth > Log2_32(SrcWidth)) {
935 Value *WidthDiff = ConstantInt::get(A->getType(), SrcWidth - AWidth);
936 Value *NarrowCtlz =
937 Builder.CreateIntrinsic(Intrinsic::ctlz, {Trunc.getType()}, {A, B});
938 return BinaryOperator::CreateAdd(NarrowCtlz, WidthDiff);
939 }
940 }
941
942 if (match(Src, m_VScale())) {
943 if (Trunc.getFunction() &&
944 Trunc.getFunction()->hasFnAttribute(Attribute::VScaleRange)) {
945 Attribute Attr =
946 Trunc.getFunction()->getFnAttribute(Attribute::VScaleRange);
947 if (std::optional<unsigned> MaxVScale = Attr.getVScaleRangeMax())
948 if (Log2_32(*MaxVScale) < DestWidth)
949 return replaceInstUsesWith(Trunc, Builder.CreateVScale(DestTy));
950 }
951 }
952
953 if (DestWidth == 1 &&
954 (Trunc.hasNoUnsignedWrap() || Trunc.hasNoSignedWrap()) &&
955 isKnownNonZero(Src, SQ.getWithInstruction(&Trunc)))
956 return replaceInstUsesWith(Trunc, ConstantInt::getTrue(DestTy));
957
958 bool Changed = false;
959 if (!Trunc.hasNoSignedWrap() &&
960 ComputeMaxSignificantBits(Src, &Trunc) <= DestWidth) {
961 Trunc.setHasNoSignedWrap(true);
962 Changed = true;
963 }
964 if (!Trunc.hasNoUnsignedWrap() &&
965 MaskedValueIsZero(Src, APInt::getBitsSetFrom(SrcWidth, DestWidth),
966 &Trunc)) {
967 Trunc.setHasNoUnsignedWrap(true);
968 Changed = true;
969 }
970
971 const APInt *C1;
972 Value *V1;
973 // OP = { lshr, ashr }
974 // trunc ( OP i8 C1, V1) to i1 -> icmp eq V1, log_2(C1) iff C1 is power of 2
975 if (DestWidth == 1 && match(Src, m_Shr(m_Power2(C1), m_Value(V1)))) {
976 Value *Right = ConstantInt::get(V1->getType(), C1->countr_zero());
977 return new ICmpInst(ICmpInst::ICMP_EQ, V1, Right);
978 }
979
980 // OP = { lshr, ashr }
981 // trunc ( OP i8 C1, V1) to i1 -> icmp ult V1, log_2(C1 + 1) iff (C1 + 1) is
982 // power of 2
983 if (DestWidth == 1 && match(Src, m_Shr(m_LowBitMask(C1), m_Value(V1)))) {
984 Value *Right = ConstantInt::get(V1->getType(), C1->countr_one());
985 return new ICmpInst(ICmpInst::ICMP_ULT, V1, Right);
986 }
987
988 // OP = { lshr, ashr }
989 // trunc ( OP i8 C1, V1) to i1 -> icmp ugt V1, cttz(C1) - 1 iff (C1) is
990 // negative power of 2
991 if (DestWidth == 1 && match(Src, m_Shr(m_NegatedPower2(C1), m_Value(V1)))) {
992 Value *Right = ConstantInt::get(V1->getType(), C1->countr_zero());
993 return new ICmpInst(ICmpInst::ICMP_UGE, V1, Right);
994 }
995
996 return Changed ? &Trunc : nullptr;
997}
998
999Instruction *InstCombinerImpl::transformZExtICmp(ICmpInst *Cmp,
1000 ZExtInst &Zext) {
1001 // If we are just checking for a icmp eq of a single bit and zext'ing it
1002 // to an integer, then shift the bit to the appropriate place and then
1003 // cast to integer to avoid the comparison.
1004
1005 // FIXME: This set of transforms does not check for extra uses and/or creates
1006 // an extra instruction (an optional final cast is not included
1007 // in the transform comments). We may also want to favor icmp over
1008 // shifts in cases of equal instructions because icmp has better
1009 // analysis in general (invert the transform).
1010
1011 const APInt *Op1CV;
1012 if (match(Cmp->getOperand(1), m_APInt(Op1CV))) {
1013
1014 // zext (x <s 0) to i32 --> x>>u31 true if signbit set.
1015 if (Cmp->getPredicate() == ICmpInst::ICMP_SLT && Op1CV->isZero()) {
1016 Value *In = Cmp->getOperand(0);
1017 Value *Sh = ConstantInt::get(In->getType(),
1018 In->getType()->getScalarSizeInBits() - 1);
1019 In = Builder.CreateLShr(In, Sh, In->getName() + ".lobit");
1020 if (In->getType() != Zext.getType())
1021 In = Builder.CreateIntCast(In, Zext.getType(), false /*ZExt*/);
1022
1023 return replaceInstUsesWith(Zext, In);
1024 }
1025
1026 // zext (X == 0) to i32 --> X^1 iff X has only the low bit set.
1027 // zext (X == 0) to i32 --> (X>>1)^1 iff X has only the 2nd bit set.
1028 // zext (X != 0) to i32 --> X iff X has only the low bit set.
1029 // zext (X != 0) to i32 --> X>>1 iff X has only the 2nd bit set.
1030
1031 if (Op1CV->isZero() && Cmp->isEquality()) {
1032 // Exactly 1 possible 1? But not the high-bit because that is
1033 // canonicalized to this form.
1034 KnownBits Known = computeKnownBits(Cmp->getOperand(0), &Zext);
1035 APInt KnownZeroMask(~Known.Zero);
1036 uint32_t ShAmt = KnownZeroMask.logBase2();
1037 bool IsExpectShAmt = KnownZeroMask.isPowerOf2() &&
1038 (Zext.getType()->getScalarSizeInBits() != ShAmt + 1);
1039 if (IsExpectShAmt &&
1040 (Cmp->getOperand(0)->getType() == Zext.getType() ||
1041 Cmp->getPredicate() == ICmpInst::ICMP_NE || ShAmt == 0)) {
1042 Value *In = Cmp->getOperand(0);
1043 if (ShAmt) {
1044 // Perform a logical shr by shiftamt.
1045 // Insert the shift to put the result in the low bit.
1046 In = Builder.CreateLShr(In, ConstantInt::get(In->getType(), ShAmt),
1047 In->getName() + ".lobit");
1048 }
1049
1050 // Toggle the low bit for "X == 0".
1051 if (Cmp->getPredicate() == ICmpInst::ICMP_EQ)
1052 In = Builder.CreateXor(In, ConstantInt::get(In->getType(), 1));
1053
1054 if (Zext.getType() == In->getType())
1055 return replaceInstUsesWith(Zext, In);
1056
1057 Value *IntCast = Builder.CreateIntCast(In, Zext.getType(), false);
1058 return replaceInstUsesWith(Zext, IntCast);
1059 }
1060 }
1061 }
1062
1063 if (Cmp->isEquality()) {
1064 // Test if a bit is clear/set using a shifted-one mask:
1065 // zext (icmp eq (and X, (1 << ShAmt)), 0) --> and (lshr (not X), ShAmt), 1
1066 // zext (icmp ne (and X, (1 << ShAmt)), 0) --> and (lshr X, ShAmt), 1
1067 Value *X, *ShAmt;
1068 if (Cmp->hasOneUse() && match(Cmp->getOperand(1), m_ZeroInt()) &&
1069 match(Cmp->getOperand(0),
1070 m_OneUse(m_c_And(m_Shl(m_One(), m_Value(ShAmt)), m_Value(X))))) {
1071 auto *And = cast<BinaryOperator>(Cmp->getOperand(0));
1072 Value *Shift = And->getOperand(X == And->getOperand(0) ? 1 : 0);
1073 if (Zext.getType() == And->getType() ||
1074 Cmp->getPredicate() != ICmpInst::ICMP_EQ || Shift->hasOneUse()) {
1075 if (Cmp->getPredicate() == ICmpInst::ICMP_EQ)
1076 X = Builder.CreateNot(X);
1077 Value *Lshr = Builder.CreateLShr(X, ShAmt);
1078 Value *And1 =
1079 Builder.CreateAnd(Lshr, ConstantInt::get(X->getType(), 1));
1080 return replaceInstUsesWith(
1081 Zext, Builder.CreateZExtOrTrunc(And1, Zext.getType()));
1082 }
1083 }
1084 }
1085
1086 return nullptr;
1087}
1088
1089/// Determine if the specified value can be computed in the specified wider type
1090/// and produce the same low bits. If not, return false.
1091///
1092/// If this function returns true, it can also return a non-zero number of bits
1093/// (in BitsToClear) which indicates that the value it computes is correct for
1094/// the zero extend, but that the additional BitsToClear bits need to be zero'd
1095/// out. For example, to promote something like:
1096///
1097/// %B = trunc i64 %A to i32
1098/// %C = lshr i32 %B, 8
1099/// %E = zext i32 %C to i64
1100///
1101/// CanEvaluateZExtd for the 'lshr' will return true, and BitsToClear will be
1102/// set to 8 to indicate that the promoted value needs to have bits 24-31
1103/// cleared in addition to bits 32-63. Since an 'and' will be generated to
1104/// clear the top bits anyway, doing this has no extra cost.
1105///
1106/// This function works on both vectors and scalars.
1107static bool canEvaluateZExtd(Value *V, Type *Ty, unsigned &BitsToClear,
1108 InstCombinerImpl &IC, Instruction *CxtI) {
1109 BitsToClear = 0;
1110 if (canAlwaysEvaluateInType(V, Ty))
1111 return true;
1112 if (canNotEvaluateInType(V, Ty))
1113 return false;
1114
1115 auto *I = cast<Instruction>(V);
1116 unsigned Tmp;
1117 switch (I->getOpcode()) {
1118 case Instruction::ZExt: // zext(zext(x)) -> zext(x).
1119 case Instruction::SExt: // zext(sext(x)) -> sext(x).
1120 case Instruction::Trunc: // zext(trunc(x)) -> trunc(x) or zext(x)
1121 return true;
1122 case Instruction::And:
1123 case Instruction::Or:
1124 case Instruction::Xor:
1125 case Instruction::Add:
1126 case Instruction::Sub:
1127 case Instruction::Mul:
1128 if (!canEvaluateZExtd(I->getOperand(0), Ty, BitsToClear, IC, CxtI) ||
1129 !canEvaluateZExtd(I->getOperand(1), Ty, Tmp, IC, CxtI))
1130 return false;
1131 // These can all be promoted if neither operand has 'bits to clear'.
1132 if (BitsToClear == 0 && Tmp == 0)
1133 return true;
1134
1135 // If the operation is an AND/OR/XOR and the bits to clear are zero in the
1136 // other side, BitsToClear is ok.
1137 if (Tmp == 0 && I->isBitwiseLogicOp()) {
1138 // We use MaskedValueIsZero here for generality, but the case we care
1139 // about the most is constant RHS.
1140 unsigned VSize = V->getType()->getScalarSizeInBits();
1141 if (IC.MaskedValueIsZero(I->getOperand(1),
1142 APInt::getHighBitsSet(VSize, BitsToClear),
1143 CxtI)) {
1144 // If this is an And instruction and all of the BitsToClear are
1145 // known to be zero we can reset BitsToClear.
1146 if (I->getOpcode() == Instruction::And)
1147 BitsToClear = 0;
1148 return true;
1149 }
1150 }
1151
1152 // Otherwise, we don't know how to analyze this BitsToClear case yet.
1153 return false;
1154
1155 case Instruction::Shl: {
1156 // We can promote shl(x, cst) if we can promote x. Since shl overwrites the
1157 // upper bits we can reduce BitsToClear by the shift amount.
1158 uint64_t ShiftAmt;
1159 if (match(I->getOperand(1), m_ConstantInt(ShiftAmt))) {
1160 if (!canEvaluateZExtd(I->getOperand(0), Ty, BitsToClear, IC, CxtI))
1161 return false;
1162 BitsToClear = ShiftAmt < BitsToClear ? BitsToClear - ShiftAmt : 0;
1163 return true;
1164 }
1165 return false;
1166 }
1167 case Instruction::LShr: {
1168 // We can promote lshr(x, cst) if we can promote x. This requires the
1169 // ultimate 'and' to clear out the high zero bits we're clearing out though.
1170 uint64_t ShiftAmt;
1171 if (match(I->getOperand(1), m_ConstantInt(ShiftAmt))) {
1172 if (!canEvaluateZExtd(I->getOperand(0), Ty, BitsToClear, IC, CxtI))
1173 return false;
1174 BitsToClear += ShiftAmt;
1175 if (BitsToClear > V->getType()->getScalarSizeInBits())
1176 BitsToClear = V->getType()->getScalarSizeInBits();
1177 return true;
1178 }
1179 // Cannot promote variable LSHR.
1180 return false;
1181 }
1182 case Instruction::Select:
1183 if (!canEvaluateZExtd(I->getOperand(1), Ty, Tmp, IC, CxtI) ||
1184 !canEvaluateZExtd(I->getOperand(2), Ty, BitsToClear, IC, CxtI) ||
1185 // TODO: If important, we could handle the case when the BitsToClear are
1186 // known zero in the disagreeing side.
1187 Tmp != BitsToClear)
1188 return false;
1189 return true;
1190
1191 case Instruction::PHI: {
1192 // We can change a phi if we can change all operands. Note that we never
1193 // get into trouble with cyclic PHIs here because we only consider
1194 // instructions with a single use.
1195 PHINode *PN = cast<PHINode>(I);
1196 if (!canEvaluateZExtd(PN->getIncomingValue(0), Ty, BitsToClear, IC, CxtI))
1197 return false;
1198 for (unsigned i = 1, e = PN->getNumIncomingValues(); i != e; ++i)
1199 if (!canEvaluateZExtd(PN->getIncomingValue(i), Ty, Tmp, IC, CxtI) ||
1200 // TODO: If important, we could handle the case when the BitsToClear
1201 // are known zero in the disagreeing input.
1202 Tmp != BitsToClear)
1203 return false;
1204 return true;
1205 }
1206 case Instruction::Call:
1207 // llvm.vscale() can always be executed in larger type, because the
1208 // value is automatically zero-extended.
1210 if (II->getIntrinsicID() == Intrinsic::vscale)
1211 return true;
1212 return false;
1213 default:
1214 // TODO: Can handle more cases here.
1215 return false;
1216 }
1217}
1218
1220 // If this zero extend is only used by a truncate, let the truncate be
1221 // eliminated before we try to optimize this zext.
1222 if (Zext.hasOneUse() && isa<TruncInst>(Zext.user_back()) &&
1223 !isa<Constant>(Zext.getOperand(0)))
1224 return nullptr;
1225
1226 // If one of the common conversion will work, do it.
1227 if (Instruction *Result = commonCastTransforms(Zext))
1228 return Result;
1229
1230 Value *Src = Zext.getOperand(0);
1231 Type *SrcTy = Src->getType(), *DestTy = Zext.getType();
1232
1233 // zext nneg bool x -> 0
1234 if (SrcTy->isIntOrIntVectorTy(1) && Zext.hasNonNeg())
1236
1237 // Try to extend the entire expression tree to the wide destination type.
1238 unsigned BitsToClear;
1239 if (shouldChangeType(SrcTy, DestTy) &&
1240 canEvaluateZExtd(Src, DestTy, BitsToClear, *this, &Zext)) {
1241 assert(BitsToClear <= SrcTy->getScalarSizeInBits() &&
1242 "Can't clear more bits than in SrcTy");
1243
1244 // Okay, we can transform this! Insert the new expression now.
1245 LLVM_DEBUG(
1246 dbgs() << "ICE: EvaluateInDifferentType converting expression type"
1247 " to avoid zero extend: "
1248 << Zext << '\n');
1249 Value *Res = EvaluateInDifferentType(Src, DestTy, false);
1250 assert(Res->getType() == DestTy);
1251
1252 // Preserve debug values referring to Src if the zext is its last use.
1253 if (auto *SrcOp = dyn_cast<Instruction>(Src))
1254 if (SrcOp->hasOneUse())
1255 replaceAllDbgUsesWith(*SrcOp, *Res, Zext, DT);
1256
1257 uint32_t SrcBitsKept = SrcTy->getScalarSizeInBits() - BitsToClear;
1258 uint32_t DestBitSize = DestTy->getScalarSizeInBits();
1259
1260 // If the high bits are already filled with zeros, just replace this
1261 // cast with the result.
1263 Res, APInt::getHighBitsSet(DestBitSize, DestBitSize - SrcBitsKept),
1264 &Zext))
1265 return replaceInstUsesWith(Zext, Res);
1266
1267 // We need to emit an AND to clear the high bits.
1268 Constant *C = ConstantInt::get(Res->getType(),
1269 APInt::getLowBitsSet(DestBitSize, SrcBitsKept));
1270 return BinaryOperator::CreateAnd(Res, C);
1271 }
1272
1273 // If this is a TRUNC followed by a ZEXT then we are dealing with integral
1274 // types and if the sizes are just right we can convert this into a logical
1275 // 'and' which will be much cheaper than the pair of casts.
1276 if (auto *CSrc = dyn_cast<TruncInst>(Src)) { // A->B->C cast
1277 // TODO: Subsume this into EvaluateInDifferentType.
1278
1279 // Get the sizes of the types involved. We know that the intermediate type
1280 // will be smaller than A or C, but don't know the relation between A and C.
1281 Value *A = CSrc->getOperand(0);
1282 unsigned SrcSize = A->getType()->getScalarSizeInBits();
1283 unsigned MidSize = CSrc->getType()->getScalarSizeInBits();
1284 unsigned DstSize = DestTy->getScalarSizeInBits();
1285 // If we're actually extending zero bits, then if
1286 // SrcSize < DstSize: zext(a & mask)
1287 // SrcSize == DstSize: a & mask
1288 // SrcSize > DstSize: trunc(a) & mask
1289 if (SrcSize < DstSize) {
1290 APInt AndValue(APInt::getLowBitsSet(SrcSize, MidSize));
1291 Constant *AndConst = ConstantInt::get(A->getType(), AndValue);
1292 Value *And = Builder.CreateAnd(A, AndConst, CSrc->getName() + ".mask");
1293 return new ZExtInst(And, DestTy);
1294 }
1295
1296 if (SrcSize == DstSize) {
1297 APInt AndValue(APInt::getLowBitsSet(SrcSize, MidSize));
1298 return BinaryOperator::CreateAnd(A, ConstantInt::get(A->getType(),
1299 AndValue));
1300 }
1301 if (SrcSize > DstSize) {
1302 Value *Trunc = Builder.CreateTrunc(A, DestTy);
1303 APInt AndValue(APInt::getLowBitsSet(DstSize, MidSize));
1304 return BinaryOperator::CreateAnd(Trunc,
1305 ConstantInt::get(Trunc->getType(),
1306 AndValue));
1307 }
1308 }
1309
1310 if (auto *Cmp = dyn_cast<ICmpInst>(Src))
1311 return transformZExtICmp(Cmp, Zext);
1312
1313 // zext(trunc(X) & C) -> (X & zext(C)).
1314 Constant *C;
1315 Value *X;
1316 if (match(Src, m_OneUse(m_And(m_Trunc(m_Value(X)), m_Constant(C)))) &&
1317 X->getType() == DestTy)
1318 return BinaryOperator::CreateAnd(X, Builder.CreateZExt(C, DestTy));
1319
1320 // zext((trunc(X) & C) ^ C) -> ((X & zext(C)) ^ zext(C)).
1321 Value *And;
1322 if (match(Src, m_OneUse(m_Xor(m_Value(And), m_Constant(C)))) &&
1324 X->getType() == DestTy) {
1325 Value *ZC = Builder.CreateZExt(C, DestTy);
1326 return BinaryOperator::CreateXor(Builder.CreateAnd(X, ZC), ZC);
1327 }
1328
1329 // If we are truncating, masking, and then zexting back to the original type,
1330 // that's just a mask. This is not handled by canEvaluateZextd if the
1331 // intermediate values have extra uses. This could be generalized further for
1332 // a non-constant mask operand.
1333 // zext (and (trunc X), C) --> and X, (zext C)
1334 if (match(Src, m_And(m_Trunc(m_Value(X)), m_Constant(C))) &&
1335 X->getType() == DestTy) {
1336 Value *ZextC = Builder.CreateZExt(C, DestTy);
1337 return BinaryOperator::CreateAnd(X, ZextC);
1338 }
1339
1340 if (match(Src, m_VScale())) {
1341 if (Zext.getFunction() &&
1342 Zext.getFunction()->hasFnAttribute(Attribute::VScaleRange)) {
1343 Attribute Attr =
1344 Zext.getFunction()->getFnAttribute(Attribute::VScaleRange);
1345 if (std::optional<unsigned> MaxVScale = Attr.getVScaleRangeMax()) {
1346 unsigned TypeWidth = Src->getType()->getScalarSizeInBits();
1347 if (Log2_32(*MaxVScale) < TypeWidth)
1348 return replaceInstUsesWith(Zext, Builder.CreateVScale(DestTy));
1349 }
1350 }
1351 }
1352
1353 if (!Zext.hasNonNeg()) {
1354 // If this zero extend is only used by a shift, add nneg flag.
1355 if (Zext.hasOneUse() &&
1356 SrcTy->getScalarSizeInBits() >
1357 Log2_64_Ceil(DestTy->getScalarSizeInBits()) &&
1358 match(Zext.user_back(), m_Shift(m_Value(), m_Specific(&Zext)))) {
1359 Zext.setNonNeg();
1360 return &Zext;
1361 }
1362
1363 if (isKnownNonNegative(Src, SQ.getWithInstruction(&Zext))) {
1364 Zext.setNonNeg();
1365 return &Zext;
1366 }
1367 }
1368
1369 return nullptr;
1370}
1371
1372/// Transform (sext icmp) to bitwise / integer operations to eliminate the icmp.
1373Instruction *InstCombinerImpl::transformSExtICmp(ICmpInst *Cmp,
1374 SExtInst &Sext) {
1375 Value *Op0 = Cmp->getOperand(0), *Op1 = Cmp->getOperand(1);
1376 ICmpInst::Predicate Pred = Cmp->getPredicate();
1377
1378 // Don't bother if Op1 isn't of vector or integer type.
1379 if (!Op1->getType()->isIntOrIntVectorTy())
1380 return nullptr;
1381
1382 if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_ZeroInt())) {
1383 // sext (x <s 0) --> ashr x, 31 (all ones if negative)
1384 Value *Sh = ConstantInt::get(Op0->getType(),
1385 Op0->getType()->getScalarSizeInBits() - 1);
1386 Value *In = Builder.CreateAShr(Op0, Sh, Op0->getName() + ".lobit");
1387 if (In->getType() != Sext.getType())
1388 In = Builder.CreateIntCast(In, Sext.getType(), true /*SExt*/);
1389
1390 return replaceInstUsesWith(Sext, In);
1391 }
1392
1393 if (ConstantInt *Op1C = dyn_cast<ConstantInt>(Op1)) {
1394 // If we know that only one bit of the LHS of the icmp can be set and we
1395 // have an equality comparison with zero or a power of 2, we can transform
1396 // the icmp and sext into bitwise/integer operations.
1397 if (Cmp->hasOneUse() &&
1398 Cmp->isEquality() && (Op1C->isZero() || Op1C->getValue().isPowerOf2())){
1399 KnownBits Known = computeKnownBits(Op0, &Sext);
1400
1401 APInt KnownZeroMask(~Known.Zero);
1402 if (KnownZeroMask.isPowerOf2()) {
1403 Value *In = Cmp->getOperand(0);
1404
1405 // If the icmp tests for a known zero bit we can constant fold it.
1406 if (!Op1C->isZero() && Op1C->getValue() != KnownZeroMask) {
1407 Value *V = Pred == ICmpInst::ICMP_NE ?
1409 ConstantInt::getNullValue(Sext.getType());
1410 return replaceInstUsesWith(Sext, V);
1411 }
1412
1413 if (!Op1C->isZero() == (Pred == ICmpInst::ICMP_NE)) {
1414 // sext ((x & 2^n) == 0) -> (x >> n) - 1
1415 // sext ((x & 2^n) != 2^n) -> (x >> n) - 1
1416 unsigned ShiftAmt = KnownZeroMask.countr_zero();
1417 // Perform a right shift to place the desired bit in the LSB.
1418 if (ShiftAmt)
1419 In = Builder.CreateLShr(In,
1420 ConstantInt::get(In->getType(), ShiftAmt));
1421
1422 // At this point "In" is either 1 or 0. Subtract 1 to turn
1423 // {1, 0} -> {0, -1}.
1424 In = Builder.CreateAdd(In,
1425 ConstantInt::getAllOnesValue(In->getType()),
1426 "sext");
1427 } else {
1428 // sext ((x & 2^n) != 0) -> (x << bitwidth-n) a>> bitwidth-1
1429 // sext ((x & 2^n) == 2^n) -> (x << bitwidth-n) a>> bitwidth-1
1430 unsigned ShiftAmt = KnownZeroMask.countl_zero();
1431 // Perform a left shift to place the desired bit in the MSB.
1432 if (ShiftAmt)
1433 In = Builder.CreateShl(In,
1434 ConstantInt::get(In->getType(), ShiftAmt));
1435
1436 // Distribute the bit over the whole bit width.
1437 In = Builder.CreateAShr(In, ConstantInt::get(In->getType(),
1438 KnownZeroMask.getBitWidth() - 1), "sext");
1439 }
1440
1441 if (Sext.getType() == In->getType())
1442 return replaceInstUsesWith(Sext, In);
1443 return CastInst::CreateIntegerCast(In, Sext.getType(), true/*SExt*/);
1444 }
1445 }
1446 }
1447
1448 return nullptr;
1449}
1450
1451/// Return true if we can take the specified value and return it as type Ty
1452/// without inserting any new casts and without changing the value of the common
1453/// low bits. This is used by code that tries to promote integer operations to
1454/// a wider types will allow us to eliminate the extension.
1455///
1456/// This function works on both vectors and scalars.
1457///
1458static bool canEvaluateSExtd(Value *V, Type *Ty) {
1459 assert(V->getType()->getScalarSizeInBits() < Ty->getScalarSizeInBits() &&
1460 "Can't sign extend type to a smaller type");
1461 if (canAlwaysEvaluateInType(V, Ty))
1462 return true;
1463 if (canNotEvaluateInType(V, Ty))
1464 return false;
1465
1466 auto *I = cast<Instruction>(V);
1467 switch (I->getOpcode()) {
1468 case Instruction::SExt: // sext(sext(x)) -> sext(x)
1469 case Instruction::ZExt: // sext(zext(x)) -> zext(x)
1470 case Instruction::Trunc: // sext(trunc(x)) -> trunc(x) or sext(x)
1471 return true;
1472 case Instruction::And:
1473 case Instruction::Or:
1474 case Instruction::Xor:
1475 case Instruction::Add:
1476 case Instruction::Sub:
1477 case Instruction::Mul:
1478 // These operators can all arbitrarily be extended if their inputs can.
1479 return canEvaluateSExtd(I->getOperand(0), Ty) &&
1480 canEvaluateSExtd(I->getOperand(1), Ty);
1481
1482 //case Instruction::Shl: TODO
1483 //case Instruction::LShr: TODO
1484
1485 case Instruction::Select:
1486 return canEvaluateSExtd(I->getOperand(1), Ty) &&
1487 canEvaluateSExtd(I->getOperand(2), Ty);
1488
1489 case Instruction::PHI: {
1490 // We can change a phi if we can change all operands. Note that we never
1491 // get into trouble with cyclic PHIs here because we only consider
1492 // instructions with a single use.
1493 PHINode *PN = cast<PHINode>(I);
1494 for (Value *IncValue : PN->incoming_values())
1495 if (!canEvaluateSExtd(IncValue, Ty)) return false;
1496 return true;
1497 }
1498 default:
1499 // TODO: Can handle more cases here.
1500 break;
1501 }
1502
1503 return false;
1504}
1505
1507 // If this sign extend is only used by a truncate, let the truncate be
1508 // eliminated before we try to optimize this sext.
1509 if (Sext.hasOneUse() && isa<TruncInst>(Sext.user_back()))
1510 return nullptr;
1511
1512 if (Instruction *I = commonCastTransforms(Sext))
1513 return I;
1514
1515 Value *Src = Sext.getOperand(0);
1516 Type *SrcTy = Src->getType(), *DestTy = Sext.getType();
1517 unsigned SrcBitSize = SrcTy->getScalarSizeInBits();
1518 unsigned DestBitSize = DestTy->getScalarSizeInBits();
1519
1520 // If the value being extended is zero or positive, use a zext instead.
1521 if (isKnownNonNegative(Src, SQ.getWithInstruction(&Sext))) {
1522 auto CI = CastInst::Create(Instruction::ZExt, Src, DestTy);
1523 CI->setNonNeg(true);
1524 return CI;
1525 }
1526
1527 // Try to extend the entire expression tree to the wide destination type.
1528 bool ShouldExtendExpression = true;
1529 Value *TruncSrc = nullptr;
1530 // It is not desirable to extend expression in the trunc + sext pattern when
1531 // destination type is narrower than original (pre-trunc) type.
1532 if (match(Src, m_Trunc(m_Value(TruncSrc))))
1533 if (TruncSrc->getType()->getScalarSizeInBits() > DestBitSize)
1534 ShouldExtendExpression = false;
1535 if (ShouldExtendExpression && shouldChangeType(SrcTy, DestTy) &&
1536 canEvaluateSExtd(Src, DestTy)) {
1537 // Okay, we can transform this! Insert the new expression now.
1538 LLVM_DEBUG(
1539 dbgs() << "ICE: EvaluateInDifferentType converting expression type"
1540 " to avoid sign extend: "
1541 << Sext << '\n');
1542 Value *Res = EvaluateInDifferentType(Src, DestTy, true);
1543 assert(Res->getType() == DestTy);
1544
1545 // If the high bits are already filled with sign bit, just replace this
1546 // cast with the result.
1547 if (ComputeNumSignBits(Res, &Sext) > DestBitSize - SrcBitSize)
1548 return replaceInstUsesWith(Sext, Res);
1549
1550 // We need to emit a shl + ashr to do the sign extend.
1551 Value *ShAmt = ConstantInt::get(DestTy, DestBitSize-SrcBitSize);
1552 return BinaryOperator::CreateAShr(Builder.CreateShl(Res, ShAmt, "sext"),
1553 ShAmt);
1554 }
1555
1556 Value *X = TruncSrc;
1557 if (X) {
1558 // If the input has more sign bits than bits truncated, then convert
1559 // directly to final type.
1560 unsigned XBitSize = X->getType()->getScalarSizeInBits();
1561 bool HasNSW = cast<TruncInst>(Src)->hasNoSignedWrap();
1562 if (HasNSW || (ComputeNumSignBits(X, &Sext) > XBitSize - SrcBitSize)) {
1563 auto *Res = CastInst::CreateIntegerCast(X, DestTy, /* isSigned */ true);
1564 if (auto *ResTrunc = dyn_cast<TruncInst>(Res); ResTrunc && HasNSW)
1565 ResTrunc->setHasNoSignedWrap(true);
1566 return Res;
1567 }
1568
1569 // If input is a trunc from the destination type, then convert into shifts.
1570 if (Src->hasOneUse() && X->getType() == DestTy) {
1571 // sext (trunc X) --> ashr (shl X, C), C
1572 Constant *ShAmt = ConstantInt::get(DestTy, DestBitSize - SrcBitSize);
1573 return BinaryOperator::CreateAShr(Builder.CreateShl(X, ShAmt), ShAmt);
1574 }
1575
1576 // If we are replacing shifted-in high zero bits with sign bits, convert
1577 // the logic shift to arithmetic shift and eliminate the cast to
1578 // intermediate type:
1579 // sext (trunc (lshr Y, C)) --> sext/trunc (ashr Y, C)
1580 Value *Y;
1581 if (Src->hasOneUse() &&
1583 m_SpecificIntAllowPoison(XBitSize - SrcBitSize)))) {
1584 Value *Ashr = Builder.CreateAShr(Y, XBitSize - SrcBitSize);
1585 return CastInst::CreateIntegerCast(Ashr, DestTy, /* isSigned */ true);
1586 }
1587 }
1588
1589 if (auto *Cmp = dyn_cast<ICmpInst>(Src))
1590 return transformSExtICmp(Cmp, Sext);
1591
1592 // If the input is a shl/ashr pair of a same constant, then this is a sign
1593 // extension from a smaller value. If we could trust arbitrary bitwidth
1594 // integers, we could turn this into a truncate to the smaller bit and then
1595 // use a sext for the whole extension. Since we don't, look deeper and check
1596 // for a truncate. If the source and dest are the same type, eliminate the
1597 // trunc and extend and just do shifts. For example, turn:
1598 // %a = trunc i32 %i to i8
1599 // %b = shl i8 %a, C
1600 // %c = ashr i8 %b, C
1601 // %d = sext i8 %c to i32
1602 // into:
1603 // %a = shl i32 %i, 32-(8-C)
1604 // %d = ashr i32 %a, 32-(8-C)
1605 Value *A = nullptr;
1606 // TODO: Eventually this could be subsumed by EvaluateInDifferentType.
1607 Constant *BA = nullptr, *CA = nullptr;
1608 if (match(Src, m_AShr(m_Shl(m_Trunc(m_Value(A)), m_Constant(BA)),
1609 m_ImmConstant(CA))) &&
1610 BA->isElementWiseEqual(CA) && A->getType() == DestTy) {
1611 Constant *WideCurrShAmt =
1612 ConstantFoldCastOperand(Instruction::SExt, CA, DestTy, DL);
1613 assert(WideCurrShAmt && "Constant folding of ImmConstant cannot fail");
1614 Constant *NumLowbitsLeft = ConstantExpr::getSub(
1615 ConstantInt::get(DestTy, SrcTy->getScalarSizeInBits()), WideCurrShAmt);
1616 Constant *NewShAmt = ConstantExpr::getSub(
1617 ConstantInt::get(DestTy, DestTy->getScalarSizeInBits()),
1618 NumLowbitsLeft);
1619 NewShAmt =
1621 A = Builder.CreateShl(A, NewShAmt, Sext.getName());
1622 return BinaryOperator::CreateAShr(A, NewShAmt);
1623 }
1624
1625 // Splatting a bit of constant-index across a value:
1626 // sext (ashr (trunc iN X to iM), M-1) to iN --> ashr (shl X, N-M), N-1
1627 // If the dest type is different, use a cast (adjust use check).
1628 if (match(Src, m_OneUse(m_AShr(m_Trunc(m_Value(X)),
1629 m_SpecificInt(SrcBitSize - 1))))) {
1630 Type *XTy = X->getType();
1631 unsigned XBitSize = XTy->getScalarSizeInBits();
1632 Constant *ShlAmtC = ConstantInt::get(XTy, XBitSize - SrcBitSize);
1633 Constant *AshrAmtC = ConstantInt::get(XTy, XBitSize - 1);
1634 if (XTy == DestTy)
1635 return BinaryOperator::CreateAShr(Builder.CreateShl(X, ShlAmtC),
1636 AshrAmtC);
1637 if (cast<BinaryOperator>(Src)->getOperand(0)->hasOneUse()) {
1638 Value *Ashr = Builder.CreateAShr(Builder.CreateShl(X, ShlAmtC), AshrAmtC);
1639 return CastInst::CreateIntegerCast(Ashr, DestTy, /* isSigned */ true);
1640 }
1641 }
1642
1643 if (match(Src, m_VScale())) {
1644 if (Sext.getFunction() &&
1645 Sext.getFunction()->hasFnAttribute(Attribute::VScaleRange)) {
1646 Attribute Attr =
1647 Sext.getFunction()->getFnAttribute(Attribute::VScaleRange);
1648 if (std::optional<unsigned> MaxVScale = Attr.getVScaleRangeMax())
1649 if (Log2_32(*MaxVScale) < (SrcBitSize - 1))
1650 return replaceInstUsesWith(Sext, Builder.CreateVScale(DestTy));
1651 }
1652 }
1653
1654 return nullptr;
1655}
1656
1657/// Return a Constant* for the specified floating-point constant if it fits
1658/// in the specified FP type without changing its value.
1659static bool fitsInFPType(APFloat F, const fltSemantics &Sem) {
1660 bool losesInfo;
1661 (void)F.convert(Sem, APFloat::rmNearestTiesToEven, &losesInfo);
1662 return !losesInfo;
1663}
1664
1666 bool PreferBFloat) {
1667 // See if the value can be truncated to bfloat and then reextended.
1668 if (PreferBFloat && fitsInFPType(F, APFloat::BFloat()))
1669 return Type::getBFloatTy(Ctx);
1670 // See if the value can be truncated to half and then reextended.
1671 if (!PreferBFloat && fitsInFPType(F, APFloat::IEEEhalf()))
1672 return Type::getHalfTy(Ctx);
1673 // See if the value can be truncated to float and then reextended.
1675 return Type::getFloatTy(Ctx);
1676 if (&F.getSemantics() == &APFloat::IEEEdouble())
1677 return nullptr; // Won't shrink.
1678 // See if the value can be truncated to double and then reextended.
1680 return Type::getDoubleTy(Ctx);
1681 // Don't try to shrink to various long double types.
1682 return nullptr;
1683}
1684
1685static Type *shrinkFPConstant(ConstantFP *CFP, bool PreferBFloat) {
1686 Type *Ty = CFP->getType();
1687 if (Ty->getScalarType()->isPPC_FP128Ty())
1688 return nullptr; // No constant folding of this.
1689
1690 Type *ShrinkTy =
1691 shrinkFPConstant(CFP->getContext(), CFP->getValueAPF(), PreferBFloat);
1692 if (ShrinkTy)
1693 if (auto *VecTy = dyn_cast<VectorType>(Ty))
1694 ShrinkTy = VectorType::get(ShrinkTy, VecTy);
1695
1696 return ShrinkTy;
1697}
1698
1699// Determine if this is a vector of ConstantFPs and if so, return the minimal
1700// type we can safely truncate all elements to.
1701static Type *shrinkFPConstantVector(Value *V, bool PreferBFloat) {
1702 auto *CV = dyn_cast<Constant>(V);
1703 auto *CVVTy = dyn_cast<FixedVectorType>(V->getType());
1704 if (!CV || !CVVTy)
1705 return nullptr;
1706
1707 Type *MinType = nullptr;
1708
1709 unsigned NumElts = CVVTy->getNumElements();
1710
1711 // For fixed-width vectors we find the minimal type by looking
1712 // through the constant values of the vector.
1713 for (unsigned i = 0; i != NumElts; ++i) {
1714 if (isa<UndefValue>(CV->getAggregateElement(i)))
1715 continue;
1716
1717 auto *CFP = dyn_cast_or_null<ConstantFP>(CV->getAggregateElement(i));
1718 if (!CFP)
1719 return nullptr;
1720
1721 Type *T = shrinkFPConstant(CFP, PreferBFloat);
1722 if (!T)
1723 return nullptr;
1724
1725 // If we haven't found a type yet or this type has a larger mantissa than
1726 // our previous type, this is our new minimal type.
1727 if (!MinType || T->getFPMantissaWidth() > MinType->getFPMantissaWidth())
1728 MinType = T;
1729 }
1730
1731 // Make a vector type from the minimal type.
1732 return MinType ? FixedVectorType::get(MinType, NumElts) : nullptr;
1733}
1734
1735/// Find the minimum FP type we can safely truncate to.
1736static Type *getMinimumFPType(Value *V, bool PreferBFloat) {
1737 if (auto *FPExt = dyn_cast<FPExtInst>(V))
1738 return FPExt->getOperand(0)->getType();
1739
1740 // If this value is a constant, return the constant in the smallest FP type
1741 // that can accurately represent it. This allows us to turn
1742 // (float)((double)X+2.0) into x+2.0f.
1743 if (auto *CFP = dyn_cast<ConstantFP>(V))
1744 if (Type *T = shrinkFPConstant(CFP, PreferBFloat))
1745 return T;
1746
1747 // Try to shrink scalable and fixed splat vectors.
1748 if (auto *FPC = dyn_cast<Constant>(V))
1749 if (auto *VTy = dyn_cast<VectorType>(V->getType()))
1750 if (auto *Splat = dyn_cast_or_null<ConstantFP>(FPC->getSplatValue()))
1751 if (Type *T = shrinkFPConstant(Splat, PreferBFloat))
1752 return VectorType::get(T, VTy);
1753
1754 // Try to shrink a vector of FP constants. This returns nullptr on scalable
1755 // vectors
1756 if (Type *T = shrinkFPConstantVector(V, PreferBFloat))
1757 return T;
1758
1759 return V->getType();
1760}
1761
1762/// Return true if the cast from integer to FP can be proven to be exact for all
1763/// possible inputs (the conversion does not lose any precision).
1765 CastInst::CastOps Opcode = I.getOpcode();
1766 assert((Opcode == CastInst::SIToFP || Opcode == CastInst::UIToFP) &&
1767 "Unexpected cast");
1768 Value *Src = I.getOperand(0);
1769 Type *SrcTy = Src->getType();
1770 Type *FPTy = I.getType();
1771 bool IsSigned = Opcode == Instruction::SIToFP;
1772 int SrcSize = (int)SrcTy->getScalarSizeInBits() - IsSigned;
1773
1774 // Easy case - if the source integer type has less bits than the FP mantissa,
1775 // then the cast must be exact.
1776 int DestNumSigBits = FPTy->getFPMantissaWidth();
1777 if (SrcSize <= DestNumSigBits)
1778 return true;
1779
1780 // Cast from FP to integer and back to FP is independent of the intermediate
1781 // integer width because of poison on overflow.
1782 Value *F;
1783 if (match(Src, m_FPToSI(m_Value(F))) || match(Src, m_FPToUI(m_Value(F)))) {
1784 // If this is uitofp (fptosi F), the source needs an extra bit to avoid
1785 // potential rounding of negative FP input values.
1786 int SrcNumSigBits = F->getType()->getFPMantissaWidth();
1787 if (!IsSigned && match(Src, m_FPToSI(m_Value())))
1788 SrcNumSigBits++;
1789
1790 // [su]itofp (fpto[su]i F) --> exact if the source type has less or equal
1791 // significant bits than the destination (and make sure neither type is
1792 // weird -- ppc_fp128).
1793 if (SrcNumSigBits > 0 && DestNumSigBits > 0 &&
1794 SrcNumSigBits <= DestNumSigBits)
1795 return true;
1796 }
1797
1798 // TODO:
1799 // Try harder to find if the source integer type has less significant bits.
1800 // For example, compute number of sign bits.
1801 KnownBits SrcKnown = IC.computeKnownBits(Src, &I);
1802 int SigBits = (int)SrcTy->getScalarSizeInBits() -
1803 SrcKnown.countMinLeadingZeros() -
1804 SrcKnown.countMinTrailingZeros();
1805 if (SigBits <= DestNumSigBits)
1806 return true;
1807
1808 return false;
1809}
1810
1813 return I;
1814
1815 // If we have fptrunc(OpI (fpextend x), (fpextend y)), we would like to
1816 // simplify this expression to avoid one or more of the trunc/extend
1817 // operations if we can do so without changing the numerical results.
1818 //
1819 // The exact manner in which the widths of the operands interact to limit
1820 // what we can and cannot do safely varies from operation to operation, and
1821 // is explained below in the various case statements.
1822 Type *Ty = FPT.getType();
1823 auto *BO = dyn_cast<BinaryOperator>(FPT.getOperand(0));
1824 if (BO && BO->hasOneUse()) {
1825 bool PreferBFloat = Ty->getScalarType()->isBFloatTy();
1826 Type *LHSMinType = getMinimumFPType(BO->getOperand(0), PreferBFloat);
1827 Type *RHSMinType = getMinimumFPType(BO->getOperand(1), PreferBFloat);
1828 unsigned OpWidth = BO->getType()->getFPMantissaWidth();
1829 unsigned LHSWidth = LHSMinType->getFPMantissaWidth();
1830 unsigned RHSWidth = RHSMinType->getFPMantissaWidth();
1831 unsigned SrcWidth = std::max(LHSWidth, RHSWidth);
1832 unsigned DstWidth = Ty->getFPMantissaWidth();
1833 switch (BO->getOpcode()) {
1834 default: break;
1835 case Instruction::FAdd:
1836 case Instruction::FSub:
1837 // For addition and subtraction, the infinitely precise result can
1838 // essentially be arbitrarily wide; proving that double rounding
1839 // will not occur because the result of OpI is exact (as we will for
1840 // FMul, for example) is hopeless. However, we *can* nonetheless
1841 // frequently know that double rounding cannot occur (or that it is
1842 // innocuous) by taking advantage of the specific structure of
1843 // infinitely-precise results that admit double rounding.
1844 //
1845 // Specifically, if OpWidth >= 2*DstWdith+1 and DstWidth is sufficient
1846 // to represent both sources, we can guarantee that the double
1847 // rounding is innocuous (See p50 of Figueroa's 2000 PhD thesis,
1848 // "A Rigorous Framework for Fully Supporting the IEEE Standard ..."
1849 // for proof of this fact).
1850 //
1851 // Note: Figueroa does not consider the case where DstFormat !=
1852 // SrcFormat. It's possible (likely even!) that this analysis
1853 // could be tightened for those cases, but they are rare (the main
1854 // case of interest here is (float)((double)float + float)).
1855 if (OpWidth >= 2*DstWidth+1 && DstWidth >= SrcWidth) {
1856 Value *LHS = Builder.CreateFPTrunc(BO->getOperand(0), Ty);
1857 Value *RHS = Builder.CreateFPTrunc(BO->getOperand(1), Ty);
1858 Instruction *RI = BinaryOperator::Create(BO->getOpcode(), LHS, RHS);
1859 RI->copyFastMathFlags(BO);
1860 return RI;
1861 }
1862 break;
1863 case Instruction::FMul:
1864 // For multiplication, the infinitely precise result has at most
1865 // LHSWidth + RHSWidth significant bits; if OpWidth is sufficient
1866 // that such a value can be exactly represented, then no double
1867 // rounding can possibly occur; we can safely perform the operation
1868 // in the destination format if it can represent both sources.
1869 if (OpWidth >= LHSWidth + RHSWidth && DstWidth >= SrcWidth) {
1870 Value *LHS = Builder.CreateFPTrunc(BO->getOperand(0), Ty);
1871 Value *RHS = Builder.CreateFPTrunc(BO->getOperand(1), Ty);
1872 return BinaryOperator::CreateFMulFMF(LHS, RHS, BO);
1873 }
1874 break;
1875 case Instruction::FDiv:
1876 // For division, we use again use the bound from Figueroa's
1877 // dissertation. I am entirely certain that this bound can be
1878 // tightened in the unbalanced operand case by an analysis based on
1879 // the diophantine rational approximation bound, but the well-known
1880 // condition used here is a good conservative first pass.
1881 // TODO: Tighten bound via rigorous analysis of the unbalanced case.
1882 if (OpWidth >= 2*DstWidth && DstWidth >= SrcWidth) {
1883 Value *LHS = Builder.CreateFPTrunc(BO->getOperand(0), Ty);
1884 Value *RHS = Builder.CreateFPTrunc(BO->getOperand(1), Ty);
1885 return BinaryOperator::CreateFDivFMF(LHS, RHS, BO);
1886 }
1887 break;
1888 case Instruction::FRem: {
1889 // Remainder is straightforward. Remainder is always exact, so the
1890 // type of OpI doesn't enter into things at all. We simply evaluate
1891 // in whichever source type is larger, then convert to the
1892 // destination type.
1893 if (SrcWidth == OpWidth)
1894 break;
1895 Value *LHS, *RHS;
1896 if (LHSWidth == SrcWidth) {
1897 LHS = Builder.CreateFPTrunc(BO->getOperand(0), LHSMinType);
1898 RHS = Builder.CreateFPTrunc(BO->getOperand(1), LHSMinType);
1899 } else {
1900 LHS = Builder.CreateFPTrunc(BO->getOperand(0), RHSMinType);
1901 RHS = Builder.CreateFPTrunc(BO->getOperand(1), RHSMinType);
1902 }
1903
1904 Value *ExactResult = Builder.CreateFRemFMF(LHS, RHS, BO);
1905 return CastInst::CreateFPCast(ExactResult, Ty);
1906 }
1907 }
1908 }
1909
1910 // (fptrunc (fneg x)) -> (fneg (fptrunc x))
1911 Value *X;
1913 if (Op && Op->hasOneUse()) {
1914 FastMathFlags FMF = FPT.getFastMathFlags();
1915 if (auto *FPMO = dyn_cast<FPMathOperator>(Op))
1916 FMF &= FPMO->getFastMathFlags();
1917
1918 if (match(Op, m_FNeg(m_Value(X)))) {
1919 Value *InnerTrunc = Builder.CreateFPTruncFMF(X, Ty, FMF);
1920 Value *Neg = Builder.CreateFNegFMF(InnerTrunc, FMF);
1921 return replaceInstUsesWith(FPT, Neg);
1922 }
1923
1924 // If we are truncating a select that has an extended operand, we can
1925 // narrow the other operand and do the select as a narrow op.
1926 Value *Cond, *X, *Y;
1928 X->getType() == Ty) {
1929 // fptrunc (select Cond, (fpext X), Y --> select Cond, X, (fptrunc Y)
1930 Value *NarrowY = Builder.CreateFPTruncFMF(Y, Ty, FMF);
1931 Value *Sel =
1932 Builder.CreateSelectFMF(Cond, X, NarrowY, FMF, "narrow.sel", Op);
1933 return replaceInstUsesWith(FPT, Sel);
1934 }
1936 X->getType() == Ty) {
1937 // fptrunc (select Cond, Y, (fpext X) --> select Cond, (fptrunc Y), X
1938 Value *NarrowY = Builder.CreateFPTruncFMF(Y, Ty, FMF);
1939 Value *Sel =
1940 Builder.CreateSelectFMF(Cond, NarrowY, X, FMF, "narrow.sel", Op);
1941 return replaceInstUsesWith(FPT, Sel);
1942 }
1943 }
1944
1945 if (auto *II = dyn_cast<IntrinsicInst>(FPT.getOperand(0))) {
1946 switch (II->getIntrinsicID()) {
1947 default: break;
1948 case Intrinsic::ceil:
1949 case Intrinsic::fabs:
1950 case Intrinsic::floor:
1951 case Intrinsic::nearbyint:
1952 case Intrinsic::rint:
1953 case Intrinsic::round:
1954 case Intrinsic::roundeven:
1955 case Intrinsic::trunc: {
1956 Value *Src = II->getArgOperand(0);
1957 if (!Src->hasOneUse())
1958 break;
1959
1960 // Except for fabs, this transformation requires the input of the unary FP
1961 // operation to be itself an fpext from the type to which we're
1962 // truncating.
1963 if (II->getIntrinsicID() != Intrinsic::fabs) {
1964 FPExtInst *FPExtSrc = dyn_cast<FPExtInst>(Src);
1965 if (!FPExtSrc || FPExtSrc->getSrcTy() != Ty)
1966 break;
1967 }
1968
1969 // Do unary FP operation on smaller type.
1970 // (fptrunc (fabs x)) -> (fabs (fptrunc x))
1971 Value *InnerTrunc = Builder.CreateFPTrunc(Src, Ty);
1973 FPT.getModule(), II->getIntrinsicID(), Ty);
1975 II->getOperandBundlesAsDefs(OpBundles);
1976 CallInst *NewCI =
1977 CallInst::Create(Overload, {InnerTrunc}, OpBundles, II->getName());
1978 // A normal value may be converted to an infinity. It means that we cannot
1979 // propagate ninf from the intrinsic. So we propagate FMF from fptrunc.
1980 NewCI->copyFastMathFlags(&FPT);
1981 return NewCI;
1982 }
1983 }
1984 }
1985
1986 if (Instruction *I = shrinkInsertElt(FPT, Builder))
1987 return I;
1988
1989 Value *Src = FPT.getOperand(0);
1990 if (isa<SIToFPInst>(Src) || isa<UIToFPInst>(Src)) {
1991 auto *FPCast = cast<CastInst>(Src);
1992 if (isKnownExactCastIntToFP(*FPCast, *this))
1993 return CastInst::Create(FPCast->getOpcode(), FPCast->getOperand(0), Ty);
1994 }
1995
1996 return nullptr;
1997}
1998
2000 // If the source operand is a cast from integer to FP and known exact, then
2001 // cast the integer operand directly to the destination type.
2002 Type *Ty = FPExt.getType();
2003 Value *Src = FPExt.getOperand(0);
2004 if (isa<SIToFPInst>(Src) || isa<UIToFPInst>(Src)) {
2005 auto *FPCast = cast<CastInst>(Src);
2006 if (isKnownExactCastIntToFP(*FPCast, *this))
2007 return CastInst::Create(FPCast->getOpcode(), FPCast->getOperand(0), Ty);
2008 }
2009
2010 return commonCastTransforms(FPExt);
2011}
2012
2013/// fpto{s/u}i({u/s}itofp(X)) --> X or zext(X) or sext(X) or trunc(X)
2014/// This is safe if the intermediate type has enough bits in its mantissa to
2015/// accurately represent all values of X. For example, this won't work with
2016/// i64 -> float -> i64.
2019 return nullptr;
2020
2021 auto *OpI = cast<CastInst>(FI.getOperand(0));
2022 Value *X = OpI->getOperand(0);
2023 Type *XType = X->getType();
2024 Type *DestType = FI.getType();
2025 bool IsOutputSigned = isa<FPToSIInst>(FI);
2026
2027 // Since we can assume the conversion won't overflow, our decision as to
2028 // whether the input will fit in the float should depend on the minimum
2029 // of the input range and output range.
2030
2031 // This means this is also safe for a signed input and unsigned output, since
2032 // a negative input would lead to undefined behavior.
2033 if (!isKnownExactCastIntToFP(*OpI, *this)) {
2034 // The first cast may not round exactly based on the source integer width
2035 // and FP width, but the overflow UB rules can still allow this to fold.
2036 // If the destination type is narrow, that means the intermediate FP value
2037 // must be large enough to hold the source value exactly.
2038 // For example, (uint8_t)((float)(uint32_t 16777217) is undefined behavior.
2039 int OutputSize = (int)DestType->getScalarSizeInBits();
2040 if (OutputSize > OpI->getType()->getFPMantissaWidth())
2041 return nullptr;
2042 }
2043
2044 if (DestType->getScalarSizeInBits() > XType->getScalarSizeInBits()) {
2045 bool IsInputSigned = isa<SIToFPInst>(OpI);
2046 if (IsInputSigned && IsOutputSigned)
2047 return new SExtInst(X, DestType);
2048 return new ZExtInst(X, DestType);
2049 }
2050 if (DestType->getScalarSizeInBits() < XType->getScalarSizeInBits())
2051 return new TruncInst(X, DestType);
2052
2053 assert(XType == DestType && "Unexpected types for int to FP to int casts");
2054 return replaceInstUsesWith(FI, X);
2055}
2056
2058 // fpto{u/s}i non-norm --> 0
2059 FPClassTest Mask =
2060 FI.getOpcode() == Instruction::FPToUI ? fcPosNormal : fcNormal;
2062 FI.getOperand(0), Mask, IC.getSimplifyQuery().getWithInstruction(&FI));
2063 if (FPClass.isKnownNever(Mask))
2065
2066 return nullptr;
2067}
2068
2070 if (Instruction *I = foldItoFPtoI(FI))
2071 return I;
2072
2073 if (Instruction *I = foldFPtoI(FI, *this))
2074 return I;
2075
2076 return commonCastTransforms(FI);
2077}
2078
2080 if (Instruction *I = foldItoFPtoI(FI))
2081 return I;
2082
2083 if (Instruction *I = foldFPtoI(FI, *this))
2084 return I;
2085
2086 return commonCastTransforms(FI);
2087}
2088
2090 if (Instruction *R = commonCastTransforms(CI))
2091 return R;
2092 if (!CI.hasNonNeg() && isKnownNonNegative(CI.getOperand(0), SQ)) {
2093 CI.setNonNeg();
2094 return &CI;
2095 }
2096 return nullptr;
2097}
2098
2100 if (Instruction *R = commonCastTransforms(CI))
2101 return R;
2102 if (isKnownNonNegative(CI.getOperand(0), SQ)) {
2103 auto *UI =
2104 CastInst::Create(Instruction::UIToFP, CI.getOperand(0), CI.getType());
2105 UI->setNonNeg(true);
2106 return UI;
2107 }
2108 return nullptr;
2109}
2110
2112 // If the source integer type is not the intptr_t type for this target, do a
2113 // trunc or zext to the intptr_t type, then inttoptr of it. This allows the
2114 // cast to be exposed to other transforms.
2115 unsigned AS = CI.getAddressSpace();
2116 if (CI.getOperand(0)->getType()->getScalarSizeInBits() !=
2117 DL.getPointerSizeInBits(AS)) {
2118 Type *Ty = CI.getOperand(0)->getType()->getWithNewType(
2119 DL.getIntPtrType(CI.getContext(), AS));
2120 Value *P = Builder.CreateZExtOrTrunc(CI.getOperand(0), Ty);
2121 return new IntToPtrInst(P, CI.getType());
2122 }
2123
2124 // Replace (inttoptr (add (ptrtoint %Base), %Offset)) with
2125 // (getelementptr i8, %Base, %Offset) if the pointer is only used as integer
2126 // value.
2127 Value *Base;
2128 Value *Offset;
2129 auto UsesPointerAsInt = [](User *U) {
2131 return true;
2132 if (auto *P = dyn_cast<PHINode>(U))
2133 return P->hasOneUse() && isa<ICmpInst, PtrToIntInst>(*P->user_begin());
2134 return false;
2135 };
2136 if (match(CI.getOperand(0),
2138 m_Value(Offset)))) &&
2140 Base->getType()->getPointerAddressSpace() &&
2141 all_of(CI.users(), UsesPointerAsInt)) {
2142 return GetElementPtrInst::Create(Builder.getInt8Ty(), Base, Offset);
2143 }
2144
2146 return I;
2147
2148 return nullptr;
2149}
2150
2152 // Look through chain of one-use GEPs.
2153 Type *PtrTy = Ptr->getType();
2155 while (true) {
2156 auto *GEP = dyn_cast<GEPOperator>(Ptr);
2157 if (!GEP || !GEP->hasOneUse())
2158 break;
2159 GEPs.push_back(GEP);
2160 Ptr = GEP->getPointerOperand();
2161 }
2162
2163 // Don't handle case where GEP converts from pointer to vector.
2164 if (GEPs.empty() || PtrTy != Ptr->getType())
2165 return nullptr;
2166
2167 // Check whether we know the integer value of the base pointer.
2168 Value *Res;
2169 Type *IdxTy = DL.getIndexType(PtrTy);
2170 if (match(Ptr, m_OneUse(m_IntToPtr(m_Value(Res)))) &&
2171 Res->getType() == IntTy && IntTy == IdxTy) {
2172 // pass
2173 } else if (isa<ConstantPointerNull>(Ptr)) {
2174 Res = Constant::getNullValue(IdxTy);
2175 } else {
2176 return nullptr;
2177 }
2178
2179 // Perform the entire operation on integers instead.
2180 for (GEPOperator *GEP : reverse(GEPs)) {
2181 Value *Offset = EmitGEPOffset(GEP);
2182 Res = Builder.CreateAdd(Res, Offset, "", GEP->hasNoUnsignedWrap());
2183 }
2184 return Builder.CreateZExtOrTrunc(Res, IntTy);
2185}
2186
2188 // If the destination integer type is not the intptr_t type for this target,
2189 // do a ptrtoint to intptr_t then do a trunc or zext. This allows the cast
2190 // to be exposed to other transforms.
2192 Type *SrcTy = SrcOp->getType();
2193 Type *Ty = CI.getType();
2194 unsigned AS = CI.getPointerAddressSpace();
2195 unsigned TySize = Ty->getScalarSizeInBits();
2196 unsigned PtrSize = DL.getPointerSizeInBits(AS);
2197 if (TySize != PtrSize) {
2198 Type *IntPtrTy =
2199 SrcTy->getWithNewType(DL.getIntPtrType(CI.getContext(), AS));
2200 Value *P = Builder.CreatePtrToInt(SrcOp, IntPtrTy);
2201 return CastInst::CreateIntegerCast(P, Ty, /*isSigned=*/false);
2202 }
2203
2204 // (ptrtoint (ptrmask P, M))
2205 // -> (and (ptrtoint P), M)
2206 // This is generally beneficial as `and` is better supported than `ptrmask`.
2207 Value *Ptr, *Mask;
2209 m_Value(Mask)))) &&
2210 Mask->getType() == Ty)
2211 return BinaryOperator::CreateAnd(Builder.CreatePtrToInt(Ptr, Ty), Mask);
2212
2213 if (Value *V = foldPtrToIntOrAddrOfGEP(Ty, SrcOp))
2214 return replaceInstUsesWith(CI, V);
2215
2216 Value *Vec, *Scalar, *Index;
2218 m_Value(Scalar), m_Value(Index)))) &&
2219 Vec->getType() == Ty) {
2220 assert(Vec->getType()->getScalarSizeInBits() == PtrSize && "Wrong type");
2221 // Convert the scalar to int followed by insert to eliminate one cast:
2222 // p2i (ins (i2p Vec), Scalar, Index --> ins Vec, (p2i Scalar), Index
2223 Value *NewCast = Builder.CreatePtrToInt(Scalar, Ty->getScalarType());
2224 return InsertElementInst::Create(Vec, NewCast, Index);
2225 }
2226
2227 return commonCastTransforms(CI);
2228}
2229
2232 Type *Ty = CI.getType();
2233
2234 // (ptrtoaddr (ptrmask P, M))
2235 // -> (and (ptrtoaddr P), M)
2236 // This is generally beneficial as `and` is better supported than `ptrmask`.
2237 Value *Ptr, *Mask;
2239 m_Value(Mask)))) &&
2240 Mask->getType() == Ty)
2241 return BinaryOperator::CreateAnd(Builder.CreatePtrToAddr(Ptr), Mask);
2242
2243 if (Value *V = foldPtrToIntOrAddrOfGEP(Ty, SrcOp))
2244 return replaceInstUsesWith(CI, V);
2245
2246 // FIXME: Implement variants of ptrtoint folds.
2247 return commonCastTransforms(CI);
2248}
2249
2250/// This input value (which is known to have vector type) is being zero extended
2251/// or truncated to the specified vector type. Since the zext/trunc is done
2252/// using an integer type, we have a (bitcast(cast(bitcast))) pattern,
2253/// endianness will impact which end of the vector that is extended or
2254/// truncated.
2255///
2256/// A vector is always stored with index 0 at the lowest address, which
2257/// corresponds to the most significant bits for a big endian stored integer and
2258/// the least significant bits for little endian. A trunc/zext of an integer
2259/// impacts the big end of the integer. Thus, we need to add/remove elements at
2260/// the front of the vector for big endian targets, and the back of the vector
2261/// for little endian targets.
2262///
2263/// Try to replace it with a shuffle (and vector/vector bitcast) if possible.
2264///
2265/// The source and destination vector types may have different element types.
2266static Instruction *
2268 InstCombinerImpl &IC) {
2269 // We can only do this optimization if the output is a multiple of the input
2270 // element size, or the input is a multiple of the output element size.
2271 // Convert the input type to have the same element type as the output.
2272 VectorType *SrcTy = cast<VectorType>(InVal->getType());
2273
2274 if (SrcTy->getElementType() != DestTy->getElementType()) {
2275 // The input types don't need to be identical, but for now they must be the
2276 // same size. There is no specific reason we couldn't handle things like
2277 // <4 x i16> -> <4 x i32> by bitcasting to <2 x i32> but haven't gotten
2278 // there yet.
2279 if (SrcTy->getElementType()->getPrimitiveSizeInBits() !=
2280 DestTy->getElementType()->getPrimitiveSizeInBits())
2281 return nullptr;
2282
2283 SrcTy =
2284 FixedVectorType::get(DestTy->getElementType(),
2285 cast<FixedVectorType>(SrcTy)->getNumElements());
2286 InVal = IC.Builder.CreateBitCast(InVal, SrcTy);
2287 }
2288
2289 bool IsBigEndian = IC.getDataLayout().isBigEndian();
2290 unsigned SrcElts = cast<FixedVectorType>(SrcTy)->getNumElements();
2291 unsigned DestElts = cast<FixedVectorType>(DestTy)->getNumElements();
2292
2293 assert(SrcElts != DestElts && "Element counts should be different.");
2294
2295 // Now that the element types match, get the shuffle mask and RHS of the
2296 // shuffle to use, which depends on whether we're increasing or decreasing the
2297 // size of the input.
2298 auto ShuffleMaskStorage = llvm::to_vector<16>(llvm::seq<int>(0, SrcElts));
2299 ArrayRef<int> ShuffleMask;
2300 Value *V2;
2301
2302 if (SrcElts > DestElts) {
2303 // If we're shrinking the number of elements (rewriting an integer
2304 // truncate), just shuffle in the elements corresponding to the least
2305 // significant bits from the input and use poison as the second shuffle
2306 // input.
2307 V2 = PoisonValue::get(SrcTy);
2308 // Make sure the shuffle mask selects the "least significant bits" by
2309 // keeping elements from back of the src vector for big endian, and from the
2310 // front for little endian.
2311 ShuffleMask = ShuffleMaskStorage;
2312 if (IsBigEndian)
2313 ShuffleMask = ShuffleMask.take_back(DestElts);
2314 else
2315 ShuffleMask = ShuffleMask.take_front(DestElts);
2316 } else {
2317 // If we're increasing the number of elements (rewriting an integer zext),
2318 // shuffle in all of the elements from InVal. Fill the rest of the result
2319 // elements with zeros from a constant zero.
2320 V2 = Constant::getNullValue(SrcTy);
2321 // Use first elt from V2 when indicating zero in the shuffle mask.
2322 uint32_t NullElt = SrcElts;
2323 // Extend with null values in the "most significant bits" by adding elements
2324 // in front of the src vector for big endian, and at the back for little
2325 // endian.
2326 unsigned DeltaElts = DestElts - SrcElts;
2327 if (IsBigEndian)
2328 ShuffleMaskStorage.insert(ShuffleMaskStorage.begin(), DeltaElts, NullElt);
2329 else
2330 ShuffleMaskStorage.append(DeltaElts, NullElt);
2331 ShuffleMask = ShuffleMaskStorage;
2332 }
2333
2334 return new ShuffleVectorInst(InVal, V2, ShuffleMask);
2335}
2336
2337static bool isMultipleOfTypeSize(unsigned Value, Type *Ty) {
2338 return Value % Ty->getPrimitiveSizeInBits() == 0;
2339}
2340
2341static unsigned getTypeSizeIndex(unsigned Value, Type *Ty) {
2342 return Value / Ty->getPrimitiveSizeInBits();
2343}
2344
2345/// V is a value which is inserted into a vector of VecEltTy.
2346/// Look through the value to see if we can decompose it into
2347/// insertions into the vector. See the example in the comment for
2348/// OptimizeIntegerToVectorInsertions for the pattern this handles.
2349/// The type of V is always a non-zero multiple of VecEltTy's size.
2350/// Shift is the number of bits between the lsb of V and the lsb of
2351/// the vector.
2352///
2353/// This returns false if the pattern can't be matched or true if it can,
2354/// filling in Elements with the elements found here.
2355static bool collectInsertionElements(Value *V, unsigned Shift,
2356 SmallVectorImpl<Value *> &Elements,
2357 Type *VecEltTy, bool isBigEndian) {
2358 assert(isMultipleOfTypeSize(Shift, VecEltTy) &&
2359 "Shift should be a multiple of the element type size");
2360
2361 // Undef values never contribute useful bits to the result.
2362 if (isa<UndefValue>(V)) return true;
2363
2364 // If we got down to a value of the right type, we win, try inserting into the
2365 // right element.
2366 if (V->getType() == VecEltTy) {
2367 // Inserting null doesn't actually insert any elements.
2368 if (Constant *C = dyn_cast<Constant>(V))
2369 if (C->isNullValue())
2370 return true;
2371
2372 unsigned ElementIndex = getTypeSizeIndex(Shift, VecEltTy);
2373 if (isBigEndian)
2374 ElementIndex = Elements.size() - ElementIndex - 1;
2375
2376 // Fail if multiple elements are inserted into this slot.
2377 if (Elements[ElementIndex])
2378 return false;
2379
2380 Elements[ElementIndex] = V;
2381 return true;
2382 }
2383
2384 if (Constant *C = dyn_cast<Constant>(V)) {
2385 // Figure out the # elements this provides, and bitcast it or slice it up
2386 // as required.
2387 unsigned NumElts = getTypeSizeIndex(C->getType()->getPrimitiveSizeInBits(),
2388 VecEltTy);
2389 // If the constant is the size of a vector element, we just need to bitcast
2390 // it to the right type so it gets properly inserted.
2391 if (NumElts == 1)
2393 Shift, Elements, VecEltTy, isBigEndian);
2394
2395 // Okay, this is a constant that covers multiple elements. Slice it up into
2396 // pieces and insert each element-sized piece into the vector.
2397 if (!isa<IntegerType>(C->getType()))
2398 C = ConstantExpr::getBitCast(C, IntegerType::get(V->getContext(),
2399 C->getType()->getPrimitiveSizeInBits()));
2400 unsigned ElementSize = VecEltTy->getPrimitiveSizeInBits();
2401 Type *ElementIntTy = IntegerType::get(C->getContext(), ElementSize);
2402
2403 for (unsigned i = 0; i != NumElts; ++i) {
2404 unsigned ShiftI = i * ElementSize;
2406 Instruction::LShr, C, ConstantInt::get(C->getType(), ShiftI));
2407 if (!Piece)
2408 return false;
2409
2410 Piece = ConstantExpr::getTrunc(Piece, ElementIntTy);
2411 if (!collectInsertionElements(Piece, ShiftI + Shift, Elements, VecEltTy,
2412 isBigEndian))
2413 return false;
2414 }
2415 return true;
2416 }
2417
2418 if (!V->hasOneUse()) return false;
2419
2421 if (!I) return false;
2422 switch (I->getOpcode()) {
2423 default: return false; // Unhandled case.
2424 case Instruction::BitCast:
2425 if (I->getOperand(0)->getType()->isVectorTy())
2426 return false;
2427 return collectInsertionElements(I->getOperand(0), Shift, Elements, VecEltTy,
2428 isBigEndian);
2429 case Instruction::ZExt:
2431 I->getOperand(0)->getType()->getPrimitiveSizeInBits(),
2432 VecEltTy))
2433 return false;
2434 return collectInsertionElements(I->getOperand(0), Shift, Elements, VecEltTy,
2435 isBigEndian);
2436 case Instruction::Or:
2437 return collectInsertionElements(I->getOperand(0), Shift, Elements, VecEltTy,
2438 isBigEndian) &&
2439 collectInsertionElements(I->getOperand(1), Shift, Elements, VecEltTy,
2440 isBigEndian);
2441 case Instruction::Shl: {
2442 // Must be shifting by a constant that is a multiple of the element size.
2443 ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1));
2444 if (!CI) return false;
2445 Shift += CI->getZExtValue();
2446 if (!isMultipleOfTypeSize(Shift, VecEltTy)) return false;
2447 return collectInsertionElements(I->getOperand(0), Shift, Elements, VecEltTy,
2448 isBigEndian);
2449 }
2450
2451 }
2452}
2453
2454
2455/// If the input is an 'or' instruction, we may be doing shifts and ors to
2456/// assemble the elements of the vector manually.
2457/// Try to rip the code out and replace it with insertelements. This is to
2458/// optimize code like this:
2459///
2460/// %tmp37 = bitcast float %inc to i32
2461/// %tmp38 = zext i32 %tmp37 to i64
2462/// %tmp31 = bitcast float %inc5 to i32
2463/// %tmp32 = zext i32 %tmp31 to i64
2464/// %tmp33 = shl i64 %tmp32, 32
2465/// %ins35 = or i64 %tmp33, %tmp38
2466/// %tmp43 = bitcast i64 %ins35 to <2 x float>
2467///
2468/// Into two insertelements that do "buildvector{%inc, %inc5}".
2470 InstCombinerImpl &IC) {
2471 auto *DestVecTy = cast<FixedVectorType>(CI.getType());
2472 Value *IntInput = CI.getOperand(0);
2473
2474 // if the int input is just an undef value do not try to optimize to vector
2475 // insertions as it will prevent undef propagation
2476 if (isa<UndefValue>(IntInput))
2477 return nullptr;
2478
2479 SmallVector<Value*, 8> Elements(DestVecTy->getNumElements());
2480 if (!collectInsertionElements(IntInput, 0, Elements,
2481 DestVecTy->getElementType(),
2482 IC.getDataLayout().isBigEndian()))
2483 return nullptr;
2484
2485 // If we succeeded, we know that all of the element are specified by Elements
2486 // or are zero if Elements has a null entry. Recast this as a set of
2487 // insertions.
2488 Value *Result = Constant::getNullValue(CI.getType());
2489 for (unsigned i = 0, e = Elements.size(); i != e; ++i) {
2490 if (!Elements[i]) continue; // Unset element.
2491
2492 Result = IC.Builder.CreateInsertElement(Result, Elements[i],
2493 IC.Builder.getInt32(i));
2494 }
2495
2496 return Result;
2497}
2498
2499/// Canonicalize scalar bitcasts of extracted elements into a bitcast of the
2500/// vector followed by extract element. The backend tends to handle bitcasts of
2501/// vectors better than bitcasts of scalars because vector registers are
2502/// usually not type-specific like scalar integer or scalar floating-point.
2504 InstCombinerImpl &IC) {
2505 Value *VecOp, *Index;
2506 if (!match(BitCast.getOperand(0),
2507 m_OneUse(m_ExtractElt(m_Value(VecOp), m_Value(Index)))))
2508 return nullptr;
2509
2510 // The bitcast must be to a vectorizable type, otherwise we can't make a new
2511 // type to extract from.
2512 Type *DestType = BitCast.getType();
2513 VectorType *VecType = cast<VectorType>(VecOp->getType());
2514 if (VectorType::isValidElementType(DestType)) {
2515 auto *NewVecType = VectorType::get(DestType, VecType);
2516 auto *NewBC = IC.Builder.CreateBitCast(VecOp, NewVecType, "bc");
2517 return ExtractElementInst::Create(NewBC, Index);
2518 }
2519
2520 // Only solve DestType is vector to avoid inverse transform in visitBitCast.
2521 // bitcast (extractelement <1 x elt>, dest) -> bitcast(<1 x elt>, dest)
2522 auto *FixedVType = dyn_cast<FixedVectorType>(VecType);
2523 if (DestType->isVectorTy() && FixedVType && FixedVType->getNumElements() == 1)
2524 return CastInst::Create(Instruction::BitCast, VecOp, DestType);
2525
2526 return nullptr;
2527}
2528
2529/// Change the type of a bitwise logic operation if we can eliminate a bitcast.
2531 InstCombiner::BuilderTy &Builder) {
2532 Type *DestTy = BitCast.getType();
2533 BinaryOperator *BO;
2534
2535 if (!match(BitCast.getOperand(0), m_OneUse(m_BinOp(BO))) ||
2536 !BO->isBitwiseLogicOp())
2537 return nullptr;
2538
2539 // FIXME: This transform is restricted to vector types to avoid backend
2540 // problems caused by creating potentially illegal operations. If a fix-up is
2541 // added to handle that situation, we can remove this check.
2542 if (!DestTy->isVectorTy() || !BO->getType()->isVectorTy())
2543 return nullptr;
2544
2545 if (DestTy->isFPOrFPVectorTy()) {
2546 Value *X, *Y;
2547 // bitcast(logic(bitcast(X), bitcast(Y))) -> bitcast'(logic(bitcast'(X), Y))
2548 if (match(BO->getOperand(0), m_OneUse(m_BitCast(m_Value(X)))) &&
2550 if (X->getType()->isFPOrFPVectorTy() &&
2551 Y->getType()->isIntOrIntVectorTy()) {
2552 Value *CastedOp =
2553 Builder.CreateBitCast(BO->getOperand(0), Y->getType());
2554 Value *NewBO = Builder.CreateBinOp(BO->getOpcode(), CastedOp, Y);
2555 return CastInst::CreateBitOrPointerCast(NewBO, DestTy);
2556 }
2557 if (X->getType()->isIntOrIntVectorTy() &&
2558 Y->getType()->isFPOrFPVectorTy()) {
2559 Value *CastedOp =
2560 Builder.CreateBitCast(BO->getOperand(1), X->getType());
2561 Value *NewBO = Builder.CreateBinOp(BO->getOpcode(), CastedOp, X);
2562 return CastInst::CreateBitOrPointerCast(NewBO, DestTy);
2563 }
2564 }
2565 return nullptr;
2566 }
2567
2568 if (!DestTy->isIntOrIntVectorTy())
2569 return nullptr;
2570
2571 Value *X;
2572 if (match(BO->getOperand(0), m_OneUse(m_BitCast(m_Value(X)))) &&
2573 X->getType() == DestTy && !isa<Constant>(X)) {
2574 // bitcast(logic(bitcast(X), Y)) --> logic'(X, bitcast(Y))
2575 Value *CastedOp1 = Builder.CreateBitCast(BO->getOperand(1), DestTy);
2576 return BinaryOperator::Create(BO->getOpcode(), X, CastedOp1);
2577 }
2578
2579 if (match(BO->getOperand(1), m_OneUse(m_BitCast(m_Value(X)))) &&
2580 X->getType() == DestTy && !isa<Constant>(X)) {
2581 // bitcast(logic(Y, bitcast(X))) --> logic'(bitcast(Y), X)
2582 Value *CastedOp0 = Builder.CreateBitCast(BO->getOperand(0), DestTy);
2583 return BinaryOperator::Create(BO->getOpcode(), CastedOp0, X);
2584 }
2585
2586 // Canonicalize vector bitcasts to come before vector bitwise logic with a
2587 // constant. This eases recognition of special constants for later ops.
2588 // Example:
2589 // icmp u/s (a ^ signmask), (b ^ signmask) --> icmp s/u a, b
2590 Constant *C;
2591 if (match(BO->getOperand(1), m_Constant(C))) {
2592 // bitcast (logic X, C) --> logic (bitcast X, C')
2593 Value *CastedOp0 = Builder.CreateBitCast(BO->getOperand(0), DestTy);
2594 Value *CastedC = Builder.CreateBitCast(C, DestTy);
2595 return BinaryOperator::Create(BO->getOpcode(), CastedOp0, CastedC);
2596 }
2597
2598 return nullptr;
2599}
2600
2601/// Change the type of a select if we can eliminate a bitcast.
2603 InstCombiner::BuilderTy &Builder) {
2604 Value *Cond, *TVal, *FVal;
2605 if (!match(BitCast.getOperand(0),
2606 m_OneUse(m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))))
2607 return nullptr;
2608
2609 // A vector select must maintain the same number of elements in its operands.
2610 Type *CondTy = Cond->getType();
2611 Type *DestTy = BitCast.getType();
2612 if (auto *CondVTy = dyn_cast<VectorType>(CondTy))
2613 if (!DestTy->isVectorTy() ||
2614 CondVTy->getElementCount() !=
2615 cast<VectorType>(DestTy)->getElementCount())
2616 return nullptr;
2617
2618 // FIXME: This transform is restricted from changing the select between
2619 // scalars and vectors to avoid backend problems caused by creating
2620 // potentially illegal operations. If a fix-up is added to handle that
2621 // situation, we can remove this check.
2622 if (DestTy->isVectorTy() != TVal->getType()->isVectorTy())
2623 return nullptr;
2624
2625 auto *Sel = cast<Instruction>(BitCast.getOperand(0));
2626 Value *X;
2627 if (match(TVal, m_OneUse(m_BitCast(m_Value(X)))) && X->getType() == DestTy &&
2628 !isa<Constant>(X)) {
2629 // bitcast(select(Cond, bitcast(X), Y)) --> select'(Cond, X, bitcast(Y))
2630 Value *CastedVal = Builder.CreateBitCast(FVal, DestTy);
2631 return SelectInst::Create(Cond, X, CastedVal, "", nullptr, Sel);
2632 }
2633
2634 if (match(FVal, m_OneUse(m_BitCast(m_Value(X)))) && X->getType() == DestTy &&
2635 !isa<Constant>(X)) {
2636 // bitcast(select(Cond, Y, bitcast(X))) --> select'(Cond, bitcast(Y), X)
2637 Value *CastedVal = Builder.CreateBitCast(TVal, DestTy);
2638 return SelectInst::Create(Cond, CastedVal, X, "", nullptr, Sel);
2639 }
2640
2641 return nullptr;
2642}
2643
2644/// Check if all users of CI are StoreInsts.
2645static bool hasStoreUsersOnly(CastInst &CI) {
2646 for (User *U : CI.users()) {
2647 if (!isa<StoreInst>(U))
2648 return false;
2649 }
2650 return true;
2651}
2652
2653/// This function handles following case
2654///
2655/// A -> B cast
2656/// PHI
2657/// B -> A cast
2658///
2659/// All the related PHI nodes can be replaced by new PHI nodes with type A.
2660/// The uses of \p CI can be changed to the new PHI node corresponding to \p PN.
2661Instruction *InstCombinerImpl::optimizeBitCastFromPhi(CastInst &CI,
2662 PHINode *PN) {
2663 // BitCast used by Store can be handled in InstCombineLoadStoreAlloca.cpp.
2664 if (hasStoreUsersOnly(CI))
2665 return nullptr;
2666
2667 Value *Src = CI.getOperand(0);
2668 Type *SrcTy = Src->getType(); // Type B
2669 Type *DestTy = CI.getType(); // Type A
2670
2671 SmallVector<PHINode *, 4> PhiWorklist;
2672 SmallSetVector<PHINode *, 4> OldPhiNodes;
2673
2674 // Find all of the A->B casts and PHI nodes.
2675 // We need to inspect all related PHI nodes, but PHIs can be cyclic, so
2676 // OldPhiNodes is used to track all known PHI nodes, before adding a new
2677 // PHI to PhiWorklist, it is checked against and added to OldPhiNodes first.
2678 PhiWorklist.push_back(PN);
2679 OldPhiNodes.insert(PN);
2680 while (!PhiWorklist.empty()) {
2681 auto *OldPN = PhiWorklist.pop_back_val();
2682 for (Value *IncValue : OldPN->incoming_values()) {
2683 if (isa<Constant>(IncValue))
2684 continue;
2685
2686 if (auto *LI = dyn_cast<LoadInst>(IncValue)) {
2687 // If there is a sequence of one or more load instructions, each loaded
2688 // value is used as address of later load instruction, bitcast is
2689 // necessary to change the value type, don't optimize it. For
2690 // simplicity we give up if the load address comes from another load.
2691 Value *Addr = LI->getOperand(0);
2692 if (Addr == &CI || isa<LoadInst>(Addr))
2693 return nullptr;
2694 // Don't tranform "load <256 x i32>, <256 x i32>*" to
2695 // "load x86_amx, x86_amx*", because x86_amx* is invalid.
2696 // TODO: Remove this check when bitcast between vector and x86_amx
2697 // is replaced with a specific intrinsic.
2698 if (DestTy->isX86_AMXTy())
2699 return nullptr;
2700 if (LI->hasOneUse() && LI->isSimple())
2701 continue;
2702 // If a LoadInst has more than one use, changing the type of loaded
2703 // value may create another bitcast.
2704 return nullptr;
2705 }
2706
2707 if (auto *PNode = dyn_cast<PHINode>(IncValue)) {
2708 if (OldPhiNodes.insert(PNode))
2709 PhiWorklist.push_back(PNode);
2710 continue;
2711 }
2712
2713 auto *BCI = dyn_cast<BitCastInst>(IncValue);
2714 // We can't handle other instructions.
2715 if (!BCI)
2716 return nullptr;
2717
2718 // Verify it's a A->B cast.
2719 Type *TyA = BCI->getOperand(0)->getType();
2720 Type *TyB = BCI->getType();
2721 if (TyA != DestTy || TyB != SrcTy)
2722 return nullptr;
2723 }
2724 }
2725
2726 // Check that each user of each old PHI node is something that we can
2727 // rewrite, so that all of the old PHI nodes can be cleaned up afterwards.
2728 for (auto *OldPN : OldPhiNodes) {
2729 for (User *V : OldPN->users()) {
2730 if (auto *SI = dyn_cast<StoreInst>(V)) {
2731 if (!SI->isSimple() || SI->getOperand(0) != OldPN)
2732 return nullptr;
2733 } else if (auto *BCI = dyn_cast<BitCastInst>(V)) {
2734 // Verify it's a B->A cast.
2735 Type *TyB = BCI->getOperand(0)->getType();
2736 Type *TyA = BCI->getType();
2737 if (TyA != DestTy || TyB != SrcTy)
2738 return nullptr;
2739 } else if (auto *PHI = dyn_cast<PHINode>(V)) {
2740 // As long as the user is another old PHI node, then even if we don't
2741 // rewrite it, the PHI web we're considering won't have any users
2742 // outside itself, so it'll be dead.
2743 if (!OldPhiNodes.contains(PHI))
2744 return nullptr;
2745 } else {
2746 return nullptr;
2747 }
2748 }
2749 }
2750
2751 // For each old PHI node, create a corresponding new PHI node with a type A.
2752 SmallDenseMap<PHINode *, PHINode *> NewPNodes;
2753 for (auto *OldPN : OldPhiNodes) {
2754 Builder.SetInsertPoint(OldPN);
2755 PHINode *NewPN = Builder.CreatePHI(DestTy, OldPN->getNumOperands());
2756 NewPNodes[OldPN] = NewPN;
2757 }
2758
2759 // Fill in the operands of new PHI nodes.
2760 for (auto *OldPN : OldPhiNodes) {
2761 PHINode *NewPN = NewPNodes[OldPN];
2762 for (unsigned j = 0, e = OldPN->getNumOperands(); j != e; ++j) {
2763 Value *V = OldPN->getOperand(j);
2764 Value *NewV = nullptr;
2765 if (auto *C = dyn_cast<Constant>(V)) {
2766 NewV = ConstantExpr::getBitCast(C, DestTy);
2767 } else if (auto *LI = dyn_cast<LoadInst>(V)) {
2768 // Explicitly perform load combine to make sure no opposing transform
2769 // can remove the bitcast in the meantime and trigger an infinite loop.
2770 Builder.SetInsertPoint(LI);
2771 NewV = combineLoadToNewType(*LI, DestTy);
2772 // Remove the old load and its use in the old phi, which itself becomes
2773 // dead once the whole transform finishes.
2774 replaceInstUsesWith(*LI, PoisonValue::get(LI->getType()));
2776 } else if (auto *BCI = dyn_cast<BitCastInst>(V)) {
2777 NewV = BCI->getOperand(0);
2778 } else if (auto *PrevPN = dyn_cast<PHINode>(V)) {
2779 NewV = NewPNodes[PrevPN];
2780 }
2781 assert(NewV);
2782 NewPN->addIncoming(NewV, OldPN->getIncomingBlock(j));
2783 }
2784 }
2785
2786 // Traverse all accumulated PHI nodes and process its users,
2787 // which are Stores and BitcCasts. Without this processing
2788 // NewPHI nodes could be replicated and could lead to extra
2789 // moves generated after DeSSA.
2790 // If there is a store with type B, change it to type A.
2791
2792
2793 // Replace users of BitCast B->A with NewPHI. These will help
2794 // later to get rid off a closure formed by OldPHI nodes.
2795 Instruction *RetVal = nullptr;
2796 for (auto *OldPN : OldPhiNodes) {
2797 PHINode *NewPN = NewPNodes[OldPN];
2798 for (User *V : make_early_inc_range(OldPN->users())) {
2799 if (auto *SI = dyn_cast<StoreInst>(V)) {
2800 assert(SI->isSimple() && SI->getOperand(0) == OldPN);
2801 Builder.SetInsertPoint(SI);
2802 auto *NewBC =
2803 cast<BitCastInst>(Builder.CreateBitCast(NewPN, SrcTy));
2804 SI->setOperand(0, NewBC);
2805 Worklist.push(SI);
2806 assert(hasStoreUsersOnly(*NewBC));
2807 }
2808 else if (auto *BCI = dyn_cast<BitCastInst>(V)) {
2809 Type *TyB = BCI->getOperand(0)->getType();
2810 Type *TyA = BCI->getType();
2811 assert(TyA == DestTy && TyB == SrcTy);
2812 (void) TyA;
2813 (void) TyB;
2814 Instruction *I = replaceInstUsesWith(*BCI, NewPN);
2815 if (BCI == &CI)
2816 RetVal = I;
2817 } else if (auto *PHI = dyn_cast<PHINode>(V)) {
2818 assert(OldPhiNodes.contains(PHI));
2819 (void) PHI;
2820 } else {
2821 llvm_unreachable("all uses should be handled");
2822 }
2823 }
2824 }
2825
2826 return RetVal;
2827}
2828
2829/// Fold (bitcast (or (and (bitcast X to int), signmask), nneg Y) to fp) to
2830/// copysign((bitcast Y to fp), X)
2832 InstCombiner::BuilderTy &Builder,
2833 const SimplifyQuery &SQ) {
2834 Value *X, *Y;
2835 Type *FTy = CI.getType();
2836 if (!FTy->isFPOrFPVectorTy())
2837 return nullptr;
2840 m_Value(Y)))))
2841 return nullptr;
2842 if (X->getType() != FTy)
2843 return nullptr;
2844 if (!isKnownNonNegative(Y, SQ))
2845 return nullptr;
2846
2847 return Builder.CreateCopySign(Builder.CreateBitCast(Y, FTy), X);
2848}
2849
2851 // If the operands are integer typed then apply the integer transforms,
2852 // otherwise just apply the common ones.
2853 Value *Src = CI.getOperand(0);
2854 Type *SrcTy = Src->getType();
2855 Type *DestTy = CI.getType();
2856
2857 // Get rid of casts from one type to the same type. These are useless and can
2858 // be replaced by the operand.
2859 if (DestTy == Src->getType())
2860 return replaceInstUsesWith(CI, Src);
2861
2862 if (isa<FixedVectorType>(DestTy)) {
2863 if (isa<IntegerType>(SrcTy)) {
2864 // If this is a cast from an integer to vector, check to see if the input
2865 // is a trunc or zext of a bitcast from vector. If so, we can replace all
2866 // the casts with a shuffle and (potentially) a bitcast.
2867 if (isa<TruncInst>(Src) || isa<ZExtInst>(Src)) {
2868 CastInst *SrcCast = cast<CastInst>(Src);
2869 if (BitCastInst *BCIn = dyn_cast<BitCastInst>(SrcCast->getOperand(0)))
2870 if (isa<VectorType>(BCIn->getOperand(0)->getType()))
2872 BCIn->getOperand(0), cast<VectorType>(DestTy), *this))
2873 return I;
2874 }
2875
2876 // If the input is an 'or' instruction, we may be doing shifts and ors to
2877 // assemble the elements of the vector manually. Try to rip the code out
2878 // and replace it with insertelements.
2879 if (Value *V = optimizeIntegerToVectorInsertions(CI, *this))
2880 return replaceInstUsesWith(CI, V);
2881 }
2882 }
2883
2884 if (FixedVectorType *SrcVTy = dyn_cast<FixedVectorType>(SrcTy)) {
2885 if (SrcVTy->getNumElements() == 1) {
2886 // If our destination is not a vector, then make this a straight
2887 // scalar-scalar cast.
2888 if (!DestTy->isVectorTy()) {
2889 Value *Elem =
2890 Builder.CreateExtractElement(Src,
2892 return CastInst::Create(Instruction::BitCast, Elem, DestTy);
2893 }
2894
2895 // Otherwise, see if our source is an insert. If so, then use the scalar
2896 // component directly:
2897 // bitcast (inselt <1 x elt> V, X, 0) to <n x m> --> bitcast X to <n x m>
2898 if (auto *InsElt = dyn_cast<InsertElementInst>(Src))
2899 return new BitCastInst(InsElt->getOperand(1), DestTy);
2900 }
2901
2902 // Convert an artificial vector insert into more analyzable bitwise logic.
2903 unsigned BitWidth = DestTy->getScalarSizeInBits();
2904 Value *X, *Y;
2905 uint64_t IndexC;
2907 m_Value(Y), m_ConstantInt(IndexC)))) &&
2908 DestTy->isIntegerTy() && X->getType() == DestTy &&
2909 Y->getType()->isIntegerTy() && isDesirableIntType(BitWidth)) {
2910 // Adjust for big endian - the LSBs are at the high index.
2911 if (DL.isBigEndian())
2912 IndexC = SrcVTy->getNumElements() - 1 - IndexC;
2913
2914 // We only handle (endian-normalized) insert to index 0. Any other insert
2915 // would require a left-shift, so that is an extra instruction.
2916 if (IndexC == 0) {
2917 // bitcast (inselt (bitcast X), Y, 0) --> or (and X, MaskC), (zext Y)
2918 unsigned EltWidth = Y->getType()->getScalarSizeInBits();
2919 APInt MaskC = APInt::getHighBitsSet(BitWidth, BitWidth - EltWidth);
2920 Value *AndX = Builder.CreateAnd(X, MaskC);
2921 Value *ZextY = Builder.CreateZExt(Y, DestTy);
2922 return BinaryOperator::CreateOr(AndX, ZextY);
2923 }
2924 }
2925 }
2926
2927 if (auto *Shuf = dyn_cast<ShuffleVectorInst>(Src)) {
2928 // Okay, we have (bitcast (shuffle ..)). Check to see if this is
2929 // a bitcast to a vector with the same # elts.
2930 Value *ShufOp0 = Shuf->getOperand(0);
2931 Value *ShufOp1 = Shuf->getOperand(1);
2932 auto ShufElts = cast<VectorType>(Shuf->getType())->getElementCount();
2933 auto SrcVecElts = cast<VectorType>(ShufOp0->getType())->getElementCount();
2934 if (Shuf->hasOneUse() && DestTy->isVectorTy() &&
2935 cast<VectorType>(DestTy)->getElementCount() == ShufElts &&
2936 ShufElts == SrcVecElts) {
2937 BitCastInst *Tmp;
2938 // If either of the operands is a cast from CI.getType(), then
2939 // evaluating the shuffle in the casted destination's type will allow
2940 // us to eliminate at least one cast.
2941 if (((Tmp = dyn_cast<BitCastInst>(ShufOp0)) &&
2942 Tmp->getOperand(0)->getType() == DestTy) ||
2943 ((Tmp = dyn_cast<BitCastInst>(ShufOp1)) &&
2944 Tmp->getOperand(0)->getType() == DestTy)) {
2945 Value *LHS = Builder.CreateBitCast(ShufOp0, DestTy);
2946 Value *RHS = Builder.CreateBitCast(ShufOp1, DestTy);
2947 // Return a new shuffle vector. Use the same element ID's, as we
2948 // know the vector types match #elts.
2949 return new ShuffleVectorInst(LHS, RHS, Shuf->getShuffleMask());
2950 }
2951 }
2952
2953 // A bitcasted-to-scalar and byte/bit reversing shuffle is better recognized
2954 // as a byte/bit swap:
2955 // bitcast <N x i8> (shuf X, undef, <N, N-1,...0>) -> bswap (bitcast X)
2956 // bitcast <N x i1> (shuf X, undef, <N, N-1,...0>) -> bitreverse (bitcast X)
2957 if (DestTy->isIntegerTy() && ShufElts.getKnownMinValue() % 2 == 0 &&
2958 Shuf->hasOneUse() && Shuf->isReverse()) {
2959 unsigned IntrinsicNum = 0;
2960 if (DL.isLegalInteger(DestTy->getScalarSizeInBits()) &&
2961 SrcTy->getScalarSizeInBits() == 8) {
2962 IntrinsicNum = Intrinsic::bswap;
2963 } else if (SrcTy->getScalarSizeInBits() == 1) {
2964 IntrinsicNum = Intrinsic::bitreverse;
2965 }
2966 if (IntrinsicNum != 0) {
2967 assert(ShufOp0->getType() == SrcTy && "Unexpected shuffle mask");
2968 assert(match(ShufOp1, m_Undef()) && "Unexpected shuffle op");
2969 Function *BswapOrBitreverse = Intrinsic::getOrInsertDeclaration(
2970 CI.getModule(), IntrinsicNum, DestTy);
2971 Value *ScalarX = Builder.CreateBitCast(ShufOp0, DestTy);
2972 return CallInst::Create(BswapOrBitreverse, {ScalarX});
2973 }
2974 }
2975 }
2976
2977 // Handle the A->B->A cast, and there is an intervening PHI node.
2978 if (PHINode *PN = dyn_cast<PHINode>(Src))
2979 if (Instruction *I = optimizeBitCastFromPhi(CI, PN))
2980 return I;
2981
2982 if (Instruction *I = canonicalizeBitCastExtElt(CI, *this))
2983 return I;
2984
2986 return I;
2987
2989 return I;
2990
2991 if (Value *V = foldCopySignIdioms(CI, Builder, SQ.getWithInstruction(&CI)))
2992 return replaceInstUsesWith(CI, V);
2993
2994 return commonCastTransforms(CI);
2995}
2996
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static bool isSigned(unsigned int Opcode)
Hexagon Common GEP
static bool collectInsertionElements(Value *V, unsigned Shift, SmallVectorImpl< Value * > &Elements, Type *VecEltTy, bool isBigEndian)
V is a value which is inserted into a vector of VecEltTy.
static bool canEvaluateSExtd(Value *V, Type *Ty)
Return true if we can take the specified value and return it as type Ty without inserting any new cas...
static bool hasStoreUsersOnly(CastInst &CI)
Check if all users of CI are StoreInsts.
static Value * foldCopySignIdioms(BitCastInst &CI, InstCombiner::BuilderTy &Builder, const SimplifyQuery &SQ)
Fold (bitcast (or (and (bitcast X to int), signmask), nneg Y) to fp) to copysign((bitcast Y to fp),...
static Type * shrinkFPConstantVector(Value *V, bool PreferBFloat)
static bool canEvaluateZExtd(Value *V, Type *Ty, unsigned &BitsToClear, InstCombinerImpl &IC, Instruction *CxtI)
Determine if the specified value can be computed in the specified wider type and produce the same low...
static Instruction * canonicalizeBitCastExtElt(BitCastInst &BitCast, InstCombinerImpl &IC)
Canonicalize scalar bitcasts of extracted elements into a bitcast of the vector followed by extract e...
static Instruction * shrinkSplatShuffle(TruncInst &Trunc, InstCombiner::BuilderTy &Builder)
Try to narrow the width of a splat shuffle.
static Instruction * foldFPtoI(Instruction &FI, InstCombiner &IC)
static Instruction * foldBitCastSelect(BitCastInst &BitCast, InstCombiner::BuilderTy &Builder)
Change the type of a select if we can eliminate a bitcast.
static Instruction * foldBitCastBitwiseLogic(BitCastInst &BitCast, InstCombiner::BuilderTy &Builder)
Change the type of a bitwise logic operation if we can eliminate a bitcast.
static bool fitsInFPType(APFloat F, const fltSemantics &Sem)
Return a Constant* for the specified floating-point constant if it fits in the specified FP type with...
static Instruction * optimizeVectorResizeWithIntegerBitCasts(Value *InVal, VectorType *DestTy, InstCombinerImpl &IC)
This input value (which is known to have vector type) is being zero extended or truncated to the spec...
static Instruction * shrinkInsertElt(CastInst &Trunc, InstCombiner::BuilderTy &Builder)
Try to narrow the width of an insert element.
static Type * getMinimumFPType(Value *V, bool PreferBFloat)
Find the minimum FP type we can safely truncate to.
static bool isMultipleOfTypeSize(unsigned Value, Type *Ty)
static Value * optimizeIntegerToVectorInsertions(BitCastInst &CI, InstCombinerImpl &IC)
If the input is an 'or' instruction, we may be doing shifts and ors to assemble the elements of the v...
static bool canAlwaysEvaluateInType(Value *V, Type *Ty)
Constants and extensions/truncates from the destination type are always free to be evaluated in that ...
static Type * shrinkFPConstant(LLVMContext &Ctx, const APFloat &F, bool PreferBFloat)
static Instruction * foldVecExtTruncToExtElt(TruncInst &Trunc, InstCombinerImpl &IC)
Whenever an element is extracted from a vector, optionally shifted down, and then truncated,...
static bool canNotEvaluateInType(Value *V, Type *Ty)
Filter out values that we can not evaluate in the destination type for free.
static bool isKnownExactCastIntToFP(CastInst &I, InstCombinerImpl &IC)
Return true if the cast from integer to FP can be proven to be exact for all possible inputs (the con...
static unsigned getTypeSizeIndex(unsigned Value, Type *Ty)
static Instruction * foldVecTruncToExtElt(TruncInst &Trunc, InstCombinerImpl &IC)
Given a vector that is bitcast to an integer, optionally logically right-shifted, and truncated,...
static bool canEvaluateTruncated(Value *V, Type *Ty, InstCombinerImpl &IC, Instruction *CxtI)
Return true if we can evaluate the specified expression tree as type Ty instead of its larger type,...
This file provides internal interfaces used to implement the InstCombine.
This file provides the interface for the instcombine pass implementation.
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
#define T
uint64_t IntrinsicInst * II
#define P(N)
const SmallVectorImpl< MachineOperand > & Cond
This file implements a set that has insertion order iteration characteristics.
#define LLVM_DEBUG(...)
Definition Debug.h:114
static unsigned getScalarSizeInBits(Type *Ty)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
static const fltSemantics & IEEEsingle()
Definition APFloat.h:296
static const fltSemantics & BFloat()
Definition APFloat.h:295
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:344
static const fltSemantics & IEEEdouble()
Definition APFloat.h:297
static const fltSemantics & IEEEhalf()
Definition APFloat.h:294
static LLVM_ABI unsigned int semanticsIntSizeInBits(const fltSemantics &, bool)
Definition APFloat.cpp:310
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition APInt.cpp:1573
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1540
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:380
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition APInt.cpp:1666
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1111
int32_t exactLogBase2() const
Definition APInt.h:1783
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1639
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:306
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:296
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition APInt.h:286
unsigned countr_one() const
Count the number of trailing one bits.
Definition APInt.h:1656
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition APInt.h:1221
This class represents a conversion between pointers from one address space to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:69
LLVM_ABI std::optional< unsigned > getVScaleRangeMax() const
Returns the maximum value for the vscale_range attribute or std::nullopt when unknown.
BinaryOps getOpcode() const
Definition InstrTypes.h:374
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
static BinaryOperator * CreateFMulFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:244
static BinaryOperator * CreateFDivFMF(Value *V1, Value *V2, FastMathFlags FMF, const Twine &Name="")
Definition InstrTypes.h:248
This class represents a no-op cast from one type to another.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This is the base class for all instructions that perform data casts.
Definition InstrTypes.h:448
Type * getSrcTy() const
Return the source type, as a convenience.
Definition InstrTypes.h:615
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition InstrTypes.h:610
static LLVM_ABI unsigned isEliminableCastPair(Instruction::CastOps firstOpcode, Instruction::CastOps secondOpcode, Type *SrcTy, Type *MidTy, Type *DstTy, const DataLayout *DL)
Determine how a pair of casts can be eliminated, if they can be at all.
static LLVM_ABI CastInst * CreateIntegerCast(Value *S, Type *Ty, bool isSigned, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a ZExt, BitCast, or Trunc for int -> int casts.
static LLVM_ABI CastInst * CreateFPCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create an FPExt, BitCast, or FPTrunc for fp -> fp casts.
static LLVM_ABI CastInst * CreateTruncOrBitCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a Trunc or BitCast cast instruction.
static LLVM_ABI CastInst * CreateBitOrPointerCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Type * getDestTy() const
Return the destination type, as a convenience.
Definition InstrTypes.h:617
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:676
@ ICMP_SLT
signed less than
Definition InstrTypes.h:705
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:700
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:701
@ ICMP_NE
not equal
Definition InstrTypes.h:698
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:702
static LLVM_ABI Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static LLVM_ABI Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_ABI Constant * getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
ConstantFP - Floating Point Values [float, double].
Definition Constants.h:277
const APFloat & getValueAPF() const
Definition Constants.h:320
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:163
This is an important base class in LLVM.
Definition Constant.h:43
static LLVM_ABI Constant * mergeUndefsWith(Constant *C, Constant *Other)
Merges undefs of a Constant with another Constant, along with the undefs already present.
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
LLVM_ABI bool isElementWiseEqual(Value *Y) const
Return true if this constant and a constant 'Y' are element-wise equal.
bool isBigEndian() const
Definition DataLayout.h:208
static ExtractElementInst * Create(Value *Vec, Value *Idx, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This class represents an extension of floating point types.
This class represents a cast from floating point to signed integer.
This class represents a cast from floating point to unsigned integer.
This class represents a truncation of floating point types.
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:22
Class to represent fixed width SIMD vectors.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:803
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:209
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:762
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:727
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
This instruction compares its operands according to the predicate given to the constructor.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition IRBuilder.h:2579
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition IRBuilder.h:522
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition IRBuilder.h:2207
static InsertElementInst * Create(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Instruction * visitZExt(ZExtInst &Zext)
Instruction * visitAddrSpaceCast(AddrSpaceCastInst &CI)
Instruction * visitSExt(SExtInst &Sext)
Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)
Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...
Instruction * visitFPToSI(FPToSIInst &FI)
Instruction * visitTrunc(TruncInst &CI)
Instruction * visitUIToFP(CastInst &CI)
Instruction * visitPtrToInt(PtrToIntInst &CI)
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false, bool SimplifyBothArms=false)
Given an instruction with a select as one operand and a constant as the other operand,...
Instruction * visitSIToFP(CastInst &CI)
Instruction * commonCastTransforms(CastInst &CI)
Implement the transforms common to all CastInst visitors.
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * foldItoFPtoI(CastInst &FI)
fpto{s/u}i({u/s}itofp(X)) --> X or zext(X) or sext(X) or trunc(X) This is safe if the intermediate ty...
Instruction * visitFPTrunc(FPTruncInst &CI)
Value * foldPtrToIntOrAddrOfGEP(Type *IntTy, Value *Ptr)
Instruction * visitBitCast(BitCastInst &CI)
Instruction * visitIntToPtr(IntToPtrInst &CI)
Instruction * visitFPToUI(FPToUIInst &FI)
Instruction * visitPtrToAddr(PtrToAddrInst &CI)
Value * EvaluateInDifferentType(Value *V, Type *Ty, bool isSigned)
Given an expression that CanEvaluateTruncated or CanEvaluateSExtd returns true for,...
bool SimplifyDemandedInstructionBits(Instruction &Inst)
Tries to simplify operands to an integer instruction based on its demanded bits.
Instruction * visitFPExt(CastInst &CI)
LoadInst * combineLoadToNewType(LoadInst &LI, Type *NewTy, const Twine &Suffix="")
Helper to combine a load to a new type.
The core instruction combiner logic.
SimplifyQuery SQ
const DataLayout & getDataLayout() const
unsigned ComputeMaxSignificantBits(const Value *Op, const Instruction *CxtI=nullptr, unsigned Depth=0) const
IRBuilder< TargetFolder, IRBuilderCallbackInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
unsigned ComputeNumSignBits(const Value *Op, const Instruction *CxtI=nullptr, unsigned Depth=0) const
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
const DataLayout & DL
void computeKnownBits(const Value *V, KnownBits &Known, const Instruction *CxtI, unsigned Depth=0) const
bool MaskedValueIsZero(const Value *V, const APInt &Mask, const Instruction *CxtI=nullptr, unsigned Depth=0) const
DominatorTree & DT
BuilderTy & Builder
const SimplifyQuery & getSimplifyQuery() const
LLVM_ABI void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
static bool isBitwiseLogicOp(unsigned Opcode)
Determine if the Opcode is and/or/xor.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI void setNonNeg(bool b=true)
Set or clear the nneg flag on this instruction, which must be a zext instruction.
LLVM_ABI bool hasNonNeg() const LLVM_READONLY
Determine whether the the nneg flag is set.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
LLVM_ABI void setIsExact(bool b=true)
Set or clear the exact flag on this instruction, which must be an operator which supports this flag.
This class represents a cast from an integer to a pointer.
unsigned getAddressSpace() const
Returns the address space of this instruction's pointer type.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:319
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
This class represents a cast from a pointer to an address (non-capturing ptrtoint).
Value * getPointerOperand()
Gets the pointer operand.
This class represents a cast from a pointer to an integer.
Value * getPointerOperand()
Gets the pointer operand.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:150
This instruction constructs a fixed permutation of two input vectors.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class represents a truncation of integer types.
void setHasNoSignedWrap(bool B)
void setHasNoUnsignedWrap(bool B)
bool hasNoSignedWrap() const
Test whether this operation is known to never undergo signed overflow, aka the nsw property.
bool hasNoUnsignedWrap() const
Test whether this operation is known to never undergo unsigned overflow, aka the nuw property.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:273
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:297
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition Type.h:246
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:352
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:198
LLVM_ABI Type * getWithNewType(Type *EltTy) const
Given vector type, change the element type, whilst keeping the old number of elements.
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:231
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:270
bool isX86_AMXTy() const
Return true if this is X86 AMX.
Definition Type.h:200
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
Definition Type.cpp:286
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:225
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:285
LLVM_ABI int getFPMantissaWidth() const
Return the width of the mantissa of this type.
Definition Type.cpp:236
LLVM_ABI const fltSemantics & getFltSemantics() const
Definition Type.cpp:107
static LLVM_ABI Type * getBFloatTy(LLVMContext &C)
Definition Type.cpp:284
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
Definition Type.cpp:283
'undef' values are things that do not have specified contents.
Definition Constants.h:1420
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Value * getOperand(unsigned i) const
Definition User.h:232
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:439
iterator_range< user_iterator > users()
Definition Value.h:426
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1099
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:396
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
static LLVM_ABI bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
This class represents zero extension of integer types.
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:238
Changed
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
cst_pred_ty< is_lowbit_mask > m_LowBitMask()
Match an integer or vector with only the low bit(s) set.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
PtrToIntSameSize_match< OpTy > m_PtrToIntSameSize(const DataLayout &DL, const OpTy &Op)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
cst_pred_ty< is_sign_mask > m_SignMask()
Match an integer or vector with only the sign bit(s) set.
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
cst_pred_ty< is_power2 > m_Power2()
Match an integer or vector power-of-2.
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
BinOpPred_match< LHS, RHS, is_right_shift_op > m_Shr(const LHS &L, const RHS &R)
Matches logical shift operations.
specific_intval< true > m_SpecificIntAllowPoison(const APInt &V)
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
IntrinsicID_match m_Intrinsic()
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
IntrinsicID_match m_VScale()
Matches a call to llvm.vscale().
BinOpPred_match< LHS, RHS, is_logical_shift_op > m_LogicalShift(const LHS &L, const RHS &R)
Matches logical shift operations.
CastInst_match< OpTy, FPToUIInst > m_FPToUI(const OpTy &Op)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
cst_pred_ty< is_negated_power2 > m_NegatedPower2()
Match a integer or vector negated power-of-2.
match_immconstant_ty m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
CastInst_match< OpTy, FPToSIInst > m_FPToSI(const OpTy &Op)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
BinOpPred_match< LHS, RHS, is_shift_op > m_Shift(const LHS &L, const RHS &R)
Matches shift operations.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
auto m_Undef()
Match an arbitrary undef constant.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
CastOperator_match< OpTy, Instruction::IntToPtr > m_IntToPtr(const OpTy &Op)
Matches IntToPtr.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
ElementWiseBitCast_match< OpTy > m_ElementWiseBitCast(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
cst_pred_ty< icmp_pred_with_threshold > m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold)
Match an integer or vector with every element comparing 'pred' (eg/ne/...) to Threshold.
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:477
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, const SimplifyQuery &SQ, unsigned Depth=0)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1725
LLVM_ABI Constant * ConstantFoldSelectInstruction(Constant *Cond, Constant *V1, Constant *V2)
Attempt to constant fold a select instruction with the specified operands.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
Definition MathExtras.h:350
LLVM_ABI Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:632
LLVM_ABI Value * simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty, const SimplifyQuery &Q)
Given operands for a CastInst, fold the result or return null.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:331
auto reverse(ContainerTy &&C)
Definition STLExtras.h:406
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:279
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
LLVM_ABI Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI bool replaceAllDbgUsesWith(Instruction &From, Value &To, Instruction &DomPoint, DominatorTree &DT)
Point debug users of From to To or salvage them.
Definition Local.cpp:2414
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2108
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:305
LLVM_ABI Constant * ConstantFoldIntegerCast(Constant *C, Type *DestTy, bool IsSigned, const DataLayout &DL)
Constant fold a zext, sext or trunc, depending on IsSigned and whether the DestTy is wider or narrowe...
LLVM_ABI bool isKnownNonNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the give value is known to be non-negative.
LLVM_ABI Constant * ConstantFoldBinaryInstruction(unsigned Opcode, Constant *V1, Constant *V2)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:869
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition KnownBits.h:242
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition KnownBits.h:248
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition KnownBits.h:145
bool isKnownNever(FPClassTest Mask) const
Return true if it's known this can never be one of the mask entries.
Matching combinators.
SimplifyQuery getWithInstruction(const Instruction *I) const