LLVM 24.0.0git
WebAssemblyTargetTransformInfo.cpp
Go to the documentation of this file.
1//===-- WebAssemblyTargetTransformInfo.cpp - WebAssembly-specific TTI -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file defines the WebAssembly-specific TargetTransformInfo
11/// implementation.
12///
13//===----------------------------------------------------------------------===//
14
17#include "llvm/IR/IntrinsicsWebAssembly.h"
19
21using namespace llvm;
22
23#define DEBUG_TYPE "wasmtti"
24
26WebAssemblyTTIImpl::getPopcntSupport(unsigned TyWidth) const {
27 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
29}
30
31unsigned WebAssemblyTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
32 unsigned Result = BaseT::getNumberOfRegisters(ClassID);
33
34 // For SIMD, use at least 16 registers, as a rough guess.
35 bool Vector = (ClassID == 1);
36 if (Vector)
37 Result = std::max(Result, 16u);
38
39 return Result;
40}
41
44 switch (K) {
46 return TypeSize::getFixed(64);
48 return TypeSize::getFixed(getST()->hasSIMD128() ? 128 : 64);
50 return TypeSize::getScalable(0);
51 }
52
53 llvm_unreachable("Unsupported register kind");
54}
55
57 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
59 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
60
61 if (ST->hasSIMD128()) {
62 static const CostTblEntry ArithCostTbl[]{
63 // extmul + (maybe awkward) shuffle
64 {ISD::MUL, MVT::v8i8, 4},
65 // 2x extmul + (okay) shuffle
66 {ISD::MUL, MVT::v16i8, 4},
67 // extmul
68 {ISD::MUL, MVT::v4i16, 1},
69 // extmul
70 {ISD::MUL, MVT::v2i32, 1},
71 };
72 EVT DstVT = TLI->getValueType(DL, Ty);
73 if (DstVT.isSimple()) {
74 int ISD = TLI->InstructionOpcodeToISD(Opcode);
75 if (const auto *Entry =
76 CostTableLookup(ArithCostTbl, ISD, DstVT.getSimpleVT()))
77 return Entry->Cost;
78 }
79 }
80
83 Opcode, Ty, CostKind, Op1Info, Op2Info);
84
85 if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
86 switch (Opcode) {
87 case Instruction::LShr:
88 case Instruction::AShr:
89 case Instruction::Shl:
90 // SIMD128's shifts currently only accept a scalar shift count. For each
91 // element, we'll need to extract, op, insert. The following is a rough
92 // approximation.
93 if (!Op2Info.isUniform())
94 Cost =
95 VTy->getNumElements() *
97 getArithmeticInstrCost(Opcode, VTy->getElementType(), CostKind) +
99 break;
100 }
101 }
102 return Cost;
103}
104
106 unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH,
107 TTI::TargetCostKind CostKind, const Instruction *I) const {
108 int ISD = TLI->InstructionOpcodeToISD(Opcode);
109 auto SrcTy = TLI->getValueType(DL, Src);
110 auto DstTy = TLI->getValueType(DL, Dst);
111
112 if (!SrcTy.isSimple() || !DstTy.isSimple()) {
113 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
114 }
115
116 if (!ST->hasSIMD128()) {
117 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
118 }
119
120 auto DstVT = DstTy.getSimpleVT();
121 auto SrcVT = SrcTy.getSimpleVT();
122
123 if (I && I->hasOneUser()) {
124 auto *SingleUser = cast<Instruction>(*I->user_begin());
125 int UserISD = TLI->InstructionOpcodeToISD(SingleUser->getOpcode());
126
127 // extmul_low support
128 if (UserISD == ISD::MUL &&
130 // Free low extensions.
131 if ((SrcVT == MVT::v8i8 && DstVT == MVT::v8i16) ||
132 (SrcVT == MVT::v4i16 && DstVT == MVT::v4i32) ||
133 (SrcVT == MVT::v2i32 && DstVT == MVT::v2i64)) {
134 return 0;
135 }
136 // Will require an additional extlow operation for the intermediate
137 // i16/i32 value.
138 if ((SrcVT == MVT::v4i8 && DstVT == MVT::v4i32) ||
139 (SrcVT == MVT::v2i16 && DstVT == MVT::v2i64)) {
140 return 1;
141 }
142 }
143 }
144
145 static constexpr TypeConversionCostTblEntry ConversionTbl[] = {
146 // extend_low
147 {ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i32, 1},
148 {ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i32, 1},
149 {ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i16, 1},
150 {ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i16, 1},
151 {ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i8, 1},
152 {ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i8, 1},
153 // 2 x extend_low
154 {ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i16, 2},
155 {ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i16, 2},
156 {ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i8, 2},
157 {ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i8, 2},
158 // extend_low, extend_high
159 {ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 2},
160 {ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 2},
161 {ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 2},
162 {ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 2},
163 {ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 2},
164 {ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 2},
165 // 2x extend_low, extend_high
166 {ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i32, 4},
167 {ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i32, 4},
168 {ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i16, 4},
169 {ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i16, 4},
170 // 6x extend_low, extend_high
171 {ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 6},
172 {ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 6},
173 // shuffle
174 {ISD::TRUNCATE, MVT::v2i16, MVT::v2i32, 2},
175 {ISD::TRUNCATE, MVT::v2i8, MVT::v2i32, 4},
176 // narrow, and
177 {ISD::TRUNCATE, MVT::v4i16, MVT::v4i32, 2},
178 // 2x narrow, and
179 {ISD::TRUNCATE, MVT::v4i8, MVT::v4i32, 3},
180 // narrow, and
181 {ISD::TRUNCATE, MVT::v4i8, MVT::v4i16, 2},
182 {ISD::TRUNCATE, MVT::v8i8, MVT::v8i16, 2},
183 // narrow, 2x and
184 {ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, 3},
185 {ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, 3},
186 // 2x narrow, 2x and
187 {ISD::TRUNCATE, MVT::v4i16, MVT::v4i64, 4},
188 {ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, 4},
189 // 3x narrow, 2x and
190 {ISD::TRUNCATE, MVT::v4i8, MVT::v4i64, 5},
191 // 2x narrow, 4x and
192 {ISD::TRUNCATE, MVT::v16i16, MVT::v16i32, 6},
193 // 3x narrow, 4x and
194 {ISD::TRUNCATE, MVT::v8i16, MVT::v8i64, 7},
195 {ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, 7},
196 // 4x narrow, 4x and
197 {ISD::TRUNCATE, MVT::v8i8, MVT::v8i64, 8},
198 // 6x narrow, 8x and
199 {ISD::TRUNCATE, MVT::v16i16, MVT::v16i64, 14},
200 // 7x narrow, 8x and
201 {ISD::TRUNCATE, MVT::v16i8, MVT::v16i64, 15},
202 // convert_i32x4
203 {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i32, 1},
204 {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, 1},
205 {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 1},
206 {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1},
207 // extend_low, convert
208 {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 2},
209 {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 2},
210 {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i16, 2},
211 {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2},
212 // extend_low x 2, convert
213 {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8, 3},
214 {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8, 3},
215 {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8, 3},
216 {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8, 3},
217 // several shuffles
218 {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i8, 10},
219 {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i8, 10},
220 {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 10},
221 {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, 10},
222 // trunc_sat, const, and, 2x narrow
223 {ISD::FP_TO_SINT, MVT::v2i8, MVT::v2f32, 5},
224 {ISD::FP_TO_UINT, MVT::v2i8, MVT::v2f32, 5},
225 // trunc_sat, const, and, 2x narrow
226 {ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f32, 5},
227 {ISD::FP_TO_UINT, MVT::v4i8, MVT::v4f32, 5},
228 // trunc_sat, const, and, narrow
229 {ISD::FP_TO_UINT, MVT::v2i16, MVT::v2f32, 4},
230 {ISD::FP_TO_SINT, MVT::v2i16, MVT::v2f32, 4},
231 {ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f32, 4},
232 {ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f32, 4},
233 // 2x trunc_sat, const, 2x and, 2x narrow
234 {ISD::FP_TO_SINT, MVT::v8i8, MVT::v8f32, 7},
235 {ISD::FP_TO_UINT, MVT::v8i8, MVT::v8f32, 7},
236 // 2x trunc_sat, const, 2x and, narrow
237 {ISD::FP_TO_SINT, MVT::v8i16, MVT::v8f32, 6},
238 {ISD::FP_TO_UINT, MVT::v8i16, MVT::v8f32, 6},
239 };
240
241 if (const auto *Entry =
242 ConvertCostTableLookup(ConversionTbl, ISD, DstVT, SrcVT)) {
243 return Entry->Cost;
244 }
245
246 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
247}
248
250WebAssemblyTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
252
253 Options.AllowOverlappingLoads = true;
254
255 if (ST->hasSIMD128())
256 Options.LoadSizes.push_back(16);
257
258 Options.LoadSizes.append({8, 4, 2, 1});
259 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
260 Options.NumLoadsPerBlock = IsZeroCmp ? Options.MaxNumLoads : 1;
261
262 return Options;
263}
264
266 unsigned Opcode, Type *Ty, Align Alignment, unsigned AddressSpace,
268 const Instruction *I) const {
269 // FIXME: Load latency isn't handled here
270 if (!ST->hasSIMD128() || !isa<FixedVectorType>(Ty) ||
271 (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency)) {
272 return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace,
273 CostKind);
274 }
275
276 EVT VT = TLI->getValueType(DL, Ty, true);
277 // Type legalization can't handle structs
278 if (VT == MVT::Other)
279 return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace,
280 CostKind);
281
282 auto LT = getTypeLegalizationCost(Ty);
283 if (!LT.first.isValid())
285
286 int ISD = TLI->InstructionOpcodeToISD(Opcode);
287 unsigned width = VT.getSizeInBits();
288 if (ISD == ISD::LOAD) {
289 // 128-bit loads are a single instruction. 32-bit and 64-bit vector loads
290 // can be lowered to load32_zero and load64_zero respectively. Assume SIMD
291 // loads are twice as expensive as scalar.
292 switch (width) {
293 default:
294 break;
295 case 32:
296 case 64:
297 case 128:
298 return 2;
299 }
300 } else if (ISD == ISD::STORE) {
301 // For stores, we can use store lane operations.
302 switch (width) {
303 default:
304 break;
305 case 8:
306 case 16:
307 case 32:
308 case 64:
309 case 128:
310 return 2;
311 }
312 }
313
314 return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace, CostKind);
315}
316
318 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
319 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
320 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
321 TTI::VectorInstrContext VIC) const {
322 // Canonicalize the ShuffleKind in case optimizations didn't.
323 // Otherwise, we might end up with the wrong ShuffleKind to match against.
324
325 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);
326
327 // Wasm SIMD128 has native splat instructions for all lane types.
328 if (ST->hasSIMD128() && Kind == TTI::SK_Broadcast &&
330 return 1;
331
332 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index, SubTp,
333 Args, CtxI);
334}
335
337 unsigned Opcode, Type *Ty, unsigned Factor, ArrayRef<unsigned> Indices,
338 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
339 bool UseMaskForCond, bool UseMaskForGaps) const {
340 assert(Factor >= 2 && "Invalid interleave factor");
341
342 auto *VecTy = cast<VectorType>(Ty);
343 if (!ST->hasSIMD128() || !isa<FixedVectorType>(VecTy)) {
345 }
346
347 if (UseMaskForCond || UseMaskForGaps)
348 return BaseT::getInterleavedMemoryOpCost(Opcode, Ty, Factor, Indices,
349 Alignment, AddressSpace, CostKind,
350 UseMaskForCond, UseMaskForGaps);
351
352 constexpr unsigned MaxInterleaveFactor = 4;
353 if (Factor <= MaxInterleaveFactor) {
354 unsigned MinElts = VecTy->getElementCount().getKnownMinValue();
355 // Ensure the number of vector elements is greater than 1.
356 if (MinElts < 2 || MinElts % Factor != 0)
358
359 unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
360 // Ensure the element type is legal.
361 if (ElSize != 8 && ElSize != 16 && ElSize != 32 && ElSize != 64)
363
364 if (Factor != 2 && Factor != 4)
366
367 auto *SubVecTy =
368 VectorType::get(VecTy->getElementType(),
369 VecTy->getElementCount().divideCoefficientBy(Factor));
370 InstructionCost MemCost =
371 getMemoryOpCost(Opcode, SubVecTy, Alignment, AddressSpace, CostKind);
372
373 unsigned VecSize = DL.getTypeSizeInBits(SubVecTy);
374 unsigned MaxVecSize = 128;
375 unsigned NumAccesses =
376 std::max<unsigned>(1, (MinElts * ElSize + MaxVecSize - 1) / VecSize);
377
378 // A stride of two is commonly supported via dedicated instructions, so it
379 // should be relatively cheap for all element sizes. A stride of four is
380 // more expensive as it will likely require more shuffles. Using two
381 // simd128 inputs is considered more expensive and we mainly account for
382 // shuffling two inputs (32 bytes), but we do model 4 x v4i32 to enable
383 // arithmetic kernels with smaller (i8/i16) inputs.
384 static const CostTblEntry ShuffleCostTbl[] = {
385 // One reg.
386 {2, MVT::v2i8, 1}, // interleave 2 x 2i8 into 4i8
387 {2, MVT::v4i8, 1}, // interleave 2 x 4i8 into 8i8
388 {2, MVT::v8i8, 1}, // interleave 2 x 8i8 into 16i8
389 {2, MVT::v2i16, 1}, // interleave 2 x 2i16 into 4i16
390 {2, MVT::v4i16, 1}, // interleave 2 x 4i16 into 8i16
391 {2, MVT::v2i32, 1}, // interleave 2 x 2i32 into 4i32
392 {2, MVT::v2f32, 1}, // interleave 2 x 2f32 into 4f32
393
394 // Two regs.
395 {2, MVT::v16i8, 2}, // interleave 2 x 16i8 into 32i8
396 {2, MVT::v8i16, 2}, // interleave 2 x 8i16 into 16i16
397 {2, MVT::v4i32, 2}, // interleave 2 x 4i32 into 8i32
398 {2, MVT::v4f32, 2}, // interleave 2 x 4f32 into 8f32
399
400 // One reg.
401 {4, MVT::v2i8, 4}, // interleave 4 x 2i8 into 8i8
402 {4, MVT::v4i8, 4}, // interleave 4 x 4i8 into 16i8
403 {4, MVT::v2i16, 4}, // interleave 4 x 2i16 into 8i16
404
405 // Two regs.
406 {4, MVT::v8i8, 16}, // interleave 4 x 8i8 into 32i8
407 {4, MVT::v4i16, 8}, // interleave 4 x 4i16 into 16i16
408 {4, MVT::v2i32, 4}, // interleave 4 x 2i32 into 8i32
409 {4, MVT::v2f32, 4}, // interleave 4 x 2f32 into 8f32
410
411 // Four regs.
412 {4, MVT::v4i32, 16}, // interleave 4 x 4i32 into 16i32
413 };
414
415 EVT ETy = TLI->getValueType(DL, SubVecTy);
416 if (const auto *Entry =
417 CostTableLookup(ShuffleCostTbl, Factor, ETy.getSimpleVT()))
418 return Entry->Cost + (NumAccesses * MemCost);
419 }
420
421 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
422 Alignment, AddressSpace, CostKind,
423 UseMaskForCond, UseMaskForGaps);
424}
425
427 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
428 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
430 Opcode, Val, CostKind, Index, Op0, Op1, VIC);
431
432 // SIMD128's insert/extract currently only take constant indices.
433 if (Index == -1u)
435
436 return Cost;
437}
438
440 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
442 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
443 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
445 if (!VF.isFixed() || !ST->hasSIMD128())
446 return Invalid;
447
448 if (CostKind != TTI::TCK_RecipThroughput)
449 return Invalid;
450
451 if (Opcode != Instruction::Add)
452 return Invalid;
453
454 EVT AccumEVT = EVT::getEVT(AccumType);
455 // TODO: Add i64 accumulator.
456 if (AccumEVT != MVT::i32)
457 return Invalid;
458
459 // Possible options:
460 // - i16x8.extadd_pairwise_i8x16_sx
461 // - i32x4.extadd_pairwise_i16x8_sx
462 // - i32x4.dot_i16x8_s
463 // Only try to support dot, for now.
464
465 EVT InputEVT = EVT::getEVT(InputTypeA);
466 if (!((InputEVT == MVT::i16 && VF.getFixedValue() == 8) ||
467 (InputEVT == MVT::i8 && VF.getFixedValue() == 16))) {
468 return Invalid;
469 }
470
471 if (OpAExtend == TTI::PR_None)
472 return Invalid;
473
475 if (!BinOp)
476 return Cost;
477
478 if (OpAExtend != OpBExtend)
479 return Invalid;
480
481 if (*BinOp != Instruction::Mul)
482 return Invalid;
483
484 if (InputTypeA != InputTypeB)
485 return Invalid;
486
487 // Signed inputs can lower to dot
488 if (InputEVT == MVT::i16 && VF.getFixedValue() == 8)
489 return OpAExtend == TTI::PR_SignExtend ? Cost : Cost * 2;
490
491 // Double the size of the lowered sequence.
492 if (InputEVT == MVT::i8 && VF.getFixedValue() == 16)
493 return OpAExtend == TTI::PR_SignExtend ? Cost * 2 : Cost * 4;
494
495 return Invalid;
496}
497
499 const IntrinsicInst *II) const {
500
501 switch (II->getIntrinsicID()) {
502 default:
503 break;
504 case Intrinsic::vector_reduce_fadd:
506 }
508}
509
512 OptimizationRemarkEmitter *ORE) const {
513 // Scan the loop: don't unroll loops with calls. This is a standard approach
514 // for most (all?) targets.
515 for (BasicBlock *BB : L->blocks())
516 for (Instruction &I : *BB)
519 if (isLoweredToCall(F))
520 return;
521
522 // The chosen threshold is within the range of 'LoopMicroOpBufferSize' of
523 // the various microarchitectures that use the BasicTTI implementation and
524 // has been selected through heuristics across multiple cores and runtimes.
525 UP.Partial = UP.Runtime = UP.UpperBound = true;
526 UP.PartialThreshold = 30;
527
528 // Avoid unrolling when optimizing for size.
529 UP.OptSizeThreshold = 0;
531
532 // Set number of instructions optimized when "back edge"
533 // becomes "fall through" to default value of 2.
534 UP.BEInsns = 2;
535}
536
538 return getST()->hasTailCall();
539}
540
543 using namespace llvm::PatternMatch;
544
545 if (!I->getType()->isVectorTy() || !I->isShift())
546 return false;
547
548 Value *V = I->getOperand(1);
549 // We dont need to sink constant splat.
550 if (isa<Constant>(V))
551 return false;
552
554 m_Value(), m_ZeroMask()))) {
555 // Sink insert
556 Ops.push_back(&cast<Instruction>(V)->getOperandUse(0));
557 // Sink shuffle
558 Ops.push_back(&I->getOperandUse(1));
559 return true;
560 }
561
562 return false;
563}
564
565/// Attempt to convert [relaxed_]swizzle to shufflevector if the mask is
566/// constant.
569 bool IsRelaxed) {
570 auto *V = dyn_cast<Constant>(II.getArgOperand(1));
571 if (!V)
572 return nullptr;
573
574 auto *VecTy = cast<FixedVectorType>(II.getType());
575 unsigned NumElts = VecTy->getNumElements();
576 assert(NumElts == 16);
577
578 // Construct a shuffle mask from constant integers or UNDEFs.
579 int Indexes[16];
580 bool AnyOutOfBounds = false;
581
582 for (unsigned I = 0; I < NumElts; ++I) {
583 Constant *COp = V->getAggregateElement(I);
584 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
585 return nullptr;
586
587 if (isa<UndefValue>(COp)) {
588 Indexes[I] = -1;
589 continue;
590 }
591
592 if (IsRelaxed && cast<ConstantInt>(COp)->getSExtValue() >= NumElts) {
593 // The relaxed_swizzle operation always returns 0 if the lane index is
594 // less than 0 when interpreted as a signed value. For lane indices above
595 // 15, however, it can choose between returning 0 or the lane at `Index %
596 // 16`. However, the choice must be made consistently. As the WebAssembly
597 // spec states:
598 //
599 // "The result of relaxed operators are implementation-dependent, because
600 // the set of possible results may depend on properties of the host
601 // environment, such as its hardware. Technically, their behaviour is
602 // controlled by a set of global parameters to the semantics that an
603 // implementation can instantiate in different ways. These choices are
604 // fixed, that is, parameters are constant during the execution of any
605 // given program."
606 //
607 // The WebAssembly runtime may choose differently from us, so we can't
608 // optimize a relaxed swizzle with lane indices above 15.
609 return nullptr;
610 }
611
612 uint64_t Index = cast<ConstantInt>(COp)->getZExtValue();
613 if (Index >= NumElts) {
614 AnyOutOfBounds = true;
615 // If there are out-of-bounds indices, the swizzle instruction returns
616 // zeroes in those lanes. We'll provide an all-zeroes vector as the
617 // second argument to shufflevector and read the first element from it.
618 Indexes[I] = NumElts;
619 continue;
620 }
621
622 Indexes[I] = Index;
623 }
624
625 auto *V1 = II.getArgOperand(0);
626 auto *V2 =
627 AnyOutOfBounds ? Constant::getNullValue(VecTy) : PoisonValue::get(VecTy);
628
629 return Builder.CreateShuffleVector(V1, V2, ArrayRef(Indexes, NumElts));
630}
631
632std::optional<Instruction *>
634 IntrinsicInst &II) const {
635 Intrinsic::ID IID = II.getIntrinsicID();
636 switch (IID) {
637 case Intrinsic::wasm_swizzle:
638 case Intrinsic::wasm_relaxed_swizzle:
639 if (Value *V = simplifyWasmSwizzle(
640 II, IC.Builder, IID == Intrinsic::wasm_relaxed_swizzle)) {
641 return IC.replaceInstUsesWith(II, V);
642 }
643 break;
644 }
645
646 return std::nullopt;
647}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
Cost tables and simple lookup functions.
static const int MaxVecSize
This file provides the interface for the instcombine pass implementation.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static LVOptions Options
Definition LVOptions.cpp:25
static const unsigned MaxInterleaveFactor
Maximum vectorization interleave count.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
static const Function * getCalledFunction(const Value *V)
uint64_t IntrinsicInst * II
static Value * simplifyWasmSwizzle(const IntrinsicInst &II, InstCombiner::BuilderTy &Builder, bool IsRelaxed)
Attempt to convert [relaxed_]swizzle to shufflevector if the mask is constant.
This file a TargetTransformInfoImplBase conforming object specific to the WebAssembly target machine.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
LLVM Basic Block Representation.
Definition BasicBlock.h:62
InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond=false, bool UseMaskForGaps=false) const override
TTI::ShuffleKind improveShuffleKindFromMask(TTI::ShuffleKind Kind, ArrayRef< int > Mask, VectorType *SrcTy, int &Index, VectorType *&SubTy) const
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Opd1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Opd2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr) const override
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef< int > Mask, int Index, VectorType *SubTp, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const override
std::pair< InstructionCost, MVT > getTypeLegalizationCost(Type *Ty) const
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
This is an important base class in LLVM.
Definition Constant.h:44
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
The core instruction combiner logic.
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
IRBuilder< TargetFolder, IRBuilderInstCombineInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
static InstructionCost getInvalid(CostType Val=0)
A wrapper class for inspecting calls to intrinsic functions.
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
The optimization diagnostic interface.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
The main scalar evolution driver.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
virtual unsigned getNumberOfRegisters(unsigned ClassID) const
virtual bool isLoweredToCall(const Function *F) const
TargetCostKind
The kind of cost model.
@ TCK_RecipThroughput
Reciprocal throughput.
@ TCK_Latency
The latency of instruction.
PopcntSupportKind
Flags indicating the kind of support for population count.
llvm::VectorInstrContext VectorInstrContext
@ TCC_Expensive
The cost of a 'div' instruction on x86.
@ TCC_Basic
The cost of a typical 'add' instruction.
ShuffleKind
The various kinds of shuffle patterns for vector queries.
@ SK_Broadcast
Broadcast element 0 to all other elements.
CastContextHint
Represents a hint about the context in which a cast is used.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Definition TypeSize.h:355
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM Value Representation.
Definition Value.h:75
Base class of all SIMD vector types.
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *Ty, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond, bool UseMaskForGaps) const override
InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, TTI::PartialReductionExtendKind OpAExtend, TTI::PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp, TTI::TargetCostKind CostKind, std::optional< FastMathFlags > FMF) const override
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef< int > Mask, int Index, VectorType *SubTp, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Op2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr) const override
std::optional< Instruction * > instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override
TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override
bool isProfitableToSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
TTI::ReductionShuffle getPreferredExpandedReductionShuffle(const IntrinsicInst *II) const override
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, OptimizationRemarkEmitter *ORE) const override
TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const override
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const override
unsigned getNumberOfRegisters(unsigned ClassID) const override
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:172
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
ISD namespace - This namespace contains an enum which represents all of the SelectionDAG node types a...
Definition ISDOpcodes.h:26
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
bool match(Val *V, const Pattern &P)
auto m_Value()
Match an arbitrary value and ignore it.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
This is an optimization pass for GlobalISel generic memory operations.
const CostTblEntryT< CostType > * CostTableLookup(ArrayRef< CostTblEntryT< CostType > > Tbl, int ISD, MVT Ty)
Find in cost table.
Definition CostTable.h:36
InstructionCost Cost
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
ArrayRef(const T &OneElt) -> ArrayRef< T >
TypeConversionCostTblEntryT< uint16_t > TypeConversionCostTblEntry
Definition CostTable.h:62
CostTblEntryT< uint16_t > CostTblEntry
Definition CostTable.h:31
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
const TypeConversionCostTblEntryT< CostType > * ConvertCostTableLookup(ArrayRef< TypeConversionCostTblEntryT< CostType > > Tbl, int ISD, MVT Dst, MVT Src)
Find in type conversion cost table.
Definition CostTable.h:67
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
Returns options for expansion of memcmp. IsZeroCmp is.
Parameters that control the generic loop unrolling transformation.
bool UpperBound
Allow using trip count upper bound to unroll loops.
unsigned PartialOptSizeThreshold
The cost threshold for the unrolled loop when optimizing for size, like OptSizeThreshold,...
unsigned PartialThreshold
The cost threshold for the unrolled loop, like Threshold, but used for partial/runtime unrolling (set...
bool Runtime
Allow runtime unrolling (unrolling of loops to expand the size of the loop body even when the number ...
bool Partial
Allow partial unrolling (unrolling of loops to expand the size of the loop body, not only to eliminat...
unsigned OptSizeThreshold
The cost threshold for the unrolled loop when optimizing for size (set to UINT_MAX to disable).