LLVM 24.0.0git
X86TargetTransformInfo.cpp
Go to the documentation of this file.
1//===-- X86TargetTransformInfo.cpp - X86 specific TTI pass ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements a TargetTransformInfo analysis pass specific to the
10/// X86 target machine. It uses the target's detailed information to provide
11/// more precise answers to certain TTI queries, while letting the target
12/// independent and default TTI implementations handle the rest.
13///
14//===----------------------------------------------------------------------===//
15/// About Cost Model numbers used below it's necessary to say the following:
16/// the numbers correspond to some "generic" X86 CPU instead of usage of a
17/// specific CPU model. Usually the numbers correspond to the CPU where the
18/// feature first appeared. For example, if we do Subtarget.hasSSE42() in
19/// the lookups below the cost is based on Nehalem as that was the first CPU
20/// to support that feature level and thus has most likely the worst case cost,
21/// although we may discard an outlying worst cost from one CPU (e.g. Atom).
22///
23/// Some examples of other technologies/CPUs:
24/// SSE 3 - Pentium4 / Athlon64
25/// SSE 4.1 - Penryn
26/// SSE 4.2 - Nehalem / Silvermont
27/// AVX - Sandy Bridge / Jaguar / Bulldozer
28/// AVX2 - Haswell / Ryzen
29/// AVX-512 - Xeon Phi / Skylake
30///
31/// And some examples of instruction target dependent costs (latency)
32/// divss sqrtss rsqrtss
33/// AMD K7 11-16 19 3
34/// Piledriver 9-24 13-15 5
35/// Jaguar 14 16 2
36/// Pentium II,III 18 30 2
37/// Nehalem 7-14 7-18 3
38/// Haswell 10-13 11 5
39///
40/// Interpreting the 4 TargetCostKind types:
41/// TCK_RecipThroughput and TCK_Latency should try to match the worst case
42/// values reported by the CPU scheduler models (and llvm-mca).
43/// TCK_CodeSize should match the instruction count (e.g. divss = 1), NOT the
44/// actual encoding size of the instruction.
45/// TCK_SizeAndLatency should match the worst case micro-op counts reported by
46/// by the CPU scheduler models (and llvm-mca), to ensure that they are
47/// compatible with the MicroOpBufferSize and LoopMicroOpBufferSize values which are
48/// often used as the cost thresholds where TCK_SizeAndLatency is requested.
49//===----------------------------------------------------------------------===//
50
60#include <optional>
61
62using namespace llvm;
63
64#define DEBUG_TYPE "x86tti"
65
66//===----------------------------------------------------------------------===//
67//
68// X86 cost model.
69//
70//===----------------------------------------------------------------------===//
71
74
76X86TTIImpl::getPopcntSupport(unsigned TyWidth) const {
77 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
78 // TODO: Currently the __builtin_popcount() implementation using SSE3
79 // instructions is inefficient. Once the problem is fixed, we should
80 // call ST->hasSSE3() instead of ST->hasPOPCNT().
81 return ST->hasPOPCNT() ? TTI::PSK_FastHardware : TTI::PSK_Software;
82}
83
84std::optional<unsigned> X86TTIImpl::getCacheSize(
86 switch (Level) {
88 // - Penryn
89 // - Nehalem
90 // - Westmere
91 // - Sandy Bridge
92 // - Ivy Bridge
93 // - Haswell
94 // - Broadwell
95 // - Skylake
96 // - Kabylake
97 return 32 * 1024; // 32 KiB
99 // - Penryn
100 // - Nehalem
101 // - Westmere
102 // - Sandy Bridge
103 // - Ivy Bridge
104 // - Haswell
105 // - Broadwell
106 // - Skylake
107 // - Kabylake
108 return 256 * 1024; // 256 KiB
109 }
110
111 llvm_unreachable("Unknown TargetTransformInfo::CacheLevel");
112}
113
114std::optional<unsigned> X86TTIImpl::getCacheAssociativity(
116 // - Penryn
117 // - Nehalem
118 // - Westmere
119 // - Sandy Bridge
120 // - Ivy Bridge
121 // - Haswell
122 // - Broadwell
123 // - Skylake
124 // - Kabylake
125 switch (Level) {
127 [[fallthrough]];
129 return 8;
130 }
131
132 llvm_unreachable("Unknown TargetTransformInfo::CacheLevel");
133}
134
136
138 return Vector ? VectorClass
139 : Ty && Ty->isFloatingPointTy() ? ScalarFPClass
140 : GPRClass;
141}
142
143unsigned X86TTIImpl::getNumberOfRegisters(unsigned ClassID) const {
144 if (ClassID == VectorClass && !ST->hasSSE1())
145 return 0;
146
147 if (!ST->is64Bit())
148 return 8;
149
150 if ((ClassID == GPRClass && ST->hasEGPR()) ||
151 (ClassID != GPRClass && ST->hasAVX512()))
152 return 32;
153
154 return 16;
155}
156
158 if (!ST->hasCF())
159 return false;
160 if (!Ty)
161 return true;
162 // Conditional faulting is supported by CFCMOV, which only accepts
163 // 16/32/64-bit operands.
164 // TODO: Support f32/f64 with VMOVSS/VMOVSD with zero mask when it's
165 // profitable.
166 auto *VTy = dyn_cast<FixedVectorType>(Ty);
167 if (!Ty->isIntegerTy() && (!VTy || VTy->getNumElements() != 1))
168 return false;
169 auto *ScalarTy = Ty->getScalarType();
170 switch (cast<IntegerType>(ScalarTy)->getBitWidth()) {
171 default:
172 return false;
173 case 16:
174 case 32:
175 case 64:
176 return true;
177 }
178}
179
182 unsigned PreferVectorWidth = ST->getPreferVectorWidth();
183 switch (K) {
185 return TypeSize::getFixed(ST->is64Bit() ? 64 : 32);
187 if (ST->hasAVX512() && PreferVectorWidth >= 512)
188 return TypeSize::getFixed(512);
189 if (ST->hasAVX() && PreferVectorWidth >= 256)
190 return TypeSize::getFixed(256);
191 if (ST->hasSSE1() && PreferVectorWidth >= 128)
192 return TypeSize::getFixed(128);
193 return TypeSize::getFixed(0);
195 return TypeSize::getScalable(0);
196 }
197
198 llvm_unreachable("Unsupported register kind");
199}
200
205
207 bool HasUnorderedReductions) const {
208 // If the loop will not be vectorized, don't interleave the loop.
209 // Let regular unroll to unroll the loop, which saves the overflow
210 // check and memory check cost.
211 if (VF.isScalar())
212 return 1;
213
214 if (ST->isAtom())
215 return 1;
216
217 // Sandybridge and Haswell have multiple execution ports and pipelined
218 // vector units.
219 if (ST->hasAVX())
220 return 4;
221
222 return 2;
223}
224
226 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
228 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
229
230 // vXi8 multiplications are always promoted to vXi16.
231 // Sub-128-bit types can be extended/packed more efficiently.
232 if (Opcode == Instruction::Mul && Ty->isVectorTy() &&
233 Ty->getPrimitiveSizeInBits() <= 64 && Ty->getScalarSizeInBits() == 8) {
234 Type *WideVecTy =
236 return getCastInstrCost(Instruction::ZExt, WideVecTy, Ty,
238 CostKind) +
239 getCastInstrCost(Instruction::Trunc, Ty, WideVecTy,
241 CostKind) +
242 getArithmeticInstrCost(Opcode, WideVecTy, CostKind, Op1Info, Op2Info);
243 }
244
245 // Legalize the type.
246 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
247
248 int ISD = TLI->InstructionOpcodeToISD(Opcode);
249 assert(ISD && "Invalid opcode");
250
251 if (ISD == ISD::MUL && Args.size() == 2 && LT.second.isVector() &&
252 (LT.second.getScalarType() == MVT::i32 ||
253 LT.second.getScalarType() == MVT::i64)) {
254 // Check if the operands can be represented as a smaller datatype.
255 bool Op1Signed = false, Op2Signed = false;
256 unsigned Op1MinSize = BaseT::minRequiredElementSize(Args[0], Op1Signed);
257 unsigned Op2MinSize = BaseT::minRequiredElementSize(Args[1], Op2Signed);
258 unsigned OpMinSize = std::max(Op1MinSize, Op2MinSize);
259 bool SignedMode = Op1Signed || Op2Signed;
260
261 // If both vXi32 are representable as i15 and at least one is constant,
262 // zero-extended, or sign-extended from vXi16 (or less pre-SSE41) then we
263 // can treat this as PMADDWD which has the same costs as a vXi16 multiply.
264 if (OpMinSize <= 15 && !ST->isPMADDWDSlow() &&
265 LT.second.getScalarType() == MVT::i32) {
266 bool Op1Constant =
267 isa<ConstantDataVector>(Args[0]) || isa<ConstantVector>(Args[0]);
268 bool Op2Constant =
269 isa<ConstantDataVector>(Args[1]) || isa<ConstantVector>(Args[1]);
270 bool Op1Sext = isa<SExtInst>(Args[0]) &&
271 (Op1MinSize == 15 || (Op1MinSize < 15 && !ST->hasSSE41()));
272 bool Op2Sext = isa<SExtInst>(Args[1]) &&
273 (Op2MinSize == 15 || (Op2MinSize < 15 && !ST->hasSSE41()));
274
275 bool IsZeroExtended = !Op1Signed || !Op2Signed;
276 bool IsConstant = Op1Constant || Op2Constant;
277 bool IsSext = Op1Sext || Op2Sext;
278 if (IsConstant || IsZeroExtended || IsSext)
279 LT.second =
280 MVT::getVectorVT(MVT::i16, 2 * LT.second.getVectorNumElements());
281 }
282
283 // Check if the vXi32 operands can be shrunk into a smaller datatype.
284 // This should match the codegen from reduceVMULWidth.
285 // TODO: Make this generic (!ST->SSE41 || ST->isPMULLDSlow()).
286 if (ST->useSLMArithCosts() && LT.second == MVT::v4i32) {
287 if (OpMinSize <= 7)
288 return LT.first * 3; // pmullw/sext
289 if (!SignedMode && OpMinSize <= 8)
290 return LT.first * 3; // pmullw/zext
291 if (OpMinSize <= 15)
292 return LT.first * 5; // pmullw/pmulhw/pshuf
293 if (!SignedMode && OpMinSize <= 16)
294 return LT.first * 5; // pmullw/pmulhw/pshuf
295 }
296
297 // If both vXi64 are representable as (unsigned) i32, then we can perform
298 // the multiple with a single PMULUDQ instruction.
299 // TODO: Add (SSE41+) PMULDQ handling for signed extensions.
300 if (!SignedMode && OpMinSize <= 32 && LT.second.getScalarType() == MVT::i64)
301 ISD = X86ISD::PMULUDQ;
302 }
303
304 // Vector multiply by pow2 will be simplified to shifts.
305 // Vector multiply by -pow2 will be simplified to shifts/negates.
306 if (ISD == ISD::MUL && Op2Info.isConstant() &&
307 (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2())) {
309 getArithmeticInstrCost(Instruction::Shl, Ty, CostKind,
310 Op1Info.getNoProps(), Op2Info.getNoProps());
311 if (Op2Info.isNegatedPowerOf2())
312 Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind);
313 return Cost;
314 }
315
316 // On X86, vector signed division by constants power-of-two are
317 // normally expanded to the sequence SRA + SRL + ADD + SRA.
318 // The OperandValue properties may not be the same as that of the previous
319 // operation; conservatively assume OP_None.
320 if ((ISD == ISD::SDIV || ISD == ISD::SREM) &&
321 Op2Info.isConstant() && Op2Info.isPowerOf2()) {
323 2 * getArithmeticInstrCost(Instruction::AShr, Ty, CostKind,
324 Op1Info.getNoProps(), Op2Info.getNoProps());
325 Cost += getArithmeticInstrCost(Instruction::LShr, Ty, CostKind,
326 Op1Info.getNoProps(), Op2Info.getNoProps());
327 Cost += getArithmeticInstrCost(Instruction::Add, Ty, CostKind,
328 Op1Info.getNoProps(), Op2Info.getNoProps());
329
330 if (ISD == ISD::SREM) {
331 // For SREM: (X % C) is the equivalent of (X - (X/C)*C)
332 Cost += getArithmeticInstrCost(Instruction::Mul, Ty, CostKind, Op1Info.getNoProps(),
333 Op2Info.getNoProps());
334 Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind, Op1Info.getNoProps(),
335 Op2Info.getNoProps());
336 }
337
338 return Cost;
339 }
340
341 // Vector unsigned division/remainder will be simplified to shifts/masks.
342 if ((ISD == ISD::UDIV || ISD == ISD::UREM) &&
343 Op2Info.isConstant() && Op2Info.isPowerOf2()) {
344 if (ISD == ISD::UDIV)
345 return getArithmeticInstrCost(Instruction::LShr, Ty, CostKind,
346 Op1Info.getNoProps(), Op2Info.getNoProps());
347 // UREM
348 return getArithmeticInstrCost(Instruction::And, Ty, CostKind,
349 Op1Info.getNoProps(), Op2Info.getNoProps());
350 }
351
352 // A rem paired with a div on the same operands in the same block is free:
353 // the pair costs the div alone. Power-of-two divisors are handled above.
354 if ((ISD == ISD::UREM || ISD == ISD::SREM) && CostKind != TTI::TCK_Latency &&
355 CtxI && CtxI->getOpcode() == Opcode) {
356 using namespace PatternMatch;
357 // Same use limit as SLP, bounds the walk over the operand users.
358 constexpr unsigned MaxUses = 64;
359 unsigned DivOpcode =
360 ISD == ISD::SREM ? Instruction::SDiv : Instruction::UDiv;
361 const Value *X = CtxI->getOperand(0);
362 const Value *Y = CtxI->getOperand(1);
363 // Constants have context-wide use lists, walk the variable operand.
364 const Value *V = isa<Constant>(Y) ? X : Y;
365 if (!isa<Constant>(V) && !V->hasNUsesOrMore(MaxUses) &&
366 any_of(V->users(), [&](const User *U) {
367 return match(U, m_BinOp(DivOpcode, m_Specific(X), m_Specific(Y))) &&
368 cast<Instruction>(U)->getParent() == CtxI->getParent();
369 }))
370 return TTI::TCC_Free;
371 }
372
373 // A scalar integer divide/remainder by a constant is not a hardware divide;
374 // it lowers to a magic-number multiply-high plus a few fixup ops. Cost it as
375 // that sequence rather than the generic single-instruction divide, so the
376 // vectorizers do not compare against an artificially cheap scalar lane. The
377 // power-of-two cases are handled above; negated powers of two are left to the
378 // generic handling.
379 if (!Ty->isVectorTy() && Op2Info.isConstant() && !Op2Info.isNegatedPowerOf2() &&
380 (ISD == ISD::UDIV || ISD == ISD::SDIV || ISD == ISD::UREM ||
381 ISD == ISD::SREM)) {
382 unsigned Cost = ISD == ISD::UREM || ISD == ISD::SREM ? 6 : 5;
383 if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
384 Cost += 2;
385 return LT.first * Cost;
386 }
387
388 static const CostKindTblEntry GFNIUniformConstCostTable[] = {
389 { ISD::SHL, MVT::v16i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
390 { ISD::SRL, MVT::v16i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
391 { ISD::SRA, MVT::v16i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
392 { ISD::SHL, MVT::v32i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
393 { ISD::SRL, MVT::v32i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
394 { ISD::SRA, MVT::v32i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
395 { ISD::SHL, MVT::v64i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
396 { ISD::SRL, MVT::v64i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
397 { ISD::SRA, MVT::v64i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
398 };
399
400 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasGFNI())
401 if (const auto *Entry =
402 CostTableLookup(GFNIUniformConstCostTable, ISD, LT.second))
403 if (auto KindCost = Entry->Cost[CostKind])
404 return LT.first * *KindCost;
405
406 static const CostKindTblEntry AVX512BWUniformConstCostTable[] = {
407 { ISD::SHL, MVT::v16i8, { 1, 7, 2, 3 } }, // psllw + pand.
408 { ISD::SRL, MVT::v16i8, { 1, 7, 2, 3 } }, // psrlw + pand.
409 { ISD::SRA, MVT::v16i8, { 1, 8, 4, 5 } }, // psrlw, pand, pxor, psubb.
410 { ISD::SHL, MVT::v32i8, { 1, 8, 2, 3 } }, // psllw + pand.
411 { ISD::SRL, MVT::v32i8, { 1, 8, 2, 3 } }, // psrlw + pand.
412 { ISD::SRA, MVT::v32i8, { 1, 9, 4, 5 } }, // psrlw, pand, pxor, psubb.
413 { ISD::SHL, MVT::v64i8, { 1, 8, 2, 3 } }, // psllw + pand.
414 { ISD::SRL, MVT::v64i8, { 1, 8, 2, 3 } }, // psrlw + pand.
415 { ISD::SRA, MVT::v64i8, { 1, 9, 4, 6 } }, // psrlw, pand, pxor, psubb.
416
417 { ISD::SHL, MVT::v16i16, { 1, 1, 1, 1 } }, // psllw
418 { ISD::SRL, MVT::v16i16, { 1, 1, 1, 1 } }, // psrlw
419 { ISD::SRA, MVT::v16i16, { 1, 1, 1, 1 } }, // psrlw
420 { ISD::SHL, MVT::v32i16, { 1, 1, 1, 1 } }, // psllw
421 { ISD::SRL, MVT::v32i16, { 1, 1, 1, 1 } }, // psrlw
422 { ISD::SRA, MVT::v32i16, { 1, 1, 1, 1 } }, // psrlw
423 };
424
425 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasBWI())
426 if (const auto *Entry =
427 CostTableLookup(AVX512BWUniformConstCostTable, ISD, LT.second))
428 if (auto KindCost = Entry->Cost[CostKind])
429 return LT.first * *KindCost;
430
431 static const CostKindTblEntry AVX512DQUniformConstCostTable[] = {
432 { ISD::SDIV, MVT::v4i64, { 15 } }, // vpmullq-based MULHS sequence
433 { ISD::SREM, MVT::v4i64, { 17 } }, // vpmullq-based MULHS+mul+sub sequence
434 { ISD::SDIV, MVT::v8i64, { 15 } }, // vpmullq-based MULHS sequence
435 { ISD::SREM, MVT::v8i64, { 17 } }, // vpmullq-based MULHS+mul+sub sequence
436 // The remainder's multiply-back is a single vpmullq with DQ, just like the
437 // pmulld the vXi32 entries above rely on. Without DQ it is another
438 // vpmuludq schoolbook, so the AVX512/AVX2 tables charge more.
439 { ISD::UREM, MVT::v4i64, { 17 } }, // MULHU + vpmullq + sub sequence
440 { ISD::UREM, MVT::v8i64, { 17 } }, // MULHU + vpmullq + sub sequence
441 };
442
443 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasDQI())
444 if (const auto *Entry =
445 CostTableLookup(AVX512DQUniformConstCostTable, ISD, LT.second))
446 if (auto KindCost = Entry->Cost[CostKind])
447 return LT.first * *KindCost;
448
449 static const CostKindTblEntry AVX512UniformConstCostTable[] = {
450 { ISD::SHL, MVT::v64i8, { 2, 12, 5, 6 } }, // psllw + pand.
451 { ISD::SRL, MVT::v64i8, { 2, 12, 5, 6 } }, // psrlw + pand.
452 { ISD::SRA, MVT::v64i8, { 3, 10, 12, 12 } }, // psrlw, pand, pxor, psubb.
453
454 { ISD::SHL, MVT::v16i16, { 2, 7, 4, 4 } }, // psllw + split.
455 { ISD::SRL, MVT::v16i16, { 2, 7, 4, 4 } }, // psrlw + split.
456 { ISD::SRA, MVT::v16i16, { 2, 7, 4, 4 } }, // psraw + split.
457
458 { ISD::SHL, MVT::v8i32, { 1, 1, 1, 1 } }, // pslld
459 { ISD::SRL, MVT::v8i32, { 1, 1, 1, 1 } }, // psrld
460 { ISD::SRA, MVT::v8i32, { 1, 1, 1, 1 } }, // psrad
461 { ISD::SHL, MVT::v16i32, { 1, 1, 1, 1 } }, // pslld
462 { ISD::SRL, MVT::v16i32, { 1, 1, 1, 1 } }, // psrld
463 { ISD::SRA, MVT::v16i32, { 1, 1, 1, 1 } }, // psrad
464
465 { ISD::SRA, MVT::v2i64, { 1, 1, 1, 1 } }, // psraq
466 { ISD::SHL, MVT::v4i64, { 1, 1, 1, 1 } }, // psllq
467 { ISD::SRL, MVT::v4i64, { 1, 1, 1, 1 } }, // psrlq
468 { ISD::SRA, MVT::v4i64, { 1, 1, 1, 1 } }, // psraq
469 { ISD::SHL, MVT::v8i64, { 1, 1, 1, 1 } }, // psllq
470 { ISD::SRL, MVT::v8i64, { 1, 1, 1, 1 } }, // psrlq
471 { ISD::SRA, MVT::v8i64, { 1, 1, 1, 1 } }, // psraq
472
473 { ISD::SDIV, MVT::v16i32, { 6 } }, // pmuludq sequence
474 { ISD::SREM, MVT::v16i32, { 8 } }, // pmuludq+mul+sub sequence
475 { ISD::UDIV, MVT::v16i32, { 5 } }, // pmuludq sequence
476 { ISD::UREM, MVT::v16i32, { 7 } }, // pmuludq+mul+sub sequence
477
478 { ISD::UDIV, MVT::v8i64, { 15 } }, // pmuludq-based MULHU sequence
479 { ISD::UREM, MVT::v8i64, { 21 } }, // pmuludq-based MULHU+mul+sub sequence
480 };
481
482 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX512())
483 if (const auto *Entry =
484 CostTableLookup(AVX512UniformConstCostTable, ISD, LT.second))
485 if (auto KindCost = Entry->Cost[CostKind])
486 return LT.first * *KindCost;
487
488 static const CostKindTblEntry AVX2UniformConstCostTable[] = {
489 { ISD::SHL, MVT::v16i8, { 1, 8, 2, 3 } }, // psllw + pand.
490 { ISD::SRL, MVT::v16i8, { 1, 8, 2, 3 } }, // psrlw + pand.
491 { ISD::SRA, MVT::v16i8, { 2, 10, 5, 6 } }, // psrlw, pand, pxor, psubb.
492 { ISD::SHL, MVT::v32i8, { 2, 8, 2, 4 } }, // psllw + pand.
493 { ISD::SRL, MVT::v32i8, { 2, 8, 2, 4 } }, // psrlw + pand.
494 { ISD::SRA, MVT::v32i8, { 3, 10, 5, 9 } }, // psrlw, pand, pxor, psubb.
495
496 { ISD::SHL, MVT::v8i16, { 1, 1, 1, 1 } }, // psllw
497 { ISD::SRL, MVT::v8i16, { 1, 1, 1, 1 } }, // psrlw
498 { ISD::SRA, MVT::v8i16, { 1, 1, 1, 1 } }, // psraw
499 { ISD::SHL, MVT::v16i16,{ 2, 2, 1, 2 } }, // psllw
500 { ISD::SRL, MVT::v16i16,{ 2, 2, 1, 2 } }, // psrlw
501 { ISD::SRA, MVT::v16i16,{ 2, 2, 1, 2 } }, // psraw
502
503 { ISD::SHL, MVT::v4i32, { 1, 1, 1, 1 } }, // pslld
504 { ISD::SRL, MVT::v4i32, { 1, 1, 1, 1 } }, // psrld
505 { ISD::SRA, MVT::v4i32, { 1, 1, 1, 1 } }, // psrad
506 { ISD::SHL, MVT::v8i32, { 2, 2, 1, 2 } }, // pslld
507 { ISD::SRL, MVT::v8i32, { 2, 2, 1, 2 } }, // psrld
508 { ISD::SRA, MVT::v8i32, { 2, 2, 1, 2 } }, // psrad
509
510 { ISD::SHL, MVT::v2i64, { 1, 1, 1, 1 } }, // psllq
511 { ISD::SRL, MVT::v2i64, { 1, 1, 1, 1 } }, // psrlq
512 { ISD::SRA, MVT::v2i64, { 2, 3, 3, 3 } }, // psrad + shuffle.
513 { ISD::SHL, MVT::v4i64, { 2, 2, 1, 2 } }, // psllq
514 { ISD::SRL, MVT::v4i64, { 2, 2, 1, 2 } }, // psrlq
515 { ISD::SRA, MVT::v4i64, { 4, 4, 3, 6 } }, // psrad + shuffle + split.
516
517 { ISD::SDIV, MVT::v8i32, { 6 } }, // pmuludq sequence
518 { ISD::SREM, MVT::v8i32, { 8 } }, // pmuludq+mul+sub sequence
519 { ISD::UDIV, MVT::v8i32, { 5 } }, // pmuludq sequence
520 { ISD::UREM, MVT::v8i32, { 7 } }, // pmuludq+mul+sub sequence
521
522 { ISD::UDIV, MVT::v4i64, { 15 } }, // pmuludq-based MULHU sequence
523 { ISD::UREM, MVT::v4i64, { 21 } }, // pmuludq-based MULHU+mul+sub sequence
524 };
525
526 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX2())
527 if (const auto *Entry =
528 CostTableLookup(AVX2UniformConstCostTable, ISD, LT.second))
529 if (auto KindCost = Entry->Cost[CostKind])
530 return LT.first * *KindCost;
531
532 static const CostKindTblEntry AVXUniformConstCostTable[] = {
533 { ISD::SHL, MVT::v16i8, { 2, 7, 2, 3 } }, // psllw + pand.
534 { ISD::SRL, MVT::v16i8, { 2, 7, 2, 3 } }, // psrlw + pand.
535 { ISD::SRA, MVT::v16i8, { 3, 9, 5, 6 } }, // psrlw, pand, pxor, psubb.
536 { ISD::SHL, MVT::v32i8, { 4, 7, 7, 8 } }, // 2*(psllw + pand) + split.
537 { ISD::SRL, MVT::v32i8, { 4, 7, 7, 8 } }, // 2*(psrlw + pand) + split.
538 { ISD::SRA, MVT::v32i8, { 7, 7, 12, 13 } }, // 2*(psrlw, pand, pxor, psubb) + split.
539
540 { ISD::SHL, MVT::v8i16, { 1, 2, 1, 1 } }, // psllw.
541 { ISD::SRL, MVT::v8i16, { 1, 2, 1, 1 } }, // psrlw.
542 { ISD::SRA, MVT::v8i16, { 1, 2, 1, 1 } }, // psraw.
543 { ISD::SHL, MVT::v16i16,{ 3, 6, 4, 5 } }, // psllw + split.
544 { ISD::SRL, MVT::v16i16,{ 3, 6, 4, 5 } }, // psrlw + split.
545 { ISD::SRA, MVT::v16i16,{ 3, 6, 4, 5 } }, // psraw + split.
546
547 { ISD::SHL, MVT::v4i32, { 1, 2, 1, 1 } }, // pslld.
548 { ISD::SRL, MVT::v4i32, { 1, 2, 1, 1 } }, // psrld.
549 { ISD::SRA, MVT::v4i32, { 1, 2, 1, 1 } }, // psrad.
550 { ISD::SHL, MVT::v8i32, { 3, 6, 4, 5 } }, // pslld + split.
551 { ISD::SRL, MVT::v8i32, { 3, 6, 4, 5 } }, // psrld + split.
552 { ISD::SRA, MVT::v8i32, { 3, 6, 4, 5 } }, // psrad + split.
553
554 { ISD::SHL, MVT::v2i64, { 1, 2, 1, 1 } }, // psllq.
555 { ISD::SRL, MVT::v2i64, { 1, 2, 1, 1 } }, // psrlq.
556 { ISD::SRA, MVT::v2i64, { 2, 3, 3, 3 } }, // psrad + shuffle.
557 { ISD::SHL, MVT::v4i64, { 3, 6, 4, 5 } }, // 2 x psllq + split.
558 { ISD::SRL, MVT::v4i64, { 3, 6, 4, 5 } }, // 2 x psllq + split.
559 { ISD::SRA, MVT::v4i64, { 5, 7, 8, 9 } }, // 2 x psrad + shuffle + split.
560
561 { ISD::SDIV, MVT::v8i32, { 14 } }, // 2*pmuludq sequence + split.
562 { ISD::SREM, MVT::v8i32, { 18 } }, // 2*pmuludq+mul+sub sequence + split.
563 { ISD::UDIV, MVT::v8i32, { 12 } }, // 2*pmuludq sequence + split.
564 { ISD::UREM, MVT::v8i32, { 16 } }, // 2*pmuludq+mul+sub sequence + split.
565 };
566
567 // XOP has faster vXi8 shifts.
568 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX() &&
569 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
570 if (const auto *Entry =
571 CostTableLookup(AVXUniformConstCostTable, ISD, LT.second))
572 if (auto KindCost = Entry->Cost[CostKind])
573 return LT.first * *KindCost;
574
575 static const CostKindTblEntry SSE2UniformConstCostTable[] = {
576 { ISD::SHL, MVT::v16i8, { 1, 7, 2, 3 } }, // psllw + pand.
577 { ISD::SRL, MVT::v16i8, { 1, 7, 2, 3 } }, // psrlw + pand.
578 { ISD::SRA, MVT::v16i8, { 3, 9, 5, 6 } }, // psrlw, pand, pxor, psubb.
579
580 { ISD::SHL, MVT::v8i16, { 1, 1, 1, 1 } }, // psllw.
581 { ISD::SRL, MVT::v8i16, { 1, 1, 1, 1 } }, // psrlw.
582 { ISD::SRA, MVT::v8i16, { 1, 1, 1, 1 } }, // psraw.
583
584 { ISD::SHL, MVT::v4i32, { 1, 1, 1, 1 } }, // pslld
585 { ISD::SRL, MVT::v4i32, { 1, 1, 1, 1 } }, // psrld.
586 { ISD::SRA, MVT::v4i32, { 1, 1, 1, 1 } }, // psrad.
587
588 { ISD::SHL, MVT::v2i64, { 1, 1, 1, 1 } }, // psllq.
589 { ISD::SRL, MVT::v2i64, { 1, 1, 1, 1 } }, // psrlq.
590 { ISD::SRA, MVT::v2i64, { 3, 5, 6, 6 } }, // 2 x psrad + shuffle.
591
592 { ISD::SDIV, MVT::v4i32, { 6 } }, // pmuludq sequence
593 { ISD::SREM, MVT::v4i32, { 8 } }, // pmuludq+mul+sub sequence
594 { ISD::UDIV, MVT::v4i32, { 5 } }, // pmuludq sequence
595 { ISD::UREM, MVT::v4i32, { 7 } }, // pmuludq+mul+sub sequence
596 };
597
598 // XOP has faster vXi8 shifts.
599 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasSSE2() &&
600 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
601 if (const auto *Entry =
602 CostTableLookup(SSE2UniformConstCostTable, ISD, LT.second))
603 if (auto KindCost = Entry->Cost[CostKind])
604 return LT.first * *KindCost;
605
606 static const CostKindTblEntry AVX512BWConstCostTable[] = {
607 { ISD::SDIV, MVT::v64i8, { 14 } }, // 2*ext+2*pmulhw sequence
608 { ISD::SREM, MVT::v64i8, { 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
609 { ISD::UDIV, MVT::v64i8, { 14 } }, // 2*ext+2*pmulhw sequence
610 { ISD::UREM, MVT::v64i8, { 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
611
612 { ISD::SDIV, MVT::v32i16, { 6 } }, // vpmulhw sequence
613 { ISD::SREM, MVT::v32i16, { 8 } }, // vpmulhw+mul+sub sequence
614 { ISD::UDIV, MVT::v32i16, { 6 } }, // vpmulhuw sequence
615 { ISD::UREM, MVT::v32i16, { 8 } }, // vpmulhuw+mul+sub sequence
616 };
617
618 if (Op2Info.isConstant() && ST->hasBWI())
619 if (const auto *Entry =
620 CostTableLookup(AVX512BWConstCostTable, ISD, LT.second))
621 if (auto KindCost = Entry->Cost[CostKind])
622 return LT.first * *KindCost;
623
624 static const CostKindTblEntry AVX512DQConstCostTable[] = {
625 { ISD::SDIV, MVT::v4i64, { 19 } }, // vpmullq-based MULHS sequence
626 { ISD::SREM, MVT::v4i64, { 21 } }, // vpmullq-based MULHS+mul+sub sequence
627 { ISD::SDIV, MVT::v8i64, { 19 } }, // vpmullq-based MULHS sequence
628 { ISD::SREM, MVT::v8i64, { 21 } }, // vpmullq-based MULHS+mul+sub sequence
629 // The remainder's multiply-back is a single vpmullq with DQ, whereas the
630 // AVX512/AVX2 tables have to charge for another vpmuludq schoolbook.
631 { ISD::UREM, MVT::v4i64, { 24 } }, // MULHU + vpmullq + sub sequence
632 { ISD::UREM, MVT::v8i64, { 24 } }, // MULHU + vpmullq + sub sequence
633 };
634
635 if (Op2Info.isConstant() && ST->hasDQI())
636 if (const auto *Entry =
637 CostTableLookup(AVX512DQConstCostTable, ISD, LT.second))
638 if (auto KindCost = Entry->Cost[CostKind])
639 return LT.first * *KindCost;
640
641 static const CostKindTblEntry AVX512ConstCostTable[] = {
642 { ISD::SDIV, MVT::v64i8, { 28 } }, // 4*ext+4*pmulhw sequence
643 { ISD::SREM, MVT::v64i8, { 32 } }, // 4*ext+4*pmulhw+mul+sub sequence
644 { ISD::UDIV, MVT::v64i8, { 28 } }, // 4*ext+4*pmulhw sequence
645 { ISD::UREM, MVT::v64i8, { 32 } }, // 4*ext+4*pmulhw+mul+sub sequence
646
647 { ISD::SDIV, MVT::v32i16, { 12 } }, // 2*vpmulhw sequence
648 { ISD::SREM, MVT::v32i16, { 16 } }, // 2*vpmulhw+mul+sub sequence
649 { ISD::UDIV, MVT::v32i16, { 12 } }, // 2*vpmulhuw sequence
650 { ISD::UREM, MVT::v32i16, { 16 } }, // 2*vpmulhuw+mul+sub sequence
651
652 { ISD::SDIV, MVT::v16i32, { 15 } }, // vpmuldq sequence
653 { ISD::SREM, MVT::v16i32, { 17 } }, // vpmuldq+mul+sub sequence
654 { ISD::UDIV, MVT::v16i32, { 15 } }, // vpmuludq sequence
655 { ISD::UREM, MVT::v16i32, { 17 } }, // vpmuludq+mul+sub sequence
656
657 { ISD::UDIV, MVT::v8i64, { 22 } }, // vpmuludq-based MULHU sequence
658 { ISD::UREM, MVT::v8i64, { 28 } }, // vpmuludq-based MULHU+mul+sub sequence
659 };
660
661 if (Op2Info.isConstant() && ST->hasAVX512())
662 if (const auto *Entry =
663 CostTableLookup(AVX512ConstCostTable, ISD, LT.second))
664 if (auto KindCost = Entry->Cost[CostKind])
665 return LT.first * *KindCost;
666
667 static const CostKindTblEntry AVX2ConstCostTable[] = {
668 { ISD::SDIV, MVT::v32i8, { 14 } }, // 2*ext+2*pmulhw sequence
669 { ISD::SREM, MVT::v32i8, { 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
670 { ISD::UDIV, MVT::v32i8, { 14 } }, // 2*ext+2*pmulhw sequence
671 { ISD::UREM, MVT::v32i8, { 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
672
673 { ISD::SDIV, MVT::v16i16, { 6 } }, // vpmulhw sequence
674 { ISD::SREM, MVT::v16i16, { 8 } }, // vpmulhw+mul+sub sequence
675 { ISD::UDIV, MVT::v16i16, { 6 } }, // vpmulhuw sequence
676 { ISD::UREM, MVT::v16i16, { 8 } }, // vpmulhuw+mul+sub sequence
677
678 { ISD::SDIV, MVT::v8i32, { 15 } }, // vpmuldq sequence
679 { ISD::SREM, MVT::v8i32, { 19 } }, // vpmuldq+mul+sub sequence
680 { ISD::UDIV, MVT::v8i32, { 15 } }, // vpmuludq sequence
681 { ISD::UREM, MVT::v8i32, { 19 } }, // vpmuludq+mul+sub sequence
682
683 { ISD::UDIV, MVT::v4i64, { 22 } }, // vpmuludq-based MULHU sequence
684 { ISD::UREM, MVT::v4i64, { 28 } }, // vpmuludq-based MULHU+mul+sub sequence
685 };
686
687 if (Op2Info.isConstant() && ST->hasAVX2())
688 if (const auto *Entry = CostTableLookup(AVX2ConstCostTable, ISD, LT.second))
689 if (auto KindCost = Entry->Cost[CostKind])
690 return LT.first * *KindCost;
691
692 static const CostKindTblEntry AVXConstCostTable[] = {
693 { ISD::SDIV, MVT::v32i8, { 30 } }, // 4*ext+4*pmulhw sequence + split.
694 { ISD::SREM, MVT::v32i8, { 34 } }, // 4*ext+4*pmulhw+mul+sub sequence + split.
695 { ISD::UDIV, MVT::v32i8, { 30 } }, // 4*ext+4*pmulhw sequence + split.
696 { ISD::UREM, MVT::v32i8, { 34 } }, // 4*ext+4*pmulhw+mul+sub sequence + split.
697
698 { ISD::SDIV, MVT::v16i16, { 14 } }, // 2*pmulhw sequence + split.
699 { ISD::SREM, MVT::v16i16, { 18 } }, // 2*pmulhw+mul+sub sequence + split.
700 { ISD::UDIV, MVT::v16i16, { 14 } }, // 2*pmulhuw sequence + split.
701 { ISD::UREM, MVT::v16i16, { 18 } }, // 2*pmulhuw+mul+sub sequence + split.
702
703 { ISD::SDIV, MVT::v8i32, { 32 } }, // vpmuludq sequence
704 { ISD::SREM, MVT::v8i32, { 38 } }, // vpmuludq+mul+sub sequence
705 { ISD::UDIV, MVT::v8i32, { 32 } }, // 2*pmuludq sequence + split.
706 { ISD::UREM, MVT::v8i32, { 42 } }, // 2*pmuludq+mul+sub sequence + split.
707 };
708
709 if (Op2Info.isConstant() && ST->hasAVX())
710 if (const auto *Entry = CostTableLookup(AVXConstCostTable, ISD, LT.second))
711 if (auto KindCost = Entry->Cost[CostKind])
712 return LT.first * *KindCost;
713
714 static const CostKindTblEntry SSE41ConstCostTable[] = {
715 { ISD::SDIV, MVT::v4i32, { 15 } }, // vpmuludq sequence
716 { ISD::SREM, MVT::v4i32, { 20 } }, // vpmuludq+mul+sub sequence
717 };
718
719 if (Op2Info.isConstant() && ST->hasSSE41())
720 if (const auto *Entry =
721 CostTableLookup(SSE41ConstCostTable, ISD, LT.second))
722 if (auto KindCost = Entry->Cost[CostKind])
723 return LT.first * *KindCost;
724
725 static const CostKindTblEntry SSE2ConstCostTable[] = {
726 { ISD::SDIV, MVT::v16i8, { 14 } }, // 2*ext+2*pmulhw sequence
727 { ISD::SREM, MVT::v16i8, { 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
728 { ISD::UDIV, MVT::v16i8, { 14 } }, // 2*ext+2*pmulhw sequence
729 { ISD::UREM, MVT::v16i8, { 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
730
731 { ISD::SDIV, MVT::v8i16, { 6 } }, // pmulhw sequence
732 { ISD::SREM, MVT::v8i16, { 8 } }, // pmulhw+mul+sub sequence
733 { ISD::UDIV, MVT::v8i16, { 6 } }, // pmulhuw sequence
734 { ISD::UREM, MVT::v8i16, { 8 } }, // pmulhuw+mul+sub sequence
735
736 { ISD::SDIV, MVT::v4i32, { 19 } }, // pmuludq sequence
737 { ISD::SREM, MVT::v4i32, { 24 } }, // pmuludq+mul+sub sequence
738 { ISD::UDIV, MVT::v4i32, { 15 } }, // pmuludq sequence
739 { ISD::UREM, MVT::v4i32, { 20 } }, // pmuludq+mul+sub sequence
740 };
741
742 if (Op2Info.isConstant() && ST->hasSSE2())
743 if (const auto *Entry = CostTableLookup(SSE2ConstCostTable, ISD, LT.second))
744 if (auto KindCost = Entry->Cost[CostKind])
745 return LT.first * *KindCost;
746
747 // rem matches div because the divider returns the remainder for free.
748 static const CostKindTblEntry ScalarVarDivCostTable[] = {
749 { ISD::SDIV, MVT::i8, { 15, 20, 2, 4 } },
750 { ISD::UDIV, MVT::i8, { 15, 20, 2, 4 } },
751 { ISD::SREM, MVT::i8, { 15, 20, 2, 4 } },
752 { ISD::UREM, MVT::i8, { 15, 20, 2, 4 } },
753 { ISD::SDIV, MVT::i16, { 17, 20, 2, 4 } },
754 { ISD::UDIV, MVT::i16, { 17, 20, 2, 4 } },
755 { ISD::SREM, MVT::i16, { 17, 20, 2, 4 } },
756 { ISD::UREM, MVT::i16, { 17, 20, 2, 4 } },
757 { ISD::SDIV, MVT::i32, { 25, 22, 2, 4 } },
758 { ISD::UDIV, MVT::i32, { 25, 22, 2, 4 } },
759 { ISD::SREM, MVT::i32, { 25, 22, 2, 4 } },
760 { ISD::UREM, MVT::i32, { 25, 22, 2, 4 } },
761 { ISD::SDIV, MVT::i64, { 41, 24, 2, 4 } },
762 { ISD::UDIV, MVT::i64, { 41, 24, 2, 4 } },
763 { ISD::SREM, MVT::i64, { 41, 24, 2, 4 } },
764 { ISD::UREM, MVT::i64, { 41, 24, 2, 4 } },
765 };
766
767 if (!LT.second.isVector() && !Op2Info.isConstant())
768 if (const auto *Entry =
769 CostTableLookup(ScalarVarDivCostTable, ISD, LT.second))
770 if (auto KindCost = Entry->Cost[CostKind])
771 return LT.first * *KindCost;
772
773 // Variable divisors lower through a float divide. strictfp needs SAE
774 // rounding which is 512-bit only.
775 bool IsStrictFP =
776 CtxI && CtxI->getFunction()->hasFnAttribute(Attribute::StrictFP);
777 bool IsDivRem = ISD == ISD::UDIV || ISD == ISD::SDIV || ISD == ISD::UREM ||
778 ISD == ISD::SREM;
779 bool VarDivToFP = IsDivRem && !Op2Info.isConstant() &&
780 (!IsStrictFP || ST->useAVX512Regs());
781
782 // i64 needs the qq converts, which are AVX512DQ only. Two tables because the
783 // lowering picks by operand value and not by type.
784 static const CostKindTblEntry AVX512DQExactVarDivCostTable[] = {
785 { ISD::UDIV, MVT::v2i64, { 5 } }, // cvt+divpd sequence
786 { ISD::SDIV, MVT::v2i64, { 5 } },
787 { ISD::UREM, MVT::v2i64, { 5 } },
788 { ISD::SREM, MVT::v2i64, { 5 } },
789 { ISD::UDIV, MVT::v4i64, { 8 } },
790 { ISD::SDIV, MVT::v4i64, { 8 } },
791 { ISD::UREM, MVT::v4i64, { 8 } },
792 { ISD::SREM, MVT::v4i64, { 8 } },
793 { ISD::UDIV, MVT::v8i64, { 16 } },
794 { ISD::SDIV, MVT::v8i64, { 16 } },
795 { ISD::UREM, MVT::v8i64, { 16 } },
796 { ISD::SREM, MVT::v8i64, { 16 } },
797 };
798
799 static const CostKindTblEntry AVX512DQVarDivCostTable[] = {
800 { ISD::UDIV, MVT::v2i64, { 16 } },
801 { ISD::SDIV, MVT::v2i64, { 16 } },
802 { ISD::UREM, MVT::v2i64, { 16 } },
803 { ISD::SREM, MVT::v2i64, { 16 } },
804 { ISD::UDIV, MVT::v4i64, { 16 } },
805 { ISD::SDIV, MVT::v4i64, { 16 } },
806 { ISD::UREM, MVT::v4i64, { 16 } },
807 { ISD::SREM, MVT::v4i64, { 16 } },
808 { ISD::UDIV, MVT::v8i64, { 16 } },
809 { ISD::SDIV, MVT::v8i64, { 18 } },
810 { ISD::UREM, MVT::v8i64, { 16 } },
811 { ISD::SREM, MVT::v8i64, { 18 } },
812 };
813
814 // The DAG combine picks between the two sequences with these same two
815 // queries, so the cost cannot disagree with what codegen emits.
816 bool IsSignedDiv = ISD == ISD::SDIV || ISD == ISD::SREM;
817 auto OperandsFit = [&](unsigned Mantissa) {
818 if (Args.size() != 2 || !CtxI)
819 return false;
820 unsigned EltBits = LT.second.getScalarSizeInBits();
821 const DataLayout &DL = CtxI->getDataLayout();
822 auto Fits = [&](const Value *V) {
823 if (IsSignedDiv)
824 return ComputeNumSignBits(V, DL, /*AC=*/nullptr, CtxI) + Mantissa >
825 EltBits;
826 return computeKnownBits(V, DL, /*AC=*/nullptr, CtxI)
827 .countMaxActiveBits() <= Mantissa;
828 };
829 return Fits(Args[0]) && Fits(Args[1]);
830 };
831
832 // An i32 divide goes through f32 when both operands fit in 24 bits, and
833 // through f64 at twice the vector width when they do not.
834 bool ExactI32 =
835 VarDivToFP && LT.second.getScalarType() == MVT::i32 &&
837
838 if (VarDivToFP && ST->hasDQI() && ST->useAVX512Regs() &&
839 LT.second.getScalarType() == MVT::i64) {
840 bool ExactFPDiv =
842
843 // Only the reciprocal chain multiplies, so only it is vpmullq gated.
844 bool SlowMultiply =
845 !ExactFPDiv && LT.second == MVT::v2i64 && ST->isPMULLQSlow();
846
847 if (!SlowMultiply) {
849 ExactFPDiv ? AVX512DQExactVarDivCostTable : AVX512DQVarDivCostTable;
850 if (const auto *Entry = CostTableLookup(Tbl, ISD, LT.second))
851 if (auto KindCost = Entry->Cost[CostKind])
852 return LT.first * *KindCost;
853 }
854 }
855
856 static const CostKindTblEntry AVX512BWVarDivCostTable[] = {
857 { ISD::UDIV, MVT::v16i8, { 10 } }, // unpack+cvt+divps sequence
858 { ISD::SDIV, MVT::v16i8, { 10 } },
859 { ISD::UREM, MVT::v16i8, { 10 } },
860 { ISD::SREM, MVT::v16i8, { 10 } },
861 { ISD::UDIV, MVT::v32i8, { 20 } },
862 { ISD::SDIV, MVT::v32i8, { 20 } },
863 { ISD::UREM, MVT::v32i8, { 20 } },
864 { ISD::SREM, MVT::v32i8, { 20 } },
865 { ISD::UDIV, MVT::v64i8, { 40 } },
866 { ISD::SDIV, MVT::v64i8, { 40 } },
867 { ISD::UREM, MVT::v64i8, { 40 } },
868 { ISD::SREM, MVT::v64i8, { 40 } },
869 { ISD::UDIV, MVT::v8i16, { 5 } },
870 { ISD::SDIV, MVT::v8i16, { 5 } },
871 { ISD::UREM, MVT::v8i16, { 5 } },
872 { ISD::SREM, MVT::v8i16, { 5 } },
873 { ISD::UDIV, MVT::v16i16, { 10 } },
874 { ISD::SDIV, MVT::v16i16, { 10 } },
875 { ISD::UREM, MVT::v16i16, { 10 } },
876 { ISD::SREM, MVT::v16i16, { 10 } },
877 { ISD::UDIV, MVT::v32i16, { 20 } },
878 { ISD::SDIV, MVT::v32i16, { 20 } },
879 { ISD::UREM, MVT::v32i16, { 20 } },
880 { ISD::SREM, MVT::v32i16, { 20 } },
881 { ISD::UDIV, MVT::v4i32, { 8 } }, // cvt+divpd sequence
882 { ISD::SDIV, MVT::v4i32, { 8 } },
883 { ISD::UREM, MVT::v4i32, { 8 } },
884 { ISD::SREM, MVT::v4i32, { 8 } },
885 { ISD::UDIV, MVT::v8i32, { 16 } },
886 { ISD::SDIV, MVT::v8i32, { 16 } },
887 { ISD::UREM, MVT::v8i32, { 16 } },
888 { ISD::SREM, MVT::v8i32, { 16 } },
889 { ISD::UDIV, MVT::v16i32, { 32 } },
890 { ISD::SDIV, MVT::v16i32, { 32 } },
891 { ISD::UREM, MVT::v16i32, { 32 } },
892 { ISD::SREM, MVT::v16i32, { 32 } },
893 };
894
895 static const CostKindTblEntry AVX512BWExactVarDivCostTable[] = {
896 { ISD::UDIV, MVT::v4i32, { 3 } }, // cvt+divps sequence
897 { ISD::SDIV, MVT::v4i32, { 3 } },
898 { ISD::UREM, MVT::v4i32, { 3 } },
899 { ISD::SREM, MVT::v4i32, { 3 } },
900 { ISD::UDIV, MVT::v8i32, { 5 } },
901 { ISD::SDIV, MVT::v8i32, { 5 } },
902 { ISD::UREM, MVT::v8i32, { 5 } },
903 { ISD::SREM, MVT::v8i32, { 5 } },
904 { ISD::UDIV, MVT::v16i32, { 10 } },
905 { ISD::SDIV, MVT::v16i32, { 10 } },
906 { ISD::UREM, MVT::v16i32, { 10 } },
907 { ISD::SREM, MVT::v16i32, { 10 } },
908 };
909
910 if (VarDivToFP && ST->hasBWI()) {
911 ArrayRef<CostKindTblEntry> Tbl = AVX512BWVarDivCostTable;
912 if (ExactI32)
913 Tbl = AVX512BWExactVarDivCostTable;
914 if (const auto *Entry = CostTableLookup(Tbl, ISD, LT.second))
915 if (auto KindCost = Entry->Cost[CostKind])
916 return LT.first * *KindCost;
917 }
918
919 static const CostKindTblEntry AVX512VarDivCostTable[] = {
920 { ISD::UDIV, MVT::v16i8, { 14 } }, // unpack+cvt+divps sequence
921 { ISD::SDIV, MVT::v16i8, { 14 } },
922 { ISD::UREM, MVT::v16i8, { 14 } },
923 { ISD::SREM, MVT::v16i8, { 14 } },
924 { ISD::UDIV, MVT::v32i8, { 28 } },
925 { ISD::SDIV, MVT::v32i8, { 28 } },
926 { ISD::UREM, MVT::v32i8, { 28 } },
927 { ISD::SREM, MVT::v32i8, { 28 } },
928 { ISD::UDIV, MVT::v64i8, { 56 } },
929 { ISD::SDIV, MVT::v64i8, { 56 } },
930 { ISD::UREM, MVT::v64i8, { 56 } },
931 { ISD::SREM, MVT::v64i8, { 56 } },
932 { ISD::UDIV, MVT::v8i16, { 14 } },
933 { ISD::SDIV, MVT::v8i16, { 14 } },
934 { ISD::UREM, MVT::v8i16, { 14 } },
935 { ISD::SREM, MVT::v8i16, { 14 } },
936 { ISD::UDIV, MVT::v16i16, { 14 } },
937 { ISD::SDIV, MVT::v16i16, { 14 } },
938 { ISD::UREM, MVT::v16i16, { 14 } },
939 { ISD::SREM, MVT::v16i16, { 14 } },
940 { ISD::UDIV, MVT::v32i16, { 28 } },
941 { ISD::SDIV, MVT::v32i16, { 28 } },
942 { ISD::UREM, MVT::v32i16, { 28 } },
943 { ISD::SREM, MVT::v32i16, { 28 } },
944 { ISD::UDIV, MVT::v4i32, { 28 } }, // cvt+divpd sequence
945 { ISD::SDIV, MVT::v4i32, { 28 } },
946 { ISD::UREM, MVT::v4i32, { 28 } },
947 { ISD::SREM, MVT::v4i32, { 28 } },
948 { ISD::UDIV, MVT::v8i32, { 28 } },
949 { ISD::SDIV, MVT::v8i32, { 28 } },
950 { ISD::UREM, MVT::v8i32, { 28 } },
951 { ISD::SREM, MVT::v8i32, { 28 } },
952 { ISD::UDIV, MVT::v16i32, { 56 } },
953 { ISD::SDIV, MVT::v16i32, { 56 } },
954 { ISD::UREM, MVT::v16i32, { 56 } },
955 { ISD::SREM, MVT::v16i32, { 56 } },
956 };
957
958 static const CostKindTblEntry AVX512ExactVarDivCostTable[] = {
959 { ISD::UDIV, MVT::v4i32, { 7 } }, // cvt+divps sequence
960 { ISD::SDIV, MVT::v4i32, { 7 } },
961 { ISD::UREM, MVT::v4i32, { 7 } },
962 { ISD::SREM, MVT::v4i32, { 7 } },
963 { ISD::UDIV, MVT::v8i32, { 14 } },
964 { ISD::SDIV, MVT::v8i32, { 14 } },
965 { ISD::UREM, MVT::v8i32, { 14 } },
966 { ISD::SREM, MVT::v8i32, { 14 } },
967 { ISD::UDIV, MVT::v16i32, { 14 } },
968 { ISD::SDIV, MVT::v16i32, { 14 } },
969 { ISD::UREM, MVT::v16i32, { 14 } },
970 { ISD::SREM, MVT::v16i32, { 14 } },
971 };
972
973 if (VarDivToFP && ST->hasAVX512()) {
974 ArrayRef<CostKindTblEntry> Tbl = AVX512VarDivCostTable;
975 if (ExactI32)
976 Tbl = AVX512ExactVarDivCostTable;
977 if (const auto *Entry = CostTableLookup(Tbl, ISD, LT.second))
978 if (auto KindCost = Entry->Cost[CostKind])
979 return LT.first * *KindCost;
980 }
981
982 static const CostKindTblEntry AVX2VarDivCostTable[] = {
983 { ISD::UDIV, MVT::v16i8, { 28 } }, // unpack+cvt+divps sequence
984 { ISD::SDIV, MVT::v16i8, { 28 } },
985 { ISD::UREM, MVT::v16i8, { 28 } },
986 { ISD::SREM, MVT::v16i8, { 28 } },
987 { ISD::UDIV, MVT::v32i8, { 56 } },
988 { ISD::SDIV, MVT::v32i8, { 56 } },
989 { ISD::UREM, MVT::v32i8, { 56 } },
990 { ISD::SREM, MVT::v32i8, { 56 } },
991 { ISD::UDIV, MVT::v8i16, { 14 } },
992 { ISD::SDIV, MVT::v8i16, { 14 } },
993 { ISD::UREM, MVT::v8i16, { 14 } },
994 { ISD::SREM, MVT::v8i16, { 14 } },
995 { ISD::UDIV, MVT::v16i16, { 28 } },
996 { ISD::SDIV, MVT::v16i16, { 28 } },
997 { ISD::UREM, MVT::v16i16, { 28 } },
998 { ISD::SREM, MVT::v16i16, { 28 } },
999 { ISD::UDIV, MVT::v4i32, { 28 } }, // cvt+divpd sequence
1000 { ISD::SDIV, MVT::v4i32, { 28 } },
1001 { ISD::UREM, MVT::v4i32, { 28 } },
1002 { ISD::SREM, MVT::v4i32, { 28 } },
1003 { ISD::UDIV, MVT::v8i32, { 56 } },
1004 { ISD::SDIV, MVT::v8i32, { 56 } },
1005 { ISD::UREM, MVT::v8i32, { 56 } },
1006 { ISD::SREM, MVT::v8i32, { 56 } },
1007 };
1008
1009 static const CostKindTblEntry AVX2ExactVarDivCostTable[] = {
1010 { ISD::UDIV, MVT::v4i32, { 9 } }, // cvt+divps sequence
1011 { ISD::SDIV, MVT::v4i32, { 8 } },
1012 { ISD::UREM, MVT::v4i32, { 9 } },
1013 { ISD::SREM, MVT::v4i32, { 8 } },
1014 { ISD::UDIV, MVT::v8i32, { 14 } },
1015 { ISD::SDIV, MVT::v8i32, { 14 } },
1016 { ISD::UREM, MVT::v8i32, { 14 } },
1017 { ISD::SREM, MVT::v8i32, { 14 } },
1018 };
1019
1020 if (VarDivToFP && ST->hasAVX2()) {
1021 ArrayRef<CostKindTblEntry> Tbl = AVX2VarDivCostTable;
1022 if (ExactI32)
1023 Tbl = AVX2ExactVarDivCostTable;
1024 if (const auto *Entry = CostTableLookup(Tbl, ISD, LT.second))
1025 if (auto KindCost = Entry->Cost[CostKind])
1026 return LT.first * *KindCost;
1027 }
1028
1029 // No unsigned i32 entries below AVX2, where the u32 to f64 converts are
1030 // emulated and the fold stays off.
1031 static const CostKindTblEntry AVX1VarDivCostTable[] = {
1032 { ISD::UDIV, MVT::v16i8, { 56 } }, // unpack+cvt+divps sequence
1033 { ISD::SDIV, MVT::v16i8, { 56 } },
1034 { ISD::UREM, MVT::v16i8, { 56 } },
1035 { ISD::SREM, MVT::v16i8, { 56 } },
1036 { ISD::UDIV, MVT::v32i8, { 112 } },
1037 { ISD::SDIV, MVT::v32i8, { 112 } },
1038 { ISD::UREM, MVT::v32i8, { 112 } },
1039 { ISD::SREM, MVT::v32i8, { 112 } },
1040 { ISD::UDIV, MVT::v8i16, { 28 } },
1041 { ISD::SDIV, MVT::v8i16, { 28 } },
1042 { ISD::UREM, MVT::v8i16, { 28 } },
1043 { ISD::SREM, MVT::v8i16, { 28 } },
1044 { ISD::UDIV, MVT::v16i16, { 56 } },
1045 { ISD::SDIV, MVT::v16i16, { 56 } },
1046 { ISD::UREM, MVT::v16i16, { 56 } },
1047 { ISD::SREM, MVT::v16i16, { 56 } },
1048 { ISD::SDIV, MVT::v4i32, { 44 } }, // cvt+divpd sequence
1049 { ISD::SREM, MVT::v4i32, { 44 } },
1050 { ISD::SDIV, MVT::v8i32, { 88 } },
1051 { ISD::SREM, MVT::v8i32, { 88 } },
1052 };
1053
1054 static const CostKindTblEntry AVX1ExactVarDivCostTable[] = {
1055 { ISD::SDIV, MVT::v4i32, { 14 } }, // cvt+divps sequence
1056 { ISD::SREM, MVT::v4i32, { 14 } },
1057 { ISD::SDIV, MVT::v8i32, { 28 } },
1058 { ISD::SREM, MVT::v8i32, { 28 } },
1059 };
1060
1061 if (VarDivToFP && ST->hasAVX()) {
1062 ArrayRef<CostKindTblEntry> Tbl = AVX1VarDivCostTable;
1063 if (ExactI32)
1064 Tbl = AVX1ExactVarDivCostTable;
1065 if (const auto *Entry = CostTableLookup(Tbl, ISD, LT.second))
1066 if (auto KindCost = Entry->Cost[CostKind])
1067 return LT.first * *KindCost;
1068 }
1069
1070 static const CostKindTblEntry SSE2VarDivCostTable[] = {
1071 { ISD::UDIV, MVT::v16i8, { 56 } }, // unpack+cvt+divps sequence
1072 { ISD::SDIV, MVT::v16i8, { 56 } },
1073 { ISD::UREM, MVT::v16i8, { 56 } },
1074 { ISD::SREM, MVT::v16i8, { 56 } },
1075 { ISD::UDIV, MVT::v8i16, { 28 } },
1076 { ISD::SDIV, MVT::v8i16, { 28 } },
1077 { ISD::UREM, MVT::v8i16, { 28 } },
1078 { ISD::SREM, MVT::v8i16, { 28 } },
1079 { ISD::SDIV, MVT::v4i32, { 44 } }, // cvt+divpd sequence
1080 { ISD::SREM, MVT::v4i32, { 44 } },
1081 };
1082
1083 static const CostKindTblEntry SSE2ExactVarDivCostTable[] = {
1084 { ISD::SDIV, MVT::v4i32, { 14 } }, // cvt+divps sequence
1085 { ISD::SREM, MVT::v4i32, { 14 } },
1086 };
1087
1088 if (VarDivToFP && ST->hasSSE2()) {
1089 ArrayRef<CostKindTblEntry> Tbl = SSE2VarDivCostTable;
1090 if (ExactI32)
1091 Tbl = SSE2ExactVarDivCostTable;
1092 if (const auto *Entry = CostTableLookup(Tbl, ISD, LT.second))
1093 if (auto KindCost = Entry->Cost[CostKind])
1094 return LT.first * *KindCost;
1095 }
1096
1097 static const CostKindTblEntry AVX512BWUniformCostTable[] = {
1098 { ISD::SHL, MVT::v16i8, { 3, 5, 5, 7 } }, // psllw + pand.
1099 { ISD::SRL, MVT::v16i8, { 3,10, 5, 8 } }, // psrlw + pand.
1100 { ISD::SRA, MVT::v16i8, { 4,12, 8,12 } }, // psrlw, pand, pxor, psubb.
1101 { ISD::SHL, MVT::v32i8, { 4, 7, 6, 8 } }, // psllw + pand.
1102 { ISD::SRL, MVT::v32i8, { 4, 8, 7, 9 } }, // psrlw + pand.
1103 { ISD::SRA, MVT::v32i8, { 5,10,10,13 } }, // psrlw, pand, pxor, psubb.
1104 { ISD::SHL, MVT::v64i8, { 4, 7, 6, 8 } }, // psllw + pand.
1105 { ISD::SRL, MVT::v64i8, { 4, 8, 7,10 } }, // psrlw + pand.
1106 { ISD::SRA, MVT::v64i8, { 5,10,10,15 } }, // psrlw, pand, pxor, psubb.
1107
1108 { ISD::SHL, MVT::v32i16, { 2, 4, 2, 3 } }, // psllw
1109 { ISD::SRL, MVT::v32i16, { 2, 4, 2, 3 } }, // psrlw
1110 { ISD::SRA, MVT::v32i16, { 2, 4, 2, 3 } }, // psrqw
1111 };
1112
1113 if (ST->hasBWI() && Op2Info.isUniform())
1114 if (const auto *Entry =
1115 CostTableLookup(AVX512BWUniformCostTable, ISD, LT.second))
1116 if (auto KindCost = Entry->Cost[CostKind])
1117 return LT.first * *KindCost;
1118
1119 static const CostKindTblEntry AVX512UniformCostTable[] = {
1120 { ISD::SHL, MVT::v32i16, { 5,10, 5, 7 } }, // psllw + split.
1121 { ISD::SRL, MVT::v32i16, { 5,10, 5, 7 } }, // psrlw + split.
1122 { ISD::SRA, MVT::v32i16, { 5,10, 5, 7 } }, // psraw + split.
1123
1124 { ISD::SHL, MVT::v16i32, { 2, 4, 2, 3 } }, // pslld
1125 { ISD::SRL, MVT::v16i32, { 2, 4, 2, 3 } }, // psrld
1126 { ISD::SRA, MVT::v16i32, { 2, 4, 2, 3 } }, // psrad
1127
1128 { ISD::SRA, MVT::v2i64, { 1, 2, 1, 2 } }, // psraq
1129 { ISD::SHL, MVT::v4i64, { 1, 4, 1, 2 } }, // psllq
1130 { ISD::SRL, MVT::v4i64, { 1, 4, 1, 2 } }, // psrlq
1131 { ISD::SRA, MVT::v4i64, { 1, 4, 1, 2 } }, // psraq
1132 { ISD::SHL, MVT::v8i64, { 1, 4, 1, 2 } }, // psllq
1133 { ISD::SRL, MVT::v8i64, { 1, 4, 1, 2 } }, // psrlq
1134 { ISD::SRA, MVT::v8i64, { 1, 4, 1, 2 } }, // psraq
1135 };
1136
1137 if (ST->hasAVX512() && Op2Info.isUniform())
1138 if (const auto *Entry =
1139 CostTableLookup(AVX512UniformCostTable, ISD, LT.second))
1140 if (auto KindCost = Entry->Cost[CostKind])
1141 return LT.first * *KindCost;
1142
1143 static const CostKindTblEntry AVX2UniformCostTable[] = {
1144 // Uniform splats are cheaper for the following instructions.
1145 { ISD::SHL, MVT::v16i8, { 3, 5, 5, 7 } }, // psllw + pand.
1146 { ISD::SRL, MVT::v16i8, { 3, 9, 5, 8 } }, // psrlw + pand.
1147 { ISD::SRA, MVT::v16i8, { 4, 5, 9,13 } }, // psrlw, pand, pxor, psubb.
1148 { ISD::SHL, MVT::v32i8, { 4, 7, 6, 8 } }, // psllw + pand.
1149 { ISD::SRL, MVT::v32i8, { 4, 8, 7, 9 } }, // psrlw + pand.
1150 { ISD::SRA, MVT::v32i8, { 6, 9,11,16 } }, // psrlw, pand, pxor, psubb.
1151
1152 { ISD::SHL, MVT::v8i16, { 1, 2, 1, 2 } }, // psllw.
1153 { ISD::SRL, MVT::v8i16, { 1, 2, 1, 2 } }, // psrlw.
1154 { ISD::SRA, MVT::v8i16, { 1, 2, 1, 2 } }, // psraw.
1155 { ISD::SHL, MVT::v16i16, { 2, 4, 2, 3 } }, // psllw.
1156 { ISD::SRL, MVT::v16i16, { 2, 4, 2, 3 } }, // psrlw.
1157 { ISD::SRA, MVT::v16i16, { 2, 4, 2, 3 } }, // psraw.
1158
1159 { ISD::SHL, MVT::v4i32, { 1, 2, 1, 2 } }, // pslld
1160 { ISD::SRL, MVT::v4i32, { 1, 2, 1, 2 } }, // psrld
1161 { ISD::SRA, MVT::v4i32, { 1, 2, 1, 2 } }, // psrad
1162 { ISD::SHL, MVT::v8i32, { 2, 4, 2, 3 } }, // pslld
1163 { ISD::SRL, MVT::v8i32, { 2, 4, 2, 3 } }, // psrld
1164 { ISD::SRA, MVT::v8i32, { 2, 4, 2, 3 } }, // psrad
1165
1166 { ISD::SHL, MVT::v2i64, { 1, 2, 1, 2 } }, // psllq
1167 { ISD::SRL, MVT::v2i64, { 1, 2, 1, 2 } }, // psrlq
1168 { ISD::SRA, MVT::v2i64, { 2, 4, 5, 7 } }, // 2 x psrad + shuffle.
1169 { ISD::SHL, MVT::v4i64, { 2, 4, 1, 2 } }, // psllq
1170 { ISD::SRL, MVT::v4i64, { 2, 4, 1, 2 } }, // psrlq
1171 { ISD::SRA, MVT::v4i64, { 4, 6, 5, 9 } }, // 2 x psrad + shuffle.
1172 };
1173
1174 if (ST->hasAVX2() && Op2Info.isUniform())
1175 if (const auto *Entry =
1176 CostTableLookup(AVX2UniformCostTable, ISD, LT.second))
1177 if (auto KindCost = Entry->Cost[CostKind])
1178 return LT.first * *KindCost;
1179
1180 static const CostKindTblEntry AVXUniformCostTable[] = {
1181 { ISD::SHL, MVT::v16i8, { 4, 4, 6, 8 } }, // psllw + pand.
1182 { ISD::SRL, MVT::v16i8, { 4, 8, 5, 8 } }, // psrlw + pand.
1183 { ISD::SRA, MVT::v16i8, { 6, 6, 9,13 } }, // psrlw, pand, pxor, psubb.
1184 { ISD::SHL, MVT::v32i8, { 7, 8,11,14 } }, // psllw + pand + split.
1185 { ISD::SRL, MVT::v32i8, { 7, 9,10,14 } }, // psrlw + pand + split.
1186 { ISD::SRA, MVT::v32i8, { 10,11,16,21 } }, // psrlw, pand, pxor, psubb + split.
1187
1188 { ISD::SHL, MVT::v8i16, { 1, 3, 1, 2 } }, // psllw.
1189 { ISD::SRL, MVT::v8i16, { 1, 3, 1, 2 } }, // psrlw.
1190 { ISD::SRA, MVT::v8i16, { 1, 3, 1, 2 } }, // psraw.
1191 { ISD::SHL, MVT::v16i16, { 3, 7, 5, 7 } }, // psllw + split.
1192 { ISD::SRL, MVT::v16i16, { 3, 7, 5, 7 } }, // psrlw + split.
1193 { ISD::SRA, MVT::v16i16, { 3, 7, 5, 7 } }, // psraw + split.
1194
1195 { ISD::SHL, MVT::v4i32, { 1, 3, 1, 2 } }, // pslld.
1196 { ISD::SRL, MVT::v4i32, { 1, 3, 1, 2 } }, // psrld.
1197 { ISD::SRA, MVT::v4i32, { 1, 3, 1, 2 } }, // psrad.
1198 { ISD::SHL, MVT::v8i32, { 3, 7, 5, 7 } }, // pslld + split.
1199 { ISD::SRL, MVT::v8i32, { 3, 7, 5, 7 } }, // psrld + split.
1200 { ISD::SRA, MVT::v8i32, { 3, 7, 5, 7 } }, // psrad + split.
1201
1202 { ISD::SHL, MVT::v2i64, { 1, 3, 1, 2 } }, // psllq.
1203 { ISD::SRL, MVT::v2i64, { 1, 3, 1, 2 } }, // psrlq.
1204 { ISD::SRA, MVT::v2i64, { 3, 4, 5, 7 } }, // 2 x psrad + shuffle.
1205 { ISD::SHL, MVT::v4i64, { 3, 7, 4, 6 } }, // psllq + split.
1206 { ISD::SRL, MVT::v4i64, { 3, 7, 4, 6 } }, // psrlq + split.
1207 { ISD::SRA, MVT::v4i64, { 6, 7,10,13 } }, // 2 x (2 x psrad + shuffle) + split.
1208 };
1209
1210 // XOP has faster vXi8 shifts.
1211 if (ST->hasAVX() && Op2Info.isUniform() &&
1212 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
1213 if (const auto *Entry =
1214 CostTableLookup(AVXUniformCostTable, ISD, LT.second))
1215 if (auto KindCost = Entry->Cost[CostKind])
1216 return LT.first * *KindCost;
1217
1218 static const CostKindTblEntry SSE2UniformCostTable[] = {
1219 // Uniform splats are cheaper for the following instructions.
1220 { ISD::SHL, MVT::v16i8, { 9, 10, 6, 9 } }, // psllw + pand.
1221 { ISD::SRL, MVT::v16i8, { 9, 13, 5, 9 } }, // psrlw + pand.
1222 { ISD::SRA, MVT::v16i8, { 11, 15, 9,13 } }, // pcmpgtb sequence.
1223
1224 { ISD::SHL, MVT::v8i16, { 2, 2, 1, 2 } }, // psllw.
1225 { ISD::SRL, MVT::v8i16, { 2, 2, 1, 2 } }, // psrlw.
1226 { ISD::SRA, MVT::v8i16, { 2, 2, 1, 2 } }, // psraw.
1227
1228 { ISD::SHL, MVT::v4i32, { 2, 2, 1, 2 } }, // pslld
1229 { ISD::SRL, MVT::v4i32, { 2, 2, 1, 2 } }, // psrld.
1230 { ISD::SRA, MVT::v4i32, { 2, 2, 1, 2 } }, // psrad.
1231
1232 { ISD::SHL, MVT::v2i64, { 2, 2, 1, 2 } }, // psllq.
1233 { ISD::SRL, MVT::v2i64, { 2, 2, 1, 2 } }, // psrlq.
1234 { ISD::SRA, MVT::v2i64, { 5, 9, 5, 7 } }, // 2*psrlq + xor + sub.
1235 };
1236
1237 if (ST->hasSSE2() && Op2Info.isUniform() &&
1238 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
1239 if (const auto *Entry =
1240 CostTableLookup(SSE2UniformCostTable, ISD, LT.second))
1241 if (auto KindCost = Entry->Cost[CostKind])
1242 return LT.first * *KindCost;
1243
1244 static const CostKindTblEntry AVX512DQCostTable[] = {
1245 { ISD::MUL, MVT::v2i64, { 2, 15, 1, 3 } }, // pmullq
1246 { ISD::MUL, MVT::v4i64, { 2, 15, 1, 3 } }, // pmullq
1247 { ISD::MUL, MVT::v8i64, { 3, 15, 1, 3 } } // pmullq
1248 };
1249
1250 // Look for AVX512DQ lowering tricks for custom cases.
1251 if (ST->hasDQI())
1252 if (const auto *Entry = CostTableLookup(AVX512DQCostTable, ISD, LT.second))
1253 if (auto KindCost = Entry->Cost[CostKind])
1254 return LT.first * *KindCost;
1255
1256 static const CostKindTblEntry AVX512BWCostTable[] = {
1257 { ISD::SHL, MVT::v16i8, { 4, 8, 4, 5 } }, // extend/vpsllvw/pack sequence.
1258 { ISD::SRL, MVT::v16i8, { 4, 8, 4, 5 } }, // extend/vpsrlvw/pack sequence.
1259 { ISD::SRA, MVT::v16i8, { 4, 8, 4, 5 } }, // extend/vpsravw/pack sequence.
1260 { ISD::SHL, MVT::v32i8, { 4, 23,11,16 } }, // extend/vpsllvw/pack sequence.
1261 { ISD::SRL, MVT::v32i8, { 4, 30,12,18 } }, // extend/vpsrlvw/pack sequence.
1262 { ISD::SRA, MVT::v32i8, { 6, 13,24,30 } }, // extend/vpsravw/pack sequence.
1263 { ISD::SHL, MVT::v64i8, { 6, 19,13,15 } }, // extend/vpsllvw/pack sequence.
1264 { ISD::SRL, MVT::v64i8, { 7, 27,15,18 } }, // extend/vpsrlvw/pack sequence.
1265 { ISD::SRA, MVT::v64i8, { 15, 15,30,30 } }, // extend/vpsravw/pack sequence.
1266
1267 { ISD::SHL, MVT::v8i16, { 1, 1, 1, 1 } }, // vpsllvw
1268 { ISD::SRL, MVT::v8i16, { 1, 1, 1, 1 } }, // vpsrlvw
1269 { ISD::SRA, MVT::v8i16, { 1, 1, 1, 1 } }, // vpsravw
1270 { ISD::SHL, MVT::v16i16, { 1, 1, 1, 1 } }, // vpsllvw
1271 { ISD::SRL, MVT::v16i16, { 1, 1, 1, 1 } }, // vpsrlvw
1272 { ISD::SRA, MVT::v16i16, { 1, 1, 1, 1 } }, // vpsravw
1273 { ISD::SHL, MVT::v32i16, { 1, 1, 1, 1 } }, // vpsllvw
1274 { ISD::SRL, MVT::v32i16, { 1, 1, 1, 1 } }, // vpsrlvw
1275 { ISD::SRA, MVT::v32i16, { 1, 1, 1, 1 } }, // vpsravw
1276
1277 { ISD::ADD, MVT::v64i8, { 1, 1, 1, 1 } }, // paddb
1278 { ISD::ADD, MVT::v32i16, { 1, 1, 1, 1 } }, // paddw
1279
1280 { ISD::ADD, MVT::v32i8, { 1, 1, 1, 1 } }, // paddb
1281 { ISD::ADD, MVT::v16i16, { 1, 1, 1, 1 } }, // paddw
1282 { ISD::ADD, MVT::v8i32, { 1, 1, 1, 1 } }, // paddd
1283 { ISD::ADD, MVT::v4i64, { 1, 1, 1, 1 } }, // paddq
1284
1285 { ISD::SUB, MVT::v64i8, { 1, 1, 1, 1 } }, // psubb
1286 { ISD::SUB, MVT::v32i16, { 1, 1, 1, 1 } }, // psubw
1287
1288 { ISD::MUL, MVT::v16i8, { 4, 12, 4, 5 } }, // extend/pmullw/trunc
1289 { ISD::MUL, MVT::v32i8, { 3, 10, 7,10 } }, // pmaddubsw
1290 { ISD::MUL, MVT::v64i8, { 3, 11, 7,10 } }, // pmaddubsw
1291 { ISD::MUL, MVT::v32i16, { 1, 5, 1, 1 } }, // pmullw
1292
1293 { ISD::SUB, MVT::v32i8, { 1, 1, 1, 1 } }, // psubb
1294 { ISD::SUB, MVT::v16i16, { 1, 1, 1, 1 } }, // psubw
1295 { ISD::SUB, MVT::v8i32, { 1, 1, 1, 1 } }, // psubd
1296 { ISD::SUB, MVT::v4i64, { 1, 1, 1, 1 } }, // psubq
1297 };
1298
1299 // Look for AVX512BW lowering tricks for custom cases.
1300 if (ST->hasBWI())
1301 if (const auto *Entry = CostTableLookup(AVX512BWCostTable, ISD, LT.second))
1302 if (auto KindCost = Entry->Cost[CostKind])
1303 return LT.first * *KindCost;
1304
1305 static const CostKindTblEntry AVX512CostTable[] = {
1306 { ISD::SHL, MVT::v64i8, { 15, 19,27,33 } }, // vpblendv+split sequence.
1307 { ISD::SRL, MVT::v64i8, { 15, 19,30,36 } }, // vpblendv+split sequence.
1308 { ISD::SRA, MVT::v64i8, { 37, 37,51,63 } }, // vpblendv+split sequence.
1309
1310 { ISD::SHL, MVT::v32i16, { 11, 16,11,15 } }, // 2*extend/vpsrlvd/pack sequence.
1311 { ISD::SRL, MVT::v32i16, { 11, 16,11,15 } }, // 2*extend/vpsrlvd/pack sequence.
1312 { ISD::SRA, MVT::v32i16, { 11, 16,11,15 } }, // 2*extend/vpsravd/pack sequence.
1313
1314 { ISD::SHL, MVT::v4i32, { 1, 1, 1, 1 } },
1315 { ISD::SRL, MVT::v4i32, { 1, 1, 1, 1 } },
1316 { ISD::SRA, MVT::v4i32, { 1, 1, 1, 1 } },
1317 { ISD::SHL, MVT::v8i32, { 1, 1, 1, 1 } },
1318 { ISD::SRL, MVT::v8i32, { 1, 1, 1, 1 } },
1319 { ISD::SRA, MVT::v8i32, { 1, 1, 1, 1 } },
1320 { ISD::SHL, MVT::v16i32, { 1, 1, 1, 1 } },
1321 { ISD::SRL, MVT::v16i32, { 1, 1, 1, 1 } },
1322 { ISD::SRA, MVT::v16i32, { 1, 1, 1, 1 } },
1323
1324 { ISD::SHL, MVT::v2i64, { 1, 1, 1, 1 } },
1325 { ISD::SRL, MVT::v2i64, { 1, 1, 1, 1 } },
1326 { ISD::SRA, MVT::v2i64, { 1, 1, 1, 1 } },
1327 { ISD::SHL, MVT::v4i64, { 1, 1, 1, 1 } },
1328 { ISD::SRL, MVT::v4i64, { 1, 1, 1, 1 } },
1329 { ISD::SRA, MVT::v4i64, { 1, 1, 1, 1 } },
1330 { ISD::SHL, MVT::v8i64, { 1, 1, 1, 1 } },
1331 { ISD::SRL, MVT::v8i64, { 1, 1, 1, 1 } },
1332 { ISD::SRA, MVT::v8i64, { 1, 1, 1, 1 } },
1333
1334 { ISD::ADD, MVT::v64i8, { 3, 7, 5, 5 } }, // 2*paddb + split
1335 { ISD::ADD, MVT::v32i16, { 3, 7, 5, 5 } }, // 2*paddw + split
1336
1337 { ISD::SUB, MVT::v64i8, { 3, 7, 5, 5 } }, // 2*psubb + split
1338 { ISD::SUB, MVT::v32i16, { 3, 7, 5, 5 } }, // 2*psubw + split
1339
1340 { ISD::AND, MVT::v32i8, { 1, 1, 1, 1 } },
1341 { ISD::AND, MVT::v16i16, { 1, 1, 1, 1 } },
1342 { ISD::AND, MVT::v8i32, { 1, 1, 1, 1 } },
1343 { ISD::AND, MVT::v4i64, { 1, 1, 1, 1 } },
1344
1345 { ISD::OR, MVT::v32i8, { 1, 1, 1, 1 } },
1346 { ISD::OR, MVT::v16i16, { 1, 1, 1, 1 } },
1347 { ISD::OR, MVT::v8i32, { 1, 1, 1, 1 } },
1348 { ISD::OR, MVT::v4i64, { 1, 1, 1, 1 } },
1349
1350 { ISD::XOR, MVT::v32i8, { 1, 1, 1, 1 } },
1351 { ISD::XOR, MVT::v16i16, { 1, 1, 1, 1 } },
1352 { ISD::XOR, MVT::v8i32, { 1, 1, 1, 1 } },
1353 { ISD::XOR, MVT::v4i64, { 1, 1, 1, 1 } },
1354
1355 { ISD::MUL, MVT::v16i32, { 1, 10, 1, 2 } }, // pmulld (Skylake from agner.org)
1356 { ISD::MUL, MVT::v8i32, { 1, 10, 1, 2 } }, // pmulld (Skylake from agner.org)
1357 { ISD::MUL, MVT::v4i32, { 1, 10, 1, 2 } }, // pmulld (Skylake from agner.org)
1358 { ISD::MUL, MVT::v8i64, { 6, 9, 8, 8 } }, // 3*pmuludq/3*shift/2*add
1359 { ISD::MUL, MVT::i64, { 1 } }, // Skylake from http://www.agner.org/
1360
1361 { X86ISD::PMULUDQ, MVT::v8i64, { 1, 5, 1, 1 } },
1362
1363 { ISD::FNEG, MVT::v8f64, { 1, 1, 1, 2 } }, // Skylake from http://www.agner.org/
1364 { ISD::FADD, MVT::v8f64, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1365 { ISD::FADD, MVT::v4f64, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1366 { ISD::FSUB, MVT::v8f64, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1367 { ISD::FSUB, MVT::v4f64, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1368 { ISD::FMUL, MVT::v8f64, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1369 { ISD::FMUL, MVT::v4f64, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1370 { ISD::FMUL, MVT::v2f64, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1371 { ISD::FMUL, MVT::f64, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1372
1373 { ISD::FDIV, MVT::f64, { 4, 14, 1, 1 } }, // Skylake from http://www.agner.org/
1374 { ISD::FDIV, MVT::v2f64, { 4, 14, 1, 1 } }, // Skylake from http://www.agner.org/
1375 { ISD::FDIV, MVT::v4f64, { 8, 14, 1, 1 } }, // Skylake from http://www.agner.org/
1376 { ISD::FDIV, MVT::v8f64, { 16, 23, 1, 3 } }, // Skylake from http://www.agner.org/
1377
1378 { ISD::FNEG, MVT::v16f32, { 1, 1, 1, 2 } }, // Skylake from http://www.agner.org/
1379 { ISD::FADD, MVT::v16f32, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1380 { ISD::FADD, MVT::v8f32, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1381 { ISD::FSUB, MVT::v16f32, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1382 { ISD::FSUB, MVT::v8f32, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1383 { ISD::FMUL, MVT::v16f32, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1384 { ISD::FMUL, MVT::v8f32, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1385 { ISD::FMUL, MVT::v4f32, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1386 { ISD::FMUL, MVT::f32, { 1, 4, 1, 1 } }, // Skylake from http://www.agner.org/
1387
1388 { ISD::FDIV, MVT::f32, { 3, 11, 1, 1 } }, // Skylake from http://www.agner.org/
1389 { ISD::FDIV, MVT::v4f32, { 3, 11, 1, 1 } }, // Skylake from http://www.agner.org/
1390 { ISD::FDIV, MVT::v8f32, { 5, 11, 1, 1 } }, // Skylake from http://www.agner.org/
1391 { ISD::FDIV, MVT::v16f32, { 10, 18, 1, 3 } }, // Skylake from http://www.agner.org/
1392 };
1393
1394 if (ST->hasAVX512())
1395 if (const auto *Entry = CostTableLookup(AVX512CostTable, ISD, LT.second))
1396 if (auto KindCost = Entry->Cost[CostKind])
1397 return LT.first * *KindCost;
1398
1399 static const CostKindTblEntry AVX2ShiftCostTable[] = {
1400 // Shifts on vXi64/vXi32 on AVX2 is legal even though we declare to
1401 // customize them to detect the cases where shift amount is a scalar one.
1402 { ISD::SHL, MVT::v4i32, { 2, 3, 1, 3 } }, // vpsllvd (Haswell from agner.org)
1403 { ISD::SRL, MVT::v4i32, { 2, 3, 1, 3 } }, // vpsrlvd (Haswell from agner.org)
1404 { ISD::SRA, MVT::v4i32, { 2, 3, 1, 3 } }, // vpsravd (Haswell from agner.org)
1405 { ISD::SHL, MVT::v8i32, { 4, 4, 1, 3 } }, // vpsllvd (Haswell from agner.org)
1406 { ISD::SRL, MVT::v8i32, { 4, 4, 1, 3 } }, // vpsrlvd (Haswell from agner.org)
1407 { ISD::SRA, MVT::v8i32, { 4, 4, 1, 3 } }, // vpsravd (Haswell from agner.org)
1408 { ISD::SHL, MVT::v2i64, { 2, 3, 1, 1 } }, // vpsllvq (Haswell from agner.org)
1409 { ISD::SRL, MVT::v2i64, { 2, 3, 1, 1 } }, // vpsrlvq (Haswell from agner.org)
1410 { ISD::SHL, MVT::v4i64, { 4, 4, 1, 2 } }, // vpsllvq (Haswell from agner.org)
1411 { ISD::SRL, MVT::v4i64, { 4, 4, 1, 2 } }, // vpsrlvq (Haswell from agner.org)
1412 };
1413
1414 if (ST->hasAVX512()) {
1415 if (ISD == ISD::SHL && LT.second == MVT::v32i16 && Op2Info.isConstant())
1416 // On AVX512, a packed v32i16 shift left by a constant build_vector
1417 // is lowered into a vector multiply (vpmullw).
1418 return getArithmeticInstrCost(Instruction::Mul, Ty, CostKind,
1419 Op1Info.getNoProps(), Op2Info.getNoProps());
1420 }
1421
1422 // Look for AVX2 lowering tricks (XOP is always better at v4i32 shifts).
1423 if (ST->hasAVX2() && !(ST->hasXOP() && LT.second == MVT::v4i32)) {
1424 if (ISD == ISD::SHL && LT.second == MVT::v16i16 &&
1425 Op2Info.isConstant())
1426 // On AVX2, a packed v16i16 shift left by a constant build_vector
1427 // is lowered into a vector multiply (vpmullw).
1428 return getArithmeticInstrCost(Instruction::Mul, Ty, CostKind,
1429 Op1Info.getNoProps(), Op2Info.getNoProps());
1430
1431 if (const auto *Entry = CostTableLookup(AVX2ShiftCostTable, ISD, LT.second))
1432 if (auto KindCost = Entry->Cost[CostKind])
1433 return LT.first * *KindCost;
1434 }
1435
1436 static const CostKindTblEntry XOPShiftCostTable[] = {
1437 // 128bit shifts take 1cy, but right shifts require negation beforehand.
1438 { ISD::SHL, MVT::v16i8, { 1, 3, 1, 1 } },
1439 { ISD::SRL, MVT::v16i8, { 2, 3, 1, 1 } },
1440 { ISD::SRA, MVT::v16i8, { 2, 3, 1, 1 } },
1441 { ISD::SHL, MVT::v8i16, { 1, 3, 1, 1 } },
1442 { ISD::SRL, MVT::v8i16, { 2, 3, 1, 1 } },
1443 { ISD::SRA, MVT::v8i16, { 2, 3, 1, 1 } },
1444 { ISD::SHL, MVT::v4i32, { 1, 3, 1, 1 } },
1445 { ISD::SRL, MVT::v4i32, { 2, 3, 1, 1 } },
1446 { ISD::SRA, MVT::v4i32, { 2, 3, 1, 1 } },
1447 { ISD::SHL, MVT::v2i64, { 1, 3, 1, 1 } },
1448 { ISD::SRL, MVT::v2i64, { 2, 3, 1, 1 } },
1449 { ISD::SRA, MVT::v2i64, { 2, 3, 1, 1 } },
1450 // 256bit shifts require splitting if AVX2 didn't catch them above.
1451 { ISD::SHL, MVT::v32i8, { 4, 7, 5, 6 } },
1452 { ISD::SRL, MVT::v32i8, { 6, 7, 5, 6 } },
1453 { ISD::SRA, MVT::v32i8, { 6, 7, 5, 6 } },
1454 { ISD::SHL, MVT::v16i16, { 4, 7, 5, 6 } },
1455 { ISD::SRL, MVT::v16i16, { 6, 7, 5, 6 } },
1456 { ISD::SRA, MVT::v16i16, { 6, 7, 5, 6 } },
1457 { ISD::SHL, MVT::v8i32, { 4, 7, 5, 6 } },
1458 { ISD::SRL, MVT::v8i32, { 6, 7, 5, 6 } },
1459 { ISD::SRA, MVT::v8i32, { 6, 7, 5, 6 } },
1460 { ISD::SHL, MVT::v4i64, { 4, 7, 5, 6 } },
1461 { ISD::SRL, MVT::v4i64, { 6, 7, 5, 6 } },
1462 { ISD::SRA, MVT::v4i64, { 6, 7, 5, 6 } },
1463 };
1464
1465 // Look for XOP lowering tricks.
1466 if (ST->hasXOP()) {
1467 // If the right shift is constant then we'll fold the negation so
1468 // it's as cheap as a left shift.
1469 int ShiftISD = ISD;
1470 if ((ShiftISD == ISD::SRL || ShiftISD == ISD::SRA) && Op2Info.isConstant())
1471 ShiftISD = ISD::SHL;
1472 if (const auto *Entry =
1473 CostTableLookup(XOPShiftCostTable, ShiftISD, LT.second))
1474 if (auto KindCost = Entry->Cost[CostKind])
1475 return LT.first * *KindCost;
1476 }
1477
1478 if (ISD == ISD::SHL && !Op2Info.isUniform() && Op2Info.isConstant()) {
1479 MVT VT = LT.second;
1480 // Vector shift left by non uniform constant can be lowered
1481 // into vector multiply.
1482 if (((VT == MVT::v8i16 || VT == MVT::v4i32) && ST->hasSSE2()) ||
1483 ((VT == MVT::v16i16 || VT == MVT::v8i32) && ST->hasAVX()))
1484 ISD = ISD::MUL;
1485 }
1486
1487 static const CostKindTblEntry GLMCostTable[] = {
1488 { ISD::FDIV, MVT::f32, { 18, 19, 1, 1 } }, // divss
1489 { ISD::FDIV, MVT::v4f32, { 35, 36, 1, 1 } }, // divps
1490 { ISD::FDIV, MVT::f64, { 33, 34, 1, 1 } }, // divsd
1491 { ISD::FDIV, MVT::v2f64, { 65, 66, 1, 1 } }, // divpd
1492 };
1493
1494 if (ST->useGLMDivSqrtCosts())
1495 if (const auto *Entry = CostTableLookup(GLMCostTable, ISD, LT.second))
1496 if (auto KindCost = Entry->Cost[CostKind])
1497 return LT.first * *KindCost;
1498
1499 static const CostKindTblEntry SLMCostTable[] = {
1500 { ISD::MUL, MVT::v4i32, { 11, 11, 1, 7 } }, // pmulld
1501 { ISD::MUL, MVT::v8i16, { 2, 5, 1, 1 } }, // pmullw
1502 { ISD::FMUL, MVT::f64, { 2, 5, 1, 1 } }, // mulsd
1503 { ISD::FMUL, MVT::f32, { 1, 4, 1, 1 } }, // mulss
1504 { ISD::FMUL, MVT::v2f64, { 4, 7, 1, 1 } }, // mulpd
1505 { ISD::FMUL, MVT::v4f32, { 2, 5, 1, 1 } }, // mulps
1506 { ISD::FDIV, MVT::f32, { 17, 19, 1, 1 } }, // divss
1507 { ISD::FDIV, MVT::v4f32, { 39, 39, 1, 6 } }, // divps
1508 { ISD::FDIV, MVT::f64, { 32, 34, 1, 1 } }, // divsd
1509 { ISD::FDIV, MVT::v2f64, { 69, 69, 1, 6 } }, // divpd
1510 { ISD::FADD, MVT::v2f64, { 2, 4, 1, 1 } }, // addpd
1511 { ISD::FSUB, MVT::v2f64, { 2, 4, 1, 1 } }, // subpd
1512 // v2i64/v4i64 mul is custom lowered as a series of long:
1513 // multiplies(3), shifts(3) and adds(2)
1514 // slm muldq version throughput is 2 and addq throughput 4
1515 // thus: 3X2 (muldq throughput) + 3X1 (shift throughput) +
1516 // 3X4 (addq throughput) = 17
1517 { ISD::MUL, MVT::v2i64, { 17, 22, 9, 9 } },
1518 // slm addq\subq throughput is 4
1519 { ISD::ADD, MVT::v2i64, { 4, 2, 1, 2 } },
1520 { ISD::SUB, MVT::v2i64, { 4, 2, 1, 2 } },
1521 };
1522
1523 if (ST->useSLMArithCosts())
1524 if (const auto *Entry = CostTableLookup(SLMCostTable, ISD, LT.second))
1525 if (auto KindCost = Entry->Cost[CostKind])
1526 return LT.first * *KindCost;
1527
1528 static const CostKindTblEntry AVX2CostTable[] = {
1529 { ISD::SHL, MVT::v16i8, { 6, 21,11,16 } }, // vpblendvb sequence.
1530 { ISD::SHL, MVT::v32i8, { 6, 23,11,22 } }, // vpblendvb sequence.
1531 { ISD::SHL, MVT::v8i16, { 5, 18, 5,10 } }, // extend/vpsrlvd/pack sequence.
1532 { ISD::SHL, MVT::v16i16, { 8, 10,10,14 } }, // extend/vpsrlvd/pack sequence.
1533
1534 { ISD::SRL, MVT::v16i8, { 6, 27,12,18 } }, // vpblendvb sequence.
1535 { ISD::SRL, MVT::v32i8, { 8, 30,12,24 } }, // vpblendvb sequence.
1536 { ISD::SRL, MVT::v8i16, { 5, 11, 5,10 } }, // extend/vpsrlvd/pack sequence.
1537 { ISD::SRL, MVT::v16i16, { 8, 10,10,14 } }, // extend/vpsrlvd/pack sequence.
1538
1539 { ISD::SRA, MVT::v16i8, { 17, 17,24,30 } }, // vpblendvb sequence.
1540 { ISD::SRA, MVT::v32i8, { 18, 20,24,43 } }, // vpblendvb sequence.
1541 { ISD::SRA, MVT::v8i16, { 5, 11, 5,10 } }, // extend/vpsravd/pack sequence.
1542 { ISD::SRA, MVT::v16i16, { 8, 10,10,14 } }, // extend/vpsravd/pack sequence.
1543 { ISD::SRA, MVT::v2i64, { 4, 5, 5, 5 } }, // srl/xor/sub sequence.
1544 { ISD::SRA, MVT::v4i64, { 8, 8, 5, 9 } }, // srl/xor/sub sequence.
1545
1546 { ISD::SUB, MVT::v32i8, { 1, 1, 1, 2 } }, // psubb
1547 { ISD::ADD, MVT::v32i8, { 1, 1, 1, 2 } }, // paddb
1548 { ISD::SUB, MVT::v16i16, { 1, 1, 1, 2 } }, // psubw
1549 { ISD::ADD, MVT::v16i16, { 1, 1, 1, 2 } }, // paddw
1550 { ISD::SUB, MVT::v8i32, { 1, 1, 1, 2 } }, // psubd
1551 { ISD::ADD, MVT::v8i32, { 1, 1, 1, 2 } }, // paddd
1552 { ISD::SUB, MVT::v4i64, { 1, 1, 1, 2 } }, // psubq
1553 { ISD::ADD, MVT::v4i64, { 1, 1, 1, 2 } }, // paddq
1554
1555 { ISD::MUL, MVT::v16i8, { 5, 18, 6,12 } }, // extend/pmullw/pack
1556 { ISD::MUL, MVT::v32i8, { 4, 8, 8,16 } }, // pmaddubsw
1557 { ISD::MUL, MVT::v16i16, { 2, 5, 1, 2 } }, // pmullw
1558 { ISD::MUL, MVT::v8i32, { 4, 10, 1, 2 } }, // pmulld
1559 { ISD::MUL, MVT::v4i32, { 2, 10, 1, 2 } }, // pmulld
1560 { ISD::MUL, MVT::v4i64, { 6, 10, 8,13 } }, // 3*pmuludq/3*shift/2*add
1561 { ISD::MUL, MVT::v2i64, { 6, 10, 8, 8 } }, // 3*pmuludq/3*shift/2*add
1562
1563 { X86ISD::PMULUDQ, MVT::v4i64, { 1, 5, 1, 1 } },
1564
1565 { ISD::FNEG, MVT::v4f64, { 1, 1, 1, 2 } }, // vxorpd
1566 { ISD::FNEG, MVT::v8f32, { 1, 1, 1, 2 } }, // vxorps
1567
1568 { ISD::FADD, MVT::f64, { 1, 4, 1, 1 } }, // vaddsd
1569 { ISD::FADD, MVT::f32, { 1, 4, 1, 1 } }, // vaddss
1570 { ISD::FADD, MVT::v2f64, { 1, 4, 1, 1 } }, // vaddpd
1571 { ISD::FADD, MVT::v4f32, { 1, 4, 1, 1 } }, // vaddps
1572 { ISD::FADD, MVT::v4f64, { 1, 4, 1, 2 } }, // vaddpd
1573 { ISD::FADD, MVT::v8f32, { 1, 4, 1, 2 } }, // vaddps
1574
1575 { ISD::FSUB, MVT::f64, { 1, 4, 1, 1 } }, // vsubsd
1576 { ISD::FSUB, MVT::f32, { 1, 4, 1, 1 } }, // vsubss
1577 { ISD::FSUB, MVT::v2f64, { 1, 4, 1, 1 } }, // vsubpd
1578 { ISD::FSUB, MVT::v4f32, { 1, 4, 1, 1 } }, // vsubps
1579 { ISD::FSUB, MVT::v4f64, { 1, 4, 1, 2 } }, // vsubpd
1580 { ISD::FSUB, MVT::v8f32, { 1, 4, 1, 2 } }, // vsubps
1581
1582 { ISD::FMUL, MVT::f64, { 1, 5, 1, 1 } }, // vmulsd
1583 { ISD::FMUL, MVT::f32, { 1, 5, 1, 1 } }, // vmulss
1584 { ISD::FMUL, MVT::v2f64, { 1, 5, 1, 1 } }, // vmulpd
1585 { ISD::FMUL, MVT::v4f32, { 1, 5, 1, 1 } }, // vmulps
1586 { ISD::FMUL, MVT::v4f64, { 1, 5, 1, 2 } }, // vmulpd
1587 { ISD::FMUL, MVT::v8f32, { 1, 5, 1, 2 } }, // vmulps
1588
1589 { ISD::FDIV, MVT::f32, { 7, 13, 1, 1 } }, // vdivss
1590 { ISD::FDIV, MVT::v4f32, { 7, 13, 1, 1 } }, // vdivps
1591 { ISD::FDIV, MVT::v8f32, { 14, 21, 1, 3 } }, // vdivps
1592 { ISD::FDIV, MVT::f64, { 14, 20, 1, 1 } }, // vdivsd
1593 { ISD::FDIV, MVT::v2f64, { 14, 20, 1, 1 } }, // vdivpd
1594 { ISD::FDIV, MVT::v4f64, { 28, 35, 1, 3 } }, // vdivpd
1595 };
1596
1597 // Look for AVX2 lowering tricks for custom cases.
1598 if (ST->hasAVX2())
1599 if (const auto *Entry = CostTableLookup(AVX2CostTable, ISD, LT.second))
1600 if (auto KindCost = Entry->Cost[CostKind])
1601 return LT.first * *KindCost;
1602
1603 static const CostKindTblEntry AVX1CostTable[] = {
1604 // We don't have to scalarize unsupported ops. We can issue two half-sized
1605 // operations and we only need to extract the upper YMM half.
1606 // Two ops + 1 extract + 1 insert = 4.
1607 { ISD::MUL, MVT::v32i8, { 10, 11, 18, 19 } }, // pmaddubsw + split
1608 { ISD::MUL, MVT::v16i8, { 5, 6, 8, 12 } }, // 2*pmaddubsw/3*and/psllw/or
1609 { ISD::MUL, MVT::v16i16, { 4, 8, 5, 6 } }, // pmullw + split
1610 { ISD::MUL, MVT::v8i32, { 5, 8, 5, 10 } }, // pmulld + split
1611 { ISD::MUL, MVT::v4i32, { 2, 5, 1, 3 } }, // pmulld
1612 { ISD::MUL, MVT::v4i64, { 12, 15, 19, 20 } },
1613
1614 { X86ISD::PMULUDQ, MVT::v4i64, { 3, 5, 5, 6 } }, // pmuludq + split
1615
1616 { ISD::AND, MVT::v32i8, { 1, 1, 1, 2 } }, // vandps
1617 { ISD::AND, MVT::v16i16, { 1, 1, 1, 2 } }, // vandps
1618 { ISD::AND, MVT::v8i32, { 1, 1, 1, 2 } }, // vandps
1619 { ISD::AND, MVT::v4i64, { 1, 1, 1, 2 } }, // vandps
1620
1621 { ISD::OR, MVT::v32i8, { 1, 1, 1, 2 } }, // vorps
1622 { ISD::OR, MVT::v16i16, { 1, 1, 1, 2 } }, // vorps
1623 { ISD::OR, MVT::v8i32, { 1, 1, 1, 2 } }, // vorps
1624 { ISD::OR, MVT::v4i64, { 1, 1, 1, 2 } }, // vorps
1625
1626 { ISD::XOR, MVT::v32i8, { 1, 1, 1, 2 } }, // vxorps
1627 { ISD::XOR, MVT::v16i16, { 1, 1, 1, 2 } }, // vxorps
1628 { ISD::XOR, MVT::v8i32, { 1, 1, 1, 2 } }, // vxorps
1629 { ISD::XOR, MVT::v4i64, { 1, 1, 1, 2 } }, // vxorps
1630
1631 { ISD::SUB, MVT::v32i8, { 4, 2, 5, 6 } }, // psubb + split
1632 { ISD::ADD, MVT::v32i8, { 4, 2, 5, 6 } }, // paddb + split
1633 { ISD::SUB, MVT::v16i16, { 4, 2, 5, 6 } }, // psubw + split
1634 { ISD::ADD, MVT::v16i16, { 4, 2, 5, 6 } }, // paddw + split
1635 { ISD::SUB, MVT::v8i32, { 4, 2, 5, 6 } }, // psubd + split
1636 { ISD::ADD, MVT::v8i32, { 4, 2, 5, 6 } }, // paddd + split
1637 { ISD::SUB, MVT::v4i64, { 4, 2, 5, 6 } }, // psubq + split
1638 { ISD::ADD, MVT::v4i64, { 4, 2, 5, 6 } }, // paddq + split
1639 { ISD::SUB, MVT::v2i64, { 1, 1, 1, 1 } }, // psubq
1640 { ISD::ADD, MVT::v2i64, { 1, 1, 1, 1 } }, // paddq
1641
1642 { ISD::SHL, MVT::v16i8, { 10, 21,11,17 } }, // pblendvb sequence.
1643 { ISD::SHL, MVT::v32i8, { 22, 22,27,40 } }, // pblendvb sequence + split.
1644 { ISD::SHL, MVT::v8i16, { 6, 9,11,11 } }, // pblendvb sequence.
1645 { ISD::SHL, MVT::v16i16, { 13, 16,24,25 } }, // pblendvb sequence + split.
1646 { ISD::SHL, MVT::v4i32, { 3, 11, 4, 6 } }, // pslld/paddd/cvttps2dq/pmulld
1647 { ISD::SHL, MVT::v8i32, { 9, 11,12,17 } }, // pslld/paddd/cvttps2dq/pmulld + split
1648 { ISD::SHL, MVT::v2i64, { 2, 4, 4, 6 } }, // Shift each lane + blend.
1649 { ISD::SHL, MVT::v4i64, { 6, 7,11,15 } }, // Shift each lane + blend + split.
1650
1651 { ISD::SRL, MVT::v16i8, { 11, 27,12,18 } }, // pblendvb sequence.
1652 { ISD::SRL, MVT::v32i8, { 23, 23,30,43 } }, // pblendvb sequence + split.
1653 { ISD::SRL, MVT::v8i16, { 13, 16,14,22 } }, // pblendvb sequence.
1654 { ISD::SRL, MVT::v16i16, { 28, 30,31,48 } }, // pblendvb sequence + split.
1655 { ISD::SRL, MVT::v4i32, { 6, 7,12,16 } }, // Shift each lane + blend.
1656 { ISD::SRL, MVT::v8i32, { 14, 14,26,34 } }, // Shift each lane + blend + split.
1657 { ISD::SRL, MVT::v2i64, { 2, 4, 4, 6 } }, // Shift each lane + blend.
1658 { ISD::SRL, MVT::v4i64, { 6, 7,11,15 } }, // Shift each lane + blend + split.
1659
1660 { ISD::SRA, MVT::v16i8, { 21, 22,24,36 } }, // pblendvb sequence.
1661 { ISD::SRA, MVT::v32i8, { 44, 45,51,76 } }, // pblendvb sequence + split.
1662 { ISD::SRA, MVT::v8i16, { 13, 16,14,22 } }, // pblendvb sequence.
1663 { ISD::SRA, MVT::v16i16, { 28, 30,31,48 } }, // pblendvb sequence + split.
1664 { ISD::SRA, MVT::v4i32, { 6, 7,12,16 } }, // Shift each lane + blend.
1665 { ISD::SRA, MVT::v8i32, { 14, 14,26,34 } }, // Shift each lane + blend + split.
1666 { ISD::SRA, MVT::v2i64, { 5, 6,10,14 } }, // Shift each lane + blend.
1667 { ISD::SRA, MVT::v4i64, { 12, 12,22,30 } }, // Shift each lane + blend + split.
1668
1669 { ISD::FNEG, MVT::v4f64, { 2, 2, 1, 2 } }, // BTVER2 from http://www.agner.org/
1670 { ISD::FNEG, MVT::v8f32, { 2, 2, 1, 2 } }, // BTVER2 from http://www.agner.org/
1671
1672 { ISD::FADD, MVT::f64, { 1, 5, 1, 1 } }, // BDVER2 from http://www.agner.org/
1673 { ISD::FADD, MVT::f32, { 1, 5, 1, 1 } }, // BDVER2 from http://www.agner.org/
1674 { ISD::FADD, MVT::v2f64, { 1, 5, 1, 1 } }, // BDVER2 from http://www.agner.org/
1675 { ISD::FADD, MVT::v4f32, { 1, 5, 1, 1 } }, // BDVER2 from http://www.agner.org/
1676 { ISD::FADD, MVT::v4f64, { 2, 5, 1, 2 } }, // BDVER2 from http://www.agner.org/
1677 { ISD::FADD, MVT::v8f32, { 2, 5, 1, 2 } }, // BDVER2 from http://www.agner.org/
1678
1679 { ISD::FSUB, MVT::f64, { 1, 5, 1, 1 } }, // BDVER2 from http://www.agner.org/
1680 { ISD::FSUB, MVT::f32, { 1, 5, 1, 1 } }, // BDVER2 from http://www.agner.org/
1681 { ISD::FSUB, MVT::v2f64, { 1, 5, 1, 1 } }, // BDVER2 from http://www.agner.org/
1682 { ISD::FSUB, MVT::v4f32, { 1, 5, 1, 1 } }, // BDVER2 from http://www.agner.org/
1683 { ISD::FSUB, MVT::v4f64, { 2, 5, 1, 2 } }, // BDVER2 from http://www.agner.org/
1684 { ISD::FSUB, MVT::v8f32, { 2, 5, 1, 2 } }, // BDVER2 from http://www.agner.org/
1685
1686 { ISD::FMUL, MVT::f64, { 2, 5, 1, 1 } }, // BTVER2 from http://www.agner.org/
1687 { ISD::FMUL, MVT::f32, { 1, 5, 1, 1 } }, // BTVER2 from http://www.agner.org/
1688 { ISD::FMUL, MVT::v2f64, { 2, 5, 1, 1 } }, // BTVER2 from http://www.agner.org/
1689 { ISD::FMUL, MVT::v4f32, { 1, 5, 1, 1 } }, // BTVER2 from http://www.agner.org/
1690 { ISD::FMUL, MVT::v4f64, { 4, 5, 1, 2 } }, // BTVER2 from http://www.agner.org/
1691 { ISD::FMUL, MVT::v8f32, { 2, 5, 1, 2 } }, // BTVER2 from http://www.agner.org/
1692
1693 { ISD::FDIV, MVT::f32, { 14, 14, 1, 1 } }, // SNB from http://www.agner.org/
1694 { ISD::FDIV, MVT::v4f32, { 14, 14, 1, 1 } }, // SNB from http://www.agner.org/
1695 { ISD::FDIV, MVT::v8f32, { 28, 29, 1, 3 } }, // SNB from http://www.agner.org/
1696 { ISD::FDIV, MVT::f64, { 22, 22, 1, 1 } }, // SNB from http://www.agner.org/
1697 { ISD::FDIV, MVT::v2f64, { 22, 22, 1, 1 } }, // SNB from http://www.agner.org/
1698 { ISD::FDIV, MVT::v4f64, { 44, 45, 1, 3 } }, // SNB from http://www.agner.org/
1699 };
1700
1701 if (ST->hasAVX())
1702 if (const auto *Entry = CostTableLookup(AVX1CostTable, ISD, LT.second))
1703 if (auto KindCost = Entry->Cost[CostKind])
1704 return LT.first * *KindCost;
1705
1706 static const CostKindTblEntry SSE42CostTable[] = {
1707 { ISD::FADD, MVT::f64, { 1, 3, 1, 1 } }, // Nehalem from http://www.agner.org/
1708 { ISD::FADD, MVT::f32, { 1, 3, 1, 1 } }, // Nehalem from http://www.agner.org/
1709 { ISD::FADD, MVT::v2f64, { 1, 3, 1, 1 } }, // Nehalem from http://www.agner.org/
1710 { ISD::FADD, MVT::v4f32, { 1, 3, 1, 1 } }, // Nehalem from http://www.agner.org/
1711
1712 { ISD::FSUB, MVT::f64, { 1, 3, 1, 1 } }, // Nehalem from http://www.agner.org/
1713 { ISD::FSUB, MVT::f32 , { 1, 3, 1, 1 } }, // Nehalem from http://www.agner.org/
1714 { ISD::FSUB, MVT::v2f64, { 1, 3, 1, 1 } }, // Nehalem from http://www.agner.org/
1715 { ISD::FSUB, MVT::v4f32, { 1, 3, 1, 1 } }, // Nehalem from http://www.agner.org/
1716
1717 { ISD::FMUL, MVT::f64, { 1, 5, 1, 1 } }, // Nehalem from http://www.agner.org/
1718 { ISD::FMUL, MVT::f32, { 1, 5, 1, 1 } }, // Nehalem from http://www.agner.org/
1719 { ISD::FMUL, MVT::v2f64, { 1, 5, 1, 1 } }, // Nehalem from http://www.agner.org/
1720 { ISD::FMUL, MVT::v4f32, { 1, 5, 1, 1 } }, // Nehalem from http://www.agner.org/
1721
1722 { ISD::FDIV, MVT::f32, { 14, 14, 1, 1 } }, // Nehalem from http://www.agner.org/
1723 { ISD::FDIV, MVT::v4f32, { 14, 14, 1, 1 } }, // Nehalem from http://www.agner.org/
1724 { ISD::FDIV, MVT::f64, { 22, 22, 1, 1 } }, // Nehalem from http://www.agner.org/
1725 { ISD::FDIV, MVT::v2f64, { 22, 22, 1, 1 } }, // Nehalem from http://www.agner.org/
1726
1727 { ISD::MUL, MVT::v2i64, { 6, 10,10,10 } } // 3*pmuludq/3*shift/2*add
1728 };
1729
1730 if (ST->hasSSE42())
1731 if (const auto *Entry = CostTableLookup(SSE42CostTable, ISD, LT.second))
1732 if (auto KindCost = Entry->Cost[CostKind])
1733 return LT.first * *KindCost;
1734
1735 static const CostKindTblEntry SSE41CostTable[] = {
1736 { ISD::SHL, MVT::v16i8, { 15, 24,17,22 } }, // pblendvb sequence.
1737 { ISD::SHL, MVT::v8i16, { 11, 14,11,11 } }, // pblendvb sequence.
1738 { ISD::SHL, MVT::v4i32, { 14, 20, 4,10 } }, // pslld/paddd/cvttps2dq/pmulld
1739
1740 { ISD::SRL, MVT::v16i8, { 16, 27,18,24 } }, // pblendvb sequence.
1741 { ISD::SRL, MVT::v8i16, { 22, 26,23,27 } }, // pblendvb sequence.
1742 { ISD::SRL, MVT::v4i32, { 16, 17,15,19 } }, // Shift each lane + blend.
1743 { ISD::SRL, MVT::v2i64, { 4, 6, 5, 7 } }, // splat+shuffle sequence.
1744
1745 { ISD::SRA, MVT::v16i8, { 38, 41,30,36 } }, // pblendvb sequence.
1746 { ISD::SRA, MVT::v8i16, { 22, 26,23,27 } }, // pblendvb sequence.
1747 { ISD::SRA, MVT::v4i32, { 16, 17,15,19 } }, // Shift each lane + blend.
1748 { ISD::SRA, MVT::v2i64, { 8, 17, 5, 7 } }, // splat+shuffle sequence.
1749
1750 { ISD::MUL, MVT::v4i32, { 2, 11, 1, 1 } } // pmulld (Nehalem from agner.org)
1751 };
1752
1753 if (ST->hasSSE41())
1754 if (const auto *Entry = CostTableLookup(SSE41CostTable, ISD, LT.second))
1755 if (auto KindCost = Entry->Cost[CostKind])
1756 return LT.first * *KindCost;
1757
1758 static const CostKindTblEntry SSSE3CostTable[] = {
1759 { ISD::MUL, MVT::v16i8, { 5, 18,10,12 } }, // 2*pmaddubsw/3*and/psllw/or
1760 };
1761
1762 if (ST->hasSSSE3())
1763 if (const auto *Entry = CostTableLookup(SSSE3CostTable, ISD, LT.second))
1764 if (auto KindCost = Entry->Cost[CostKind])
1765 return LT.first * *KindCost;
1766
1767 static const CostKindTblEntry SSE2CostTable[] = {
1768 // We don't correctly identify costs of casts because they are marked as
1769 // custom.
1770 { ISD::SHL, MVT::v16i8, { 13, 21,26,28 } }, // cmpgtb sequence.
1771 { ISD::SHL, MVT::v8i16, { 24, 27,16,20 } }, // cmpgtw sequence.
1772 { ISD::SHL, MVT::v4i32, { 17, 19,10,12 } }, // pslld/paddd/cvttps2dq/pmuludq.
1773 { ISD::SHL, MVT::v2i64, { 4, 6, 5, 7 } }, // splat+shuffle sequence.
1774
1775 { ISD::SRL, MVT::v16i8, { 14, 28,27,30 } }, // cmpgtb sequence.
1776 { ISD::SRL, MVT::v8i16, { 16, 19,31,31 } }, // cmpgtw sequence.
1777 { ISD::SRL, MVT::v4i32, { 12, 12,15,19 } }, // Shift each lane + blend.
1778 { ISD::SRL, MVT::v2i64, { 4, 6, 5, 7 } }, // splat+shuffle sequence.
1779
1780 { ISD::SRA, MVT::v16i8, { 27, 30,54,54 } }, // unpacked cmpgtb sequence.
1781 { ISD::SRA, MVT::v8i16, { 16, 19,31,31 } }, // cmpgtw sequence.
1782 { ISD::SRA, MVT::v4i32, { 12, 12,15,19 } }, // Shift each lane + blend.
1783 { ISD::SRA, MVT::v2i64, { 8, 11,12,16 } }, // srl/xor/sub splat+shuffle sequence.
1784
1785 { ISD::AND, MVT::v16i8, { 1, 1, 1, 1 } }, // pand
1786 { ISD::AND, MVT::v8i16, { 1, 1, 1, 1 } }, // pand
1787 { ISD::AND, MVT::v4i32, { 1, 1, 1, 1 } }, // pand
1788 { ISD::AND, MVT::v2i64, { 1, 1, 1, 1 } }, // pand
1789
1790 { ISD::OR, MVT::v16i8, { 1, 1, 1, 1 } }, // por
1791 { ISD::OR, MVT::v8i16, { 1, 1, 1, 1 } }, // por
1792 { ISD::OR, MVT::v4i32, { 1, 1, 1, 1 } }, // por
1793 { ISD::OR, MVT::v2i64, { 1, 1, 1, 1 } }, // por
1794
1795 { ISD::XOR, MVT::v16i8, { 1, 1, 1, 1 } }, // pxor
1796 { ISD::XOR, MVT::v8i16, { 1, 1, 1, 1 } }, // pxor
1797 { ISD::XOR, MVT::v4i32, { 1, 1, 1, 1 } }, // pxor
1798 { ISD::XOR, MVT::v2i64, { 1, 1, 1, 1 } }, // pxor
1799
1800 { ISD::ADD, MVT::v2i64, { 1, 2, 1, 2 } }, // paddq
1801 { ISD::SUB, MVT::v2i64, { 1, 2, 1, 2 } }, // psubq
1802
1803 { ISD::MUL, MVT::v16i8, { 6, 18,12,12 } }, // 2*unpack/2*pmullw/2*and/pack
1804 { ISD::MUL, MVT::v8i16, { 1, 5, 1, 1 } }, // pmullw
1805 { ISD::MUL, MVT::v4i32, { 6, 8, 7, 7 } }, // 3*pmuludq/4*shuffle
1806 { ISD::MUL, MVT::v2i64, { 7, 10,10,10 } }, // 3*pmuludq/3*shift/2*add
1807
1808 { X86ISD::PMULUDQ, MVT::v2i64, { 1, 5, 1, 1 } },
1809
1810 { ISD::FDIV, MVT::f32, { 23, 23, 1, 1 } }, // Pentium IV from http://www.agner.org/
1811 { ISD::FDIV, MVT::v4f32, { 39, 39, 1, 1 } }, // Pentium IV from http://www.agner.org/
1812 { ISD::FDIV, MVT::f64, { 38, 38, 1, 1 } }, // Pentium IV from http://www.agner.org/
1813 { ISD::FDIV, MVT::v2f64, { 69, 69, 1, 1 } }, // Pentium IV from http://www.agner.org/
1814
1815 { ISD::FNEG, MVT::f32, { 1, 1, 1, 1 } }, // Pentium IV from http://www.agner.org/
1816 { ISD::FNEG, MVT::f64, { 1, 1, 1, 1 } }, // Pentium IV from http://www.agner.org/
1817 { ISD::FNEG, MVT::v4f32, { 1, 1, 1, 1 } }, // Pentium IV from http://www.agner.org/
1818 { ISD::FNEG, MVT::v2f64, { 1, 1, 1, 1 } }, // Pentium IV from http://www.agner.org/
1819
1820 { ISD::FADD, MVT::f32, { 2, 3, 1, 1 } }, // Pentium IV from http://www.agner.org/
1821 { ISD::FADD, MVT::f64, { 2, 3, 1, 1 } }, // Pentium IV from http://www.agner.org/
1822 { ISD::FADD, MVT::v2f64, { 2, 3, 1, 1 } }, // Pentium IV from http://www.agner.org/
1823
1824 { ISD::FSUB, MVT::f32, { 2, 3, 1, 1 } }, // Pentium IV from http://www.agner.org/
1825 { ISD::FSUB, MVT::f64, { 2, 3, 1, 1 } }, // Pentium IV from http://www.agner.org/
1826 { ISD::FSUB, MVT::v2f64, { 2, 3, 1, 1 } }, // Pentium IV from http://www.agner.org/
1827
1828 { ISD::FMUL, MVT::f64, { 2, 5, 1, 1 } }, // Pentium IV from http://www.agner.org/
1829 { ISD::FMUL, MVT::v2f64, { 2, 5, 1, 1 } }, // Pentium IV from http://www.agner.org/
1830 };
1831
1832 if (ST->hasSSE2())
1833 if (const auto *Entry = CostTableLookup(SSE2CostTable, ISD, LT.second))
1834 if (auto KindCost = Entry->Cost[CostKind])
1835 return LT.first * *KindCost;
1836
1837 static const CostKindTblEntry SSE1CostTable[] = {
1838 { ISD::FDIV, MVT::f32, { 17, 18, 1, 1 } }, // Pentium III from http://www.agner.org/
1839 { ISD::FDIV, MVT::v4f32, { 34, 48, 1, 1 } }, // Pentium III from http://www.agner.org/
1840
1841 { ISD::FNEG, MVT::f32, { 2, 2, 1, 2 } }, // Pentium III from http://www.agner.org/
1842 { ISD::FNEG, MVT::v4f32, { 2, 2, 1, 2 } }, // Pentium III from http://www.agner.org/
1843
1844 { ISD::FADD, MVT::f32, { 1, 3, 1, 1 } }, // Pentium III from http://www.agner.org/
1845 { ISD::FADD, MVT::v4f32, { 2, 3, 1, 1 } }, // Pentium III from http://www.agner.org/
1846
1847 { ISD::FSUB, MVT::f32, { 1, 3, 1, 1 } }, // Pentium III from http://www.agner.org/
1848 { ISD::FSUB, MVT::v4f32, { 2, 3, 1, 1 } }, // Pentium III from http://www.agner.org/
1849
1850 { ISD::FMUL, MVT::f32, { 2, 5, 1, 1 } }, // Pentium III from http://www.agner.org/
1851 { ISD::FMUL, MVT::v4f32, { 2, 5, 1, 1 } }, // Pentium III from http://www.agner.org/
1852 };
1853
1854 if (ST->hasSSE1())
1855 if (const auto *Entry = CostTableLookup(SSE1CostTable, ISD, LT.second))
1856 if (auto KindCost = Entry->Cost[CostKind])
1857 return LT.first * *KindCost;
1858
1859 static const CostKindTblEntry X64CostTbl[] = { // 64-bit targets
1860 { ISD::ADD, MVT::i64, { 1 } }, // Core (Merom) from http://www.agner.org/
1861 { ISD::SUB, MVT::i64, { 1 } }, // Core (Merom) from http://www.agner.org/
1862 { ISD::MUL, MVT::i64, { 2, 6, 1, 2 } },
1863 };
1864
1865 if (ST->is64Bit())
1866 if (const auto *Entry = CostTableLookup(X64CostTbl, ISD, LT.second))
1867 if (auto KindCost = Entry->Cost[CostKind])
1868 return LT.first * *KindCost;
1869
1870 static const CostKindTblEntry X86CostTbl[] = { // 32 or 64-bit targets
1871 { ISD::ADD, MVT::i8, { 1 } }, // Pentium III from http://www.agner.org/
1872 { ISD::ADD, MVT::i16, { 1 } }, // Pentium III from http://www.agner.org/
1873 { ISD::ADD, MVT::i32, { 1 } }, // Pentium III from http://www.agner.org/
1874
1875 { ISD::SUB, MVT::i8, { 1 } }, // Pentium III from http://www.agner.org/
1876 { ISD::SUB, MVT::i16, { 1 } }, // Pentium III from http://www.agner.org/
1877 { ISD::SUB, MVT::i32, { 1 } }, // Pentium III from http://www.agner.org/
1878
1879 { ISD::MUL, MVT::i8, { 3, 4, 1, 1 } },
1880 { ISD::MUL, MVT::i16, { 2, 4, 1, 1 } },
1881 { ISD::MUL, MVT::i32, { 1, 4, 1, 1 } },
1882
1883 { ISD::FNEG, MVT::f64, { 2, 2, 1, 3 } }, // (x87)
1884 { ISD::FADD, MVT::f64, { 2, 3, 1, 1 } }, // (x87)
1885 { ISD::FSUB, MVT::f64, { 2, 3, 1, 1 } }, // (x87)
1886 { ISD::FMUL, MVT::f64, { 2, 5, 1, 1 } }, // (x87)
1887 { ISD::FDIV, MVT::f64, { 38, 38, 1, 1 } }, // (x87)
1888 };
1889
1890 if (const auto *Entry = CostTableLookup(X86CostTbl, ISD, LT.second))
1891 if (auto KindCost = Entry->Cost[CostKind])
1892 return LT.first * *KindCost;
1893
1894 // It is not a good idea to vectorize division. We have to scalarize it and
1895 // in the process we will often end up having to spilling regular
1896 // registers. The overhead of division is going to dominate most kernels
1897 // anyways so try hard to prevent vectorization of division - it is
1898 // generally a bad idea. Assume somewhat arbitrarily that we have to be able
1899 // to hide "20 cycles" for each lane.
1900 if (CostKind == TTI::TCK_RecipThroughput && LT.second.isVector() &&
1901 (ISD == ISD::SDIV || ISD == ISD::SREM || ISD == ISD::UDIV ||
1902 ISD == ISD::UREM)) {
1903 InstructionCost ScalarCost =
1904 getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind,
1905 Op1Info.getNoProps(), Op2Info.getNoProps());
1906 return 20 * LT.first * LT.second.getVectorNumElements() * ScalarCost;
1907 }
1908
1909 // Handle some basic single instruction code size cases.
1910 if (CostKind == TTI::TCK_CodeSize) {
1911 switch (ISD) {
1912 case ISD::FADD:
1913 case ISD::FSUB:
1914 case ISD::FMUL:
1915 case ISD::FDIV:
1916 case ISD::FNEG:
1917 case ISD::AND:
1918 case ISD::OR:
1919 case ISD::XOR:
1920 return LT.first;
1921 break;
1922 }
1923 }
1924
1925 // Fallback to the default implementation.
1926 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
1927 Args, CtxI);
1928}
1929
1932 unsigned Opcode1, const SmallBitVector &OpcodeMask,
1933 TTI::TargetCostKind CostKind,
1934 ArrayRef<const Value *> Scalars) const {
1935 if (isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask, Scalars))
1936 return TTI::TCC_Basic;
1938}
1939
1941 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
1942 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
1943 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
1944 TTI::VectorInstrContext VIC) const {
1945 assert((Mask.empty() || DstTy->isScalableTy() ||
1946 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
1947 "Expected the Mask to match the return size if given");
1948 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
1949 "Expected the same scalar types");
1950
1951 // 64-bit packed float vectors (v2f32) are widened to type v4f32.
1952 // 64-bit packed integer vectors (v2i32) are widened to type v4i32.
1953 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcTy);
1954
1955 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);
1956
1957 // If all args are constant than this will be constant folded away.
1958 if (!Args.empty() &&
1959 all_of(Args, [](const Value *Arg) { return isa<Constant>(Arg); }))
1960 return TTI::TCC_Free;
1961
1962 // Recognize a basic concat_vector shuffle.
1963 if (Kind == TTI::SK_PermuteTwoSrc &&
1964 Mask.size() == (2 * SrcTy->getElementCount().getKnownMinValue()) &&
1965 ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
1969 CostKind, Mask, Mask.size() / 2, SrcTy);
1970
1971 // Treat Transpose as 2-op shuffles - there's no difference in lowering.
1972 if (Kind == TTI::SK_Transpose)
1973 if (LT.second != MVT::v4f64 && LT.second != MVT::v4i64)
1974 Kind = TTI::SK_PermuteTwoSrc;
1975
1976 if (Kind == TTI::SK_Broadcast) {
1977 // For Broadcasts we are splatting the first element from the first input
1978 // register, so only need to reference that input and all the output
1979 // registers are the same.
1980 LT.first = 1;
1981
1982 // If we're broadcasting a load then AVX/AVX2 can do this for free.
1983 // If many-used-load whose every use is one of a small set of operations
1984 // that SLP can rewrite into a single vector lane, codegen can fold it into
1985 // the free broadcast.
1986 using namespace PatternMatch;
1987 auto IsBroadcastLoadFoldUser = [&](const User *U) {
1988 if (isa<InsertElementInst>(U) && U->getOperand(1) == Args[0])
1989 return true;
1990 if (U->getType()->isVectorTy())
1991 return false;
1992 // Terminators (return/branch/switch/indirectbr/resume/invoke EH)
1993 // and phis carry the value across control flow.
1994 if (const auto *I = dyn_cast<Instruction>(U))
1995 if (I->isTerminator() ||
1997 return false;
1998 // Only pure calls can be folded.
1999 if (const auto *CB = dyn_cast<CallBase>(U))
2000 return CB->doesNotAccessMemory() && !CB->mayHaveSideEffects();
2001 return true;
2002 };
2003 auto IsFoldableSLPBroadcastLoad = [&]() {
2004 if (!match(Args[0], m_Load(m_Value())))
2005 return false;
2006 auto *FVT = dyn_cast<FixedVectorType>(DstTy);
2007 if (!FVT)
2008 return false;
2009 // getNumUses() counts each Use, matching the per-lane broadcast
2010 // accounting (a use like `op %x, %x` consumes two broadcast lanes).
2011 if (Args[0]->getNumUses() != FVT->getNumElements())
2012 return false;
2013 return all_of(Args[0]->users(), IsBroadcastLoadFoldUser);
2014 };
2015 if (!Args.empty() &&
2016 (match(Args[0], m_OneUse(m_Load(m_Value()))) ||
2017 IsFoldableSLPBroadcastLoad()) &&
2018 (ST->hasAVX2() ||
2019 (ST->hasAVX() && LT.second.getScalarSizeInBits() >= 32)))
2020 return TTI::TCC_Free;
2021 }
2022
2023 // Attempt to detect a cheaper inlane shuffle, avoiding 128-bit subvector
2024 // permutation.
2025 // Attempt to detect a shuffle mask with a single defined element.
2026 bool IsInLaneShuffle = false;
2027 bool IsSingleElementMask = false;
2028 if (SrcTy->getPrimitiveSizeInBits() > 0 &&
2029 (SrcTy->getPrimitiveSizeInBits() % 128) == 0 &&
2030 SrcTy->getScalarSizeInBits() == LT.second.getScalarSizeInBits() &&
2031 Mask.size() == SrcTy->getElementCount().getKnownMinValue()) {
2032 unsigned NumLanes = SrcTy->getPrimitiveSizeInBits() / 128;
2033 unsigned NumEltsPerLane = Mask.size() / NumLanes;
2034 if ((Mask.size() % NumLanes) == 0) {
2035 IsInLaneShuffle = all_of(enumerate(Mask), [&](const auto &P) {
2036 return P.value() == PoisonMaskElem ||
2037 ((P.value() % Mask.size()) / NumEltsPerLane) ==
2038 (P.index() / NumEltsPerLane);
2039 });
2040 IsSingleElementMask =
2041 (Mask.size() - 1) == static_cast<unsigned>(count_if(Mask, [](int M) {
2042 return M == PoisonMaskElem;
2043 }));
2044 }
2045 }
2046
2047 // Treat <X x bfloat> shuffles as <X x half>.
2048 if (LT.second.isVectorOf(MVT::bf16))
2049 LT.second = LT.second.changeVectorElementType(MVT::f16);
2050
2051 // Subvector extractions are free if they start at the beginning of a
2052 // vector and cheap if the subvectors are aligned.
2053 if (Kind == TTI::SK_ExtractSubvector && LT.second.isVector()) {
2054 int NumElts = LT.second.getVectorNumElements();
2055 if ((Index % NumElts) == 0)
2056 return TTI::TCC_Free;
2057 std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(SubTp);
2058 if (SubLT.second.isVector()) {
2059 int NumSubElts = SubLT.second.getVectorNumElements();
2060 if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
2061 return SubLT.first;
2062 // Handle some cases for widening legalization. For now we only handle
2063 // cases where the original subvector was naturally aligned and evenly
2064 // fit in its legalized subvector type.
2065 // FIXME: Remove some of the alignment restrictions.
2066 // FIXME: We can use permq for 64-bit or larger extracts from 256-bit
2067 // vectors.
2068 int OrigSubElts = cast<FixedVectorType>(SubTp)->getNumElements();
2069 if (NumSubElts > OrigSubElts && (Index % OrigSubElts) == 0 &&
2070 (NumSubElts % OrigSubElts) == 0 &&
2071 LT.second.getVectorElementType() ==
2072 SubLT.second.getVectorElementType() &&
2073 LT.second.getVectorElementType().getSizeInBits() ==
2074 SrcTy->getElementType()->getPrimitiveSizeInBits()) {
2075 assert(NumElts >= NumSubElts && NumElts > OrigSubElts &&
2076 "Unexpected number of elements!");
2077 auto *VecTy = FixedVectorType::get(SrcTy->getElementType(),
2078 LT.second.getVectorNumElements());
2079 auto *SubTy = FixedVectorType::get(SrcTy->getElementType(),
2080 SubLT.second.getVectorNumElements());
2081 int ExtractIndex = alignDown((Index % NumElts), NumSubElts);
2082 InstructionCost ExtractCost =
2083 getShuffleCost(TTI::SK_ExtractSubvector, VecTy, VecTy, CostKind, {},
2084 ExtractIndex, SubTy);
2085
2086 // If the original size is 32-bits or more, we can use pshufd. Otherwise
2087 // if we have SSSE3 we can use pshufb.
2088 if (SubTp->getPrimitiveSizeInBits() >= 32 || ST->hasSSSE3())
2089 return ExtractCost + 1; // pshufd or pshufb
2090
2091 assert(SubTp->getPrimitiveSizeInBits() == 16 &&
2092 "Unexpected vector size");
2093
2094 return ExtractCost + 2; // worst case pshufhw + pshufd
2095 }
2096 }
2097 // If the extract subvector is not optimal, treat it as single op shuffle.
2099 }
2100
2101 // Subvector insertions are cheap if the subvectors are aligned.
2102 // Note that in general, the insertion starting at the beginning of a vector
2103 // isn't free, because we need to preserve the rest of the wide vector,
2104 // but if the destination vector legalizes to the same width as the subvector
2105 // then the insertion will simplify to a (free) register copy.
2106 if (Kind == TTI::SK_InsertSubvector && LT.second.isVector()) {
2107 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(DstTy);
2108 int NumElts = DstLT.second.getVectorNumElements();
2109 std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(SubTp);
2110 if (SubLT.second.isVector()) {
2111 int NumSubElts = SubLT.second.getVectorNumElements();
2112 bool MatchingTypes =
2113 NumElts == NumSubElts &&
2114 (SubTp->getElementCount().getKnownMinValue() % NumSubElts) == 0;
2115 if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
2116 return MatchingTypes ? TTI::TCC_Free : SubLT.first;
2117 }
2118
2119 // Attempt to match MOVSS (Idx == 0) or INSERTPS pattern. This will have
2120 // been matched by improveShuffleKindFromMask as a SK_InsertSubvector of
2121 // v1f32 (legalised to f32) into a v4f32.
2122 if (LT.first == 1 && LT.second == MVT::v4f32 && SubLT.first == 1 &&
2123 SubLT.second == MVT::f32 && (Index == 0 || ST->hasSSE41()))
2124 return 1;
2125
2126 // If the insertion is the lowest subvector then it will be blended
2127 // otherwise treat it like a 2-op shuffle.
2128 Kind =
2129 (Index == 0 && LT.first == 1) ? TTI::SK_Select : TTI::SK_PermuteTwoSrc;
2130 }
2131
2132 // Handle some common (illegal) sub-vector types as they are often very cheap
2133 // to shuffle even on targets without PSHUFB.
2134 EVT VT = TLI->getValueType(DL, SrcTy);
2135 if (VT.isSimple() && VT.isVector() && VT.getSizeInBits() < 128 &&
2136 !ST->hasSSSE3()) {
2137 static const CostKindTblEntry SSE2SubVectorShuffleTbl[] = {
2138 {TTI::SK_Broadcast, MVT::v4i16, {1,1,1,1}}, // pshuflw
2139 {TTI::SK_Broadcast, MVT::v2i16, {1,1,1,1}}, // pshuflw
2140 {TTI::SK_Broadcast, MVT::v8i8, {2,2,2,2}}, // punpck/pshuflw
2141 {TTI::SK_Broadcast, MVT::v4i8, {2,2,2,2}}, // punpck/pshuflw
2142 {TTI::SK_Broadcast, MVT::v2i8, {1,1,1,1}}, // punpck
2143
2144 {TTI::SK_Reverse, MVT::v4i16, {1,1,1,1}}, // pshuflw
2145 {TTI::SK_Reverse, MVT::v2i16, {1,1,1,1}}, // pshuflw
2146 {TTI::SK_Reverse, MVT::v4i8, {3,3,3,3}}, // punpck/pshuflw/packus
2147 {TTI::SK_Reverse, MVT::v2i8, {1,1,1,1}}, // punpck
2148
2149 {TTI::SK_Splice, MVT::v4i16, {2,2,2,2}}, // punpck+psrldq
2150 {TTI::SK_Splice, MVT::v2i16, {2,2,2,2}}, // punpck+psrldq
2151 {TTI::SK_Splice, MVT::v4i8, {2,2,2,2}}, // punpck+psrldq
2152 {TTI::SK_Splice, MVT::v2i8, {2,2,2,2}}, // punpck+psrldq
2153
2154 {TTI::SK_PermuteTwoSrc, MVT::v4i16, {2,2,2,2}}, // punpck/pshuflw
2155 {TTI::SK_PermuteTwoSrc, MVT::v2i16, {2,2,2,2}}, // punpck/pshuflw
2156 {TTI::SK_PermuteTwoSrc, MVT::v8i8, {7,7,7,7}}, // punpck/pshuflw
2157 {TTI::SK_PermuteTwoSrc, MVT::v4i8, {4,4,4,4}}, // punpck/pshuflw
2158 {TTI::SK_PermuteTwoSrc, MVT::v2i8, {2,2,2,2}}, // punpck
2159
2160 {TTI::SK_PermuteSingleSrc, MVT::v4i16, {1,1,1,1}}, // pshuflw
2161 {TTI::SK_PermuteSingleSrc, MVT::v2i16, {1,1,1,1}}, // pshuflw
2162 {TTI::SK_PermuteSingleSrc, MVT::v8i8, {5,5,5,5}}, // punpck/pshuflw
2163 {TTI::SK_PermuteSingleSrc, MVT::v4i8, {3,3,3,3}}, // punpck/pshuflw
2164 {TTI::SK_PermuteSingleSrc, MVT::v2i8, {1,1,1,1}}, // punpck
2165 };
2166
2167 if (ST->hasSSE2())
2168 if (const auto *Entry =
2169 CostTableLookup(SSE2SubVectorShuffleTbl, Kind, VT.getSimpleVT()))
2170 if (auto KindCost = Entry->Cost[CostKind])
2171 return LT.first * *KindCost;
2172 }
2173
2174 // We are going to permute multiple sources and the result will be in multiple
2175 // destinations. Providing an accurate cost only for splits where the element
2176 // type remains the same.
2177 if (LT.first != 1) {
2178 MVT LegalVT = LT.second;
2179 if (LegalVT.isVector() &&
2180 LegalVT.getVectorElementType().getSizeInBits() ==
2181 SrcTy->getElementType()->getPrimitiveSizeInBits() &&
2182 LegalVT.getVectorNumElements() <
2183 cast<FixedVectorType>(SrcTy)->getNumElements()) {
2184 unsigned VecTySize = DL.getTypeStoreSize(SrcTy);
2185 unsigned LegalVTSize = LegalVT.getStoreSize();
2186 // Number of source vectors after legalization:
2187 unsigned NumOfSrcs = (VecTySize + LegalVTSize - 1) / LegalVTSize;
2188 // Number of destination vectors after legalization:
2189 InstructionCost NumOfDests = LT.first;
2190
2191 auto *SingleOpTy = FixedVectorType::get(SrcTy->getElementType(),
2192 LegalVT.getVectorNumElements());
2193
2194 if (!Mask.empty() && NumOfDests.isValid()) {
2195 // Try to perform better estimation of the permutation.
2196 // 1. Split the source/destination vectors into real registers.
2197 // 2. Do the mask analysis to identify which real registers are
2198 // permuted. If more than 1 source registers are used for the
2199 // destination register building, the cost for this destination register
2200 // is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
2201 // source register is used, build mask and calculate the cost as a cost
2202 // of PermuteSingleSrc.
2203 // Also, for the single register permute we try to identify if the
2204 // destination register is just a copy of the source register or the
2205 // copy of the previous destination register (the cost is
2206 // TTI::TCC_Basic). If the source register is just reused, the cost for
2207 // this operation is TTI::TCC_Free.
2208 NumOfDests =
2210 FixedVectorType::get(SrcTy->getElementType(), Mask.size()))
2211 .first;
2212 unsigned E = NumOfDests.getValue();
2213 unsigned NormalizedVF =
2214 LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E);
2215 unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
2216 unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
2217 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
2218 copy(Mask, NormalizedMask.begin());
2219 unsigned PrevSrcReg = 0;
2220 ArrayRef<int> PrevRegMask;
2223 NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {},
2224 [this, SingleOpTy, CostKind, &PrevSrcReg, &PrevRegMask,
2225 &Cost](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
2226 if (!ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size())) {
2227 // Check if the previous register can be just copied to the next
2228 // one.
2229 if (PrevRegMask.empty() || PrevSrcReg != SrcReg ||
2230 PrevRegMask != RegMask)
2231 Cost +=
2233 SingleOpTy, CostKind, RegMask, 0, nullptr);
2234 else
2235 // Just a copy of previous destination register.
2237 return;
2238 }
2239 if (SrcReg != DestReg &&
2240 any_of(RegMask, not_equal_to(PoisonMaskElem))) {
2241 // Just a copy of the source register.
2243 }
2244 PrevSrcReg = SrcReg;
2245 PrevRegMask = RegMask;
2246 },
2247 [this, SingleOpTy, CostKind,
2248 &Cost](ArrayRef<int> RegMask, unsigned /*Unused*/,
2249 unsigned /*Unused*/, bool /*Unused*/) {
2251 SingleOpTy, CostKind, RegMask, 0, nullptr);
2252 });
2253 return Cost;
2254 }
2255
2256 InstructionCost NumOfShuffles = (NumOfSrcs - 1) * NumOfDests;
2257 return NumOfShuffles * getShuffleCost(TTI::SK_PermuteTwoSrc, SingleOpTy,
2258 SingleOpTy, CostKind, {}, 0,
2259 nullptr);
2260 }
2261
2262 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
2263 SubTp);
2264 }
2265
2266 // If we're just moving a single element around (probably as an alternative to
2267 // extracting it), we can assume this is cheap.
2268 if (LT.first == 1 && IsInLaneShuffle && IsSingleElementMask)
2269 return TTI::TCC_Basic;
2270
2271 static const CostKindTblEntry AVX512VBMIShuffleTbl[] = {
2272 { TTI::SK_Reverse, MVT::v64i8, { 1, 1, 1, 1 } }, // vpermb
2273 { TTI::SK_Reverse, MVT::v32i8, { 1, 1, 1, 1 } }, // vpermb
2274 { TTI::SK_PermuteSingleSrc, MVT::v64i8, { 1, 1, 1, 1 } }, // vpermb
2275 { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 1, 1, 1, 1 } }, // vpermb
2276 { TTI::SK_PermuteTwoSrc, MVT::v64i8, { 2, 2, 2, 2 } }, // vpermt2b
2277 { TTI::SK_PermuteTwoSrc, MVT::v32i8, { 2, 2, 2, 2 } }, // vpermt2b
2278 { TTI::SK_PermuteTwoSrc, MVT::v16i8, { 2, 2, 2, 2 } } // vpermt2b
2279 };
2280
2281 if (ST->hasVBMI())
2282 if (const auto *Entry =
2283 CostTableLookup(AVX512VBMIShuffleTbl, Kind, LT.second))
2284 if (auto KindCost = Entry->Cost[CostKind])
2285 return LT.first * *KindCost;
2286
2287 static const CostKindTblEntry AVX512BWInLaneShuffleTbl[] = {
2288 {TTI::SK_PermuteTwoSrc, MVT::v64i8, {2, 2, 2, 2}}, // 2*vpshufb + vpor
2289 };
2290
2291 if (IsInLaneShuffle && ST->hasBWI())
2292 if (const auto *Entry =
2293 CostTableLookup(AVX512BWInLaneShuffleTbl, Kind, LT.second))
2294 if (auto KindCost = Entry->Cost[CostKind])
2295 return LT.first * *KindCost;
2296
2297 static const CostKindTblEntry AVX512BWShuffleTbl[] = {
2298 { TTI::SK_Broadcast, MVT::v32i16, { 1, 3, 1, 1 } }, // vpbroadcastw
2299 { TTI::SK_Broadcast, MVT::v32f16, { 1, 3, 1, 1 } }, // vpbroadcastw
2300 { TTI::SK_Broadcast, MVT::v64i8, { 1, 3, 1, 1 } }, // vpbroadcastb
2301
2302 { TTI::SK_Reverse, MVT::v32i16, { 2, 6, 2, 4 } }, // vpermw
2303 { TTI::SK_Reverse, MVT::v32f16, { 2, 6, 2, 4 } }, // vpermw
2304 { TTI::SK_Reverse, MVT::v16i16, { 2, 2, 2, 2 } }, // vpermw
2305 { TTI::SK_Reverse, MVT::v16f16, { 2, 2, 2, 2 } }, // vpermw
2306 { TTI::SK_Reverse, MVT::v64i8, { 2, 9, 2, 3 } }, // pshufb + vshufi64x2
2307
2308 { TTI::SK_PermuteSingleSrc, MVT::v32i16, { 2, 2, 2, 2 } }, // vpermw
2309 { TTI::SK_PermuteSingleSrc, MVT::v32f16, { 2, 2, 2, 2 } }, // vpermw
2310 { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 2, 2, 2, 2 } }, // vpermw
2311 { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 2, 2, 2, 2 } }, // vpermw
2312 { TTI::SK_PermuteSingleSrc, MVT::v64i8, { 8, 8, 8, 8 } }, // extend to v32i16
2313
2314 { TTI::SK_PermuteTwoSrc, MVT::v32i16,{ 2, 2, 2, 2 } }, // vpermt2w
2315 { TTI::SK_PermuteTwoSrc, MVT::v32f16,{ 2, 2, 2, 2 } }, // vpermt2w
2316 { TTI::SK_PermuteTwoSrc, MVT::v16i16,{ 2, 2, 2, 2 } }, // vpermt2w
2317 { TTI::SK_PermuteTwoSrc, MVT::v8i16, { 2, 2, 2, 2 } }, // vpermt2w
2318 { TTI::SK_PermuteTwoSrc, MVT::v64i8, { 19, 19, 19, 19 } }, // 6 * v32i8 + 1
2319
2320 { TTI::SK_Select, MVT::v32i16, { 1, 1, 1, 1 } }, // vblendmw
2321 { TTI::SK_Select, MVT::v64i8, { 1, 1, 1, 1 } }, // vblendmb
2322
2323 { TTI::SK_Splice, MVT::v32i16, { 2, 2, 2, 2 } }, // vshufi64x2 + palignr
2324 { TTI::SK_Splice, MVT::v32f16, { 2, 2, 2, 2 } }, // vshufi64x2 + palignr
2325 { TTI::SK_Splice, MVT::v64i8, { 2, 2, 2, 2 } }, // vshufi64x2 + palignr
2326 };
2327
2328 if (ST->hasBWI())
2329 if (const auto *Entry =
2330 CostTableLookup(AVX512BWShuffleTbl, Kind, LT.second))
2331 if (auto KindCost = Entry->Cost[CostKind])
2332 return LT.first * *KindCost;
2333
2334 static const CostKindTblEntry AVX512InLaneShuffleTbl[] = {
2335 {TTI::SK_PermuteTwoSrc, MVT::v8f64, { 1, 3, 1, 1 } },
2336 {TTI::SK_PermuteTwoSrc, MVT::v16f32, { 1, 3, 1, 1 } },
2337 {TTI::SK_PermuteTwoSrc, MVT::v8i64, { 1, 3, 1, 1 } },
2338 {TTI::SK_PermuteTwoSrc, MVT::v16i32, { 1, 3, 1, 1 } },
2339 {TTI::SK_PermuteTwoSrc, MVT::v4f64, { 1, 3, 1, 1 } },
2340 {TTI::SK_PermuteTwoSrc, MVT::v8f32, { 1, 3, 1, 1 } },
2341 {TTI::SK_PermuteTwoSrc, MVT::v4i64, { 1, 3, 1, 1 } },
2342 {TTI::SK_PermuteTwoSrc, MVT::v8i32, { 1, 3, 1, 1 } },
2343 };
2344
2345 if (IsInLaneShuffle && ST->hasAVX512())
2346 if (const auto *Entry =
2347 CostTableLookup(AVX512InLaneShuffleTbl, Kind, LT.second))
2348 if (auto KindCost = Entry->Cost[CostKind])
2349 return LT.first * *KindCost;
2350
2351 static const CostKindTblEntry AVX512ShuffleTbl[] = {
2352 {TTI::SK_Broadcast, MVT::v8f64, { 1, 3, 1, 1 } }, // vbroadcastsd
2353 {TTI::SK_Broadcast, MVT::v4f64, { 1, 3, 1, 1 } }, // vbroadcastsd
2354 {TTI::SK_Broadcast, MVT::v16f32, { 1, 3, 1, 1 } }, // vbroadcastss
2355 {TTI::SK_Broadcast, MVT::v8f32, { 1, 3, 1, 1 } }, // vbroadcastss
2356 {TTI::SK_Broadcast, MVT::v8i64, { 1, 3, 1, 1 } }, // vpbroadcastq
2357 {TTI::SK_Broadcast, MVT::v4i64, { 1, 3, 1, 1 } }, // vpbroadcastq
2358 {TTI::SK_Broadcast, MVT::v16i32, { 1, 3, 1, 1 } }, // vpbroadcastd
2359 {TTI::SK_Broadcast, MVT::v8i32, { 1, 3, 1, 1 } }, // vpbroadcastd
2360 {TTI::SK_Broadcast, MVT::v32i16, { 1, 3, 1, 1 } }, // vpbroadcastw
2361 {TTI::SK_Broadcast, MVT::v16i16, { 1, 3, 1, 1 } }, // vpbroadcastw
2362 {TTI::SK_Broadcast, MVT::v32f16, { 1, 3, 1, 1 } }, // vpbroadcastw
2363 {TTI::SK_Broadcast, MVT::v16f16, { 1, 3, 1, 1 } }, // vpbroadcastw
2364 {TTI::SK_Broadcast, MVT::v64i8, { 1, 3, 1, 1 } }, // vpbroadcastb
2365 {TTI::SK_Broadcast, MVT::v32i8, { 1, 3, 1, 1 }}, // vpbroadcastb
2366
2367 {TTI::SK_Reverse, MVT::v8f64, { 1, 5, 2, 3 } }, // vpermpd
2368 {TTI::SK_Reverse, MVT::v16f32, { 1, 3, 2, 3 } }, // vpermps
2369 {TTI::SK_Reverse, MVT::v8i64, { 1, 5, 2, 3 } }, // vpermq
2370 {TTI::SK_Reverse, MVT::v16i32, { 1, 3, 2, 3 } }, // vpermd
2371 {TTI::SK_Reverse, MVT::v32i16, { 7, 7, 7, 7 } }, // per mca
2372 {TTI::SK_Reverse, MVT::v32f16, { 7, 7, 7, 7 } }, // per mca
2373 {TTI::SK_Reverse, MVT::v64i8, { 7, 7, 7, 7 } }, // per mca
2374
2375 {TTI::SK_Splice, MVT::v8f64, { 1, 1, 1, 1 } }, // vpalignd
2376 {TTI::SK_Splice, MVT::v4f64, { 1, 1, 1, 1 } }, // vpalignd
2377 {TTI::SK_Splice, MVT::v16f32, { 1, 1, 1, 1 } }, // vpalignd
2378 {TTI::SK_Splice, MVT::v8f32, { 1, 1, 1, 1 } }, // vpalignd
2379 {TTI::SK_Splice, MVT::v8i64, { 1, 1, 1, 1 } }, // vpalignd
2380 {TTI::SK_Splice, MVT::v4i64, { 1, 1, 1, 1 } }, // vpalignd
2381 {TTI::SK_Splice, MVT::v16i32, { 1, 1, 1, 1 } }, // vpalignd
2382 {TTI::SK_Splice, MVT::v8i32, { 1, 1, 1, 1 } }, // vpalignd
2383 {TTI::SK_Splice, MVT::v32i16, { 4, 4, 4, 4 } }, // split + palignr
2384 {TTI::SK_Splice, MVT::v32f16, { 4, 4, 4, 4 } }, // split + palignr
2385 {TTI::SK_Splice, MVT::v64i8, { 4, 4, 4, 4 } }, // split + palignr
2386
2387 {TTI::SK_PermuteSingleSrc, MVT::v8f64, { 1, 3, 1, 1 } }, // vpermpd
2388 {TTI::SK_PermuteSingleSrc, MVT::v4f64, { 1, 3, 1, 1 } }, // vpermpd
2389 {TTI::SK_PermuteSingleSrc, MVT::v2f64, { 1, 3, 1, 1 } }, // vpermpd
2390 {TTI::SK_PermuteSingleSrc, MVT::v16f32, { 1, 3, 1, 1 } }, // vpermps
2391 {TTI::SK_PermuteSingleSrc, MVT::v8f32, { 1, 3, 1, 1 } }, // vpermps
2392 {TTI::SK_PermuteSingleSrc, MVT::v4f32, { 1, 3, 1, 1 } }, // vpermps
2393 {TTI::SK_PermuteSingleSrc, MVT::v8i64, { 1, 3, 1, 1 } }, // vpermq
2394 {TTI::SK_PermuteSingleSrc, MVT::v4i64, { 1, 3, 1, 1 } }, // vpermq
2395 {TTI::SK_PermuteSingleSrc, MVT::v2i64, { 1, 3, 1, 1 } }, // vpermq
2396 {TTI::SK_PermuteSingleSrc, MVT::v16i32, { 1, 3, 1, 1 } }, // vpermd
2397 {TTI::SK_PermuteSingleSrc, MVT::v8i32, { 1, 3, 1, 1 } }, // vpermd
2398 {TTI::SK_PermuteSingleSrc, MVT::v4i32, { 1, 3, 1, 1 } }, // vpermd
2399 {TTI::SK_PermuteSingleSrc, MVT::v16i8, { 1, 3, 1, 1 } }, // pshufb
2400
2401 {TTI::SK_PermuteTwoSrc, MVT::v8f64, { 2, 3, 1, 1 } }, // vpermt2pd
2402 {TTI::SK_PermuteTwoSrc, MVT::v16f32, { 2, 3, 1, 1 } }, // vpermt2ps
2403 {TTI::SK_PermuteTwoSrc, MVT::v8i64, { 2, 3, 1, 1 } }, // vpermt2q
2404 {TTI::SK_PermuteTwoSrc, MVT::v16i32, { 2, 3, 1, 1 } }, // vpermt2d
2405 {TTI::SK_PermuteTwoSrc, MVT::v4f64, { 2, 3, 1, 1 } }, // vpermt2pd
2406 {TTI::SK_PermuteTwoSrc, MVT::v8f32, { 2, 3, 1, 1 } }, // vpermt2ps
2407 {TTI::SK_PermuteTwoSrc, MVT::v4i64, { 2, 3, 1, 1 } }, // vpermt2q
2408 {TTI::SK_PermuteTwoSrc, MVT::v8i32, { 2, 3, 1, 1 } }, // vpermt2d
2409 {TTI::SK_PermuteTwoSrc, MVT::v2f64, { 1, 3, 1, 1 } },
2410 {TTI::SK_PermuteTwoSrc, MVT::v4f32, { 1, 3, 1, 1 } },
2411 {TTI::SK_PermuteTwoSrc, MVT::v2i64, { 1, 3, 1, 1 } },
2412 {TTI::SK_PermuteTwoSrc, MVT::v4i32, { 1, 3, 1, 1 } },
2413
2414 // FIXME: This just applies the type legalization cost rules above
2415 // assuming these completely split.
2416 {TTI::SK_PermuteSingleSrc, MVT::v32i16, { 14, 14, 14, 14 } },
2417 {TTI::SK_PermuteSingleSrc, MVT::v32f16, { 14, 14, 14, 14 } },
2418 {TTI::SK_PermuteSingleSrc, MVT::v64i8, { 14, 14, 14, 14 } },
2419 {TTI::SK_PermuteTwoSrc, MVT::v32i16, { 42, 42, 42, 42 } },
2420 {TTI::SK_PermuteTwoSrc, MVT::v32f16, { 42, 42, 42, 42 } },
2421 {TTI::SK_PermuteTwoSrc, MVT::v64i8, { 42, 42, 42, 42 } },
2422
2423 {TTI::SK_Select, MVT::v32i16, { 1, 1, 1, 1 } }, // vpternlogq
2424 {TTI::SK_Select, MVT::v32f16, { 1, 1, 1, 1 } }, // vpternlogq
2425 {TTI::SK_Select, MVT::v64i8, { 1, 1, 1, 1 } }, // vpternlogq
2426 {TTI::SK_Select, MVT::v8f64, { 1, 1, 1, 1 } }, // vblendmpd
2427 {TTI::SK_Select, MVT::v16f32, { 1, 1, 1, 1 } }, // vblendmps
2428 {TTI::SK_Select, MVT::v8i64, { 1, 1, 1, 1 } }, // vblendmq
2429 {TTI::SK_Select, MVT::v16i32, { 1, 1, 1, 1 } }, // vblendmd
2430 };
2431
2432 if (ST->hasAVX512())
2433 if (const auto *Entry = CostTableLookup(AVX512ShuffleTbl, Kind, LT.second))
2434 if (auto KindCost = Entry->Cost[CostKind])
2435 return LT.first * *KindCost;
2436
2437 static const CostKindTblEntry AVX2InLaneShuffleTbl[] = {
2438 { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 1, 1, 1, 1 } }, // vpshufb
2439 { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 1, 1, 1, 1 } }, // vpshufb
2440 { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 1, 1, 1, 1 } }, // vpshufb
2441
2442 { TTI::SK_Transpose, MVT::v4f64, { 1, 1, 1, 1 } }, // vshufpd/vunpck
2443 { TTI::SK_Transpose, MVT::v4i64, { 1, 1, 1, 1 } }, // vshufpd/vunpck
2444
2445 { TTI::SK_PermuteTwoSrc, MVT::v4f64, { 2, 2, 2, 2 } }, // 2*vshufpd + vblendpd
2446 { TTI::SK_PermuteTwoSrc, MVT::v8f32, { 2, 2, 2, 2 } }, // 2*vshufps + vblendps
2447 { TTI::SK_PermuteTwoSrc, MVT::v4i64, { 2, 2, 2, 2 } }, // 2*vpshufd + vpblendd
2448 { TTI::SK_PermuteTwoSrc, MVT::v8i32, { 2, 2, 2, 2 } }, // 2*vpshufd + vpblendd
2449 { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 2, 2, 2, 2 } }, // 2*vpshufb + vpor
2450 { TTI::SK_PermuteTwoSrc, MVT::v16f16, { 2, 2, 2, 2 } }, // 2*vpshufb + vpor
2451 { TTI::SK_PermuteTwoSrc, MVT::v32i8, { 2, 2, 2, 2 } }, // 2*vpshufb + vpor
2452 };
2453
2454 if (IsInLaneShuffle && ST->hasAVX2())
2455 if (const auto *Entry =
2456 CostTableLookup(AVX2InLaneShuffleTbl, Kind, LT.second))
2457 if (auto KindCost = Entry->Cost[CostKind])
2458 return LT.first * *KindCost;
2459
2460 static const CostKindTblEntry AVX2ShuffleTbl[] = {
2461 { TTI::SK_Broadcast, MVT::v4f64, { 1, 3, 1, 2 } }, // vbroadcastpd
2462 { TTI::SK_Broadcast, MVT::v8f32, { 1, 3, 1, 2 } }, // vbroadcastps
2463 { TTI::SK_Broadcast, MVT::v4i64, { 1, 3, 1, 2 } }, // vpbroadcastq
2464 { TTI::SK_Broadcast, MVT::v8i32, { 1, 3, 1, 2 } }, // vpbroadcastd
2465 { TTI::SK_Broadcast, MVT::v16i16, { 1, 3, 1, 2 } }, // vpbroadcastw
2466 { TTI::SK_Broadcast, MVT::v8i16, { 1, 3, 1, 1 } }, // vpbroadcastw
2467 { TTI::SK_Broadcast, MVT::v16f16, { 1, 3, 1, 2 } }, // vpbroadcastw
2468 { TTI::SK_Broadcast, MVT::v8f16, { 1, 3, 1, 1 } }, // vpbroadcastw
2469 { TTI::SK_Broadcast, MVT::v32i8, { 1, 3, 1, 2 } }, // vpbroadcastb
2470 { TTI::SK_Broadcast, MVT::v16i8, { 1, 3, 1, 1 } }, // vpbroadcastb
2471
2472 { TTI::SK_Reverse, MVT::v4f64, { 1, 6, 1, 2 } }, // vpermpd
2473 { TTI::SK_Reverse, MVT::v8f32, { 2, 7, 2, 4 } }, // vpermps
2474 { TTI::SK_Reverse, MVT::v4i64, { 1, 6, 1, 2 } }, // vpermq
2475 { TTI::SK_Reverse, MVT::v8i32, { 2, 7, 2, 4 } }, // vpermd
2476 { TTI::SK_Reverse, MVT::v16i16, { 2, 9, 2, 4 } }, // vperm2i128 + pshufb
2477 { TTI::SK_Reverse, MVT::v16f16, { 2, 9, 2, 4 } }, // vperm2i128 + pshufb
2478 { TTI::SK_Reverse, MVT::v32i8, { 2, 9, 2, 4 } }, // vperm2i128 + pshufb
2479
2480 { TTI::SK_Select, MVT::v16i16, { 1, 1, 1, 1 } }, // vpblendvb
2481 { TTI::SK_Select, MVT::v16f16, { 1, 1, 1, 1 } }, // vpblendvb
2482 { TTI::SK_Select, MVT::v32i8, { 1, 1, 1, 1 } }, // vpblendvb
2483
2484 { TTI::SK_Splice, MVT::v8i32, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
2485 { TTI::SK_Splice, MVT::v8f32, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
2486 { TTI::SK_Splice, MVT::v16i16, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
2487 { TTI::SK_Splice, MVT::v16f16, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
2488 { TTI::SK_Splice, MVT::v32i8, { 2, 2, 2, 2 } }, // vperm2i128 + vpalignr
2489
2490 { TTI::SK_PermuteSingleSrc, MVT::v4f64, { 1, 1, 1, 1 } }, // vpermpd
2491 { TTI::SK_PermuteSingleSrc, MVT::v8f32, { 1, 1, 1, 1 } }, // vpermps
2492 { TTI::SK_PermuteSingleSrc, MVT::v4i64, { 1, 1, 1, 1 } }, // vpermq
2493 { TTI::SK_PermuteSingleSrc, MVT::v8i32, { 1, 1, 1, 1 } }, // vpermd
2494 { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 4, 4, 4, 4 } },
2495 { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 4, 4, 4, 4 } },
2496 { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 4, 4, 4, 4 } },
2497
2498 { TTI::SK_PermuteTwoSrc, MVT::v4f64, { 3, 3, 3, 3 } }, // 2*vpermpd + vblendpd
2499 { TTI::SK_PermuteTwoSrc, MVT::v8f32, { 3, 3, 3, 3 } }, // 2*vpermps + vblendps
2500 { TTI::SK_PermuteTwoSrc, MVT::v4i64, { 3, 3, 3, 3 } }, // 2*vpermq + vpblendd
2501 { TTI::SK_PermuteTwoSrc, MVT::v8i32, { 3, 3, 3, 3 } }, // 2*vpermd + vpblendd
2502 { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 7, 7, 7, 7 } },
2503 { TTI::SK_PermuteTwoSrc, MVT::v16f16, { 7, 7, 7, 7 } },
2504 { TTI::SK_PermuteTwoSrc, MVT::v32i8, { 7, 7, 7, 7 } },
2505 };
2506
2507 if (ST->hasAVX2())
2508 if (const auto *Entry = CostTableLookup(AVX2ShuffleTbl, Kind, LT.second))
2509 if (auto KindCost = Entry->Cost[CostKind])
2510 return LT.first * *KindCost;
2511
2512 static const CostKindTblEntry XOPShuffleTbl[] = {
2513 { TTI::SK_PermuteSingleSrc, MVT::v4f64, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2pd
2514 { TTI::SK_PermuteSingleSrc, MVT::v8f32, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2ps
2515 { TTI::SK_PermuteSingleSrc, MVT::v4i64, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2pd
2516 { TTI::SK_PermuteSingleSrc, MVT::v8i32, { 2, 2, 2, 2 } }, // vperm2f128 + vpermil2ps
2517 { TTI::SK_PermuteSingleSrc, MVT::v16i16,{ 4, 4, 4, 4 } }, // vextractf128 + 2*vpperm
2518 // + vinsertf128
2519 { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 4, 4, 4, 4 } }, // vextractf128 + 2*vpperm
2520 // + vinsertf128
2521
2522 { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 9, 9, 9, 9 } }, // 2*vextractf128 + 6*vpperm
2523 // + vinsertf128
2524
2525 { TTI::SK_PermuteTwoSrc, MVT::v8i16, { 1, 1, 1, 1 } }, // vpperm
2526 { TTI::SK_PermuteTwoSrc, MVT::v32i8, { 9, 9, 9, 9 } }, // 2*vextractf128 + 6*vpperm
2527 // + vinsertf128
2528 { TTI::SK_PermuteTwoSrc, MVT::v16i8, { 1, 1, 1, 1 } }, // vpperm
2529 };
2530
2531 if (ST->hasXOP())
2532 if (const auto *Entry = CostTableLookup(XOPShuffleTbl, Kind, LT.second))
2533 if (auto KindCost = Entry->Cost[CostKind])
2534 return LT.first * *KindCost;
2535
2536 static const CostKindTblEntry AVX1InLaneShuffleTbl[] = {
2537 { TTI::SK_PermuteSingleSrc, MVT::v4f64, { 1, 1, 1, 1 } }, // vpermilpd
2538 { TTI::SK_PermuteSingleSrc, MVT::v4i64, { 1, 1, 1, 1 } }, // vpermilpd
2539 { TTI::SK_PermuteSingleSrc, MVT::v8f32, { 1, 1, 1, 1 } }, // vpermilps
2540 { TTI::SK_PermuteSingleSrc, MVT::v8i32, { 1, 1, 1, 1 } }, // vpermilps
2541
2542 { TTI::SK_PermuteSingleSrc, MVT::v16i16, { 4, 4, 4, 4 } }, // vextractf128 + 2*pshufb
2543 // + vpor + vinsertf128
2544 { TTI::SK_PermuteSingleSrc, MVT::v16f16, { 4, 4, 4, 4 } }, // vextractf128 + 2*pshufb
2545 // + vpor + vinsertf128
2546 { TTI::SK_PermuteSingleSrc, MVT::v32i8, { 4, 4, 4, 4 } }, // vextractf128 + 2*pshufb
2547 // + vpor + vinsertf128
2548
2549 { TTI::SK_Transpose, MVT::v4f64, { 1, 1, 1, 1 } }, // vshufpd/vunpck
2550 { TTI::SK_Transpose, MVT::v4i64, { 1, 1, 1, 1 } }, // vshufpd/vunpck
2551
2552 { TTI::SK_PermuteTwoSrc, MVT::v4f64, { 2, 2, 2, 2 } }, // 2*vshufpd + vblendpd
2553 { TTI::SK_PermuteTwoSrc, MVT::v8f32, { 2, 2, 2, 2 } }, // 2*vshufps + vblendps
2554 { TTI::SK_PermuteTwoSrc, MVT::v4i64, { 2, 2, 2, 2 } }, // 2*vpermilpd + vblendpd
2555 { TTI::SK_PermuteTwoSrc, MVT::v8i32, { 2, 2, 2, 2 } }, // 2*vpermilps + vblendps
2556 { TTI::SK_PermuteTwoSrc, MVT::v16i16, { 9, 9, 9, 9 } }, // 2*vextractf128 + 4*pshufb
2557 // + 2*vpor + vinsertf128
2558 { TTI::SK_PermuteTwoSrc, MVT::v16f16, { 9, 9, 9, 9 } }, // 2*vextractf128 + 4*pshufb
2559 // + 2*vpor + vinsertf128
2560 { TTI::SK_PermuteTwoSrc, MVT::v32i8, { 9, 9, 9, 9 } }, // 2*vextractf128 + 4*pshufb
2561 // + 2*vpor + vinsertf128
2562 };
2563
2564 if (IsInLaneShuffle && ST->hasAVX())
2565 if (const auto *Entry =
2566 CostTableLookup(AVX1InLaneShuffleTbl, Kind, LT.second))
2567 if (auto KindCost = Entry->Cost[CostKind])
2568 return LT.first * *KindCost;
2569
2570 static const CostKindTblEntry AVX1ShuffleTbl[] = {
2571 {TTI::SK_Broadcast, MVT::v4f64, {2,3,2,3}}, // vperm2f128 + vpermilpd
2572 {TTI::SK_Broadcast, MVT::v8f32, {2,3,2,3}}, // vperm2f128 + vpermilps
2573 {TTI::SK_Broadcast, MVT::v4i64, {2,3,2,3}}, // vperm2f128 + vpermilpd
2574 {TTI::SK_Broadcast, MVT::v8i32, {2,3,2,3}}, // vperm2f128 + vpermilps
2575 {TTI::SK_Broadcast, MVT::v16i16, {2,3,3,4}}, // vpshuflw + vpshufd + vinsertf128
2576 {TTI::SK_Broadcast, MVT::v16f16, {2,3,3,4}}, // vpshuflw + vpshufd + vinsertf128
2577 {TTI::SK_Broadcast, MVT::v32i8, {3,4,3,6}}, // vpshufb + vinsertf128
2578
2579 {TTI::SK_Reverse, MVT::v4f64, {2,6,2,2}}, // vperm2f128 + vpermilpd
2580 {TTI::SK_Reverse, MVT::v8f32, {2,7,2,4}}, // vperm2f128 + vpermilps
2581 {TTI::SK_Reverse, MVT::v4i64, {2,6,2,2}}, // vperm2f128 + vpermilpd
2582 {TTI::SK_Reverse, MVT::v8i32, {2,7,2,4}}, // vperm2f128 + vpermilps
2583 {TTI::SK_Reverse, MVT::v16i16, {2,9,5,5}}, // vextractf128 + 2*pshufb
2584 // + vinsertf128
2585 {TTI::SK_Reverse, MVT::v16f16, {2,9,5,5}}, // vextractf128 + 2*pshufb
2586 // + vinsertf128
2587 {TTI::SK_Reverse, MVT::v32i8, {2,9,5,5}}, // vextractf128 + 2*pshufb
2588 // + vinsertf128
2589
2590 {TTI::SK_Select, MVT::v4i64, {1,1,1,1}}, // vblendpd
2591 {TTI::SK_Select, MVT::v4f64, {1,1,1,1}}, // vblendpd
2592 {TTI::SK_Select, MVT::v8i32, {1,1,1,1}}, // vblendps
2593 {TTI::SK_Select, MVT::v8f32, {1,1,1,1}}, // vblendps
2594 {TTI::SK_Select, MVT::v16i16, {3,3,3,3}}, // vpand + vpandn + vpor
2595 {TTI::SK_Select, MVT::v16f16, {3,3,3,3}}, // vpand + vpandn + vpor
2596 {TTI::SK_Select, MVT::v32i8, {3,3,3,3}}, // vpand + vpandn + vpor
2597
2598 {TTI::SK_Splice, MVT::v4i64, {2,2,2,2}}, // vperm2f128 + shufpd
2599 {TTI::SK_Splice, MVT::v4f64, {2,2,2,2}}, // vperm2f128 + shufpd
2600 {TTI::SK_Splice, MVT::v8i32, {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
2601 {TTI::SK_Splice, MVT::v8f32, {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
2602 {TTI::SK_Splice, MVT::v16i16, {5,5,5,5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
2603 {TTI::SK_Splice, MVT::v16f16, {5,5,5,5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
2604 {TTI::SK_Splice, MVT::v32i8, {5,5,5,5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
2605
2606 {TTI::SK_PermuteSingleSrc, MVT::v4f64, {2,2,2,2}}, // vperm2f128 + vshufpd
2607 {TTI::SK_PermuteSingleSrc, MVT::v4i64, {2,2,2,2}}, // vperm2f128 + vshufpd
2608 {TTI::SK_PermuteSingleSrc, MVT::v8f32, {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
2609 {TTI::SK_PermuteSingleSrc, MVT::v8i32, {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
2610 {TTI::SK_PermuteSingleSrc, MVT::v16i16,{8,8,8,8}}, // vextractf128 + 4*pshufb
2611 // + 2*por + vinsertf128
2612 {TTI::SK_PermuteSingleSrc, MVT::v16f16,{8,8,8,8}}, // vextractf128 + 4*pshufb
2613 // + 2*por + vinsertf128
2614 {TTI::SK_PermuteSingleSrc, MVT::v32i8, {8,8,8,8}}, // vextractf128 + 4*pshufb
2615 // + 2*por + vinsertf128
2616
2617 {TTI::SK_PermuteTwoSrc, MVT::v4f64, {3,3,3,3}}, // 2*vperm2f128 + vshufpd
2618 {TTI::SK_PermuteTwoSrc, MVT::v4i64, {3,3,3,3}}, // 2*vperm2f128 + vshufpd
2619 {TTI::SK_PermuteTwoSrc, MVT::v8f32, {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
2620 {TTI::SK_PermuteTwoSrc, MVT::v8i32, {4,4,4,4}}, // 2*vperm2f128 + 2*vshufps
2621 {TTI::SK_PermuteTwoSrc, MVT::v16i16,{15,15,15,15}}, // 2*vextractf128 + 8*pshufb
2622 // + 4*por + vinsertf128
2623 {TTI::SK_PermuteTwoSrc, MVT::v16f16,{15,15,15,15}}, // 2*vextractf128 + 8*pshufb
2624 // + 4*por + vinsertf128
2625 {TTI::SK_PermuteTwoSrc, MVT::v32i8, {15,15,15,15}}, // 2*vextractf128 + 8*pshufb
2626 // + 4*por + vinsertf128
2627 };
2628
2629 if (ST->hasAVX())
2630 if (const auto *Entry = CostTableLookup(AVX1ShuffleTbl, Kind, LT.second))
2631 if (auto KindCost = Entry->Cost[CostKind])
2632 return LT.first * *KindCost;
2633
2634 static const CostKindTblEntry SSE41ShuffleTbl[] = {
2635 {TTI::SK_Select, MVT::v2i64, {1,1,1,1}}, // pblendw
2636 {TTI::SK_Select, MVT::v2f64, {1,1,1,1}}, // movsd
2637 {TTI::SK_Select, MVT::v4i32, {1,1,1,1}}, // pblendw
2638 {TTI::SK_Select, MVT::v4f32, {1,1,1,1}}, // blendps
2639 {TTI::SK_Select, MVT::v8i16, {1,1,1,1}}, // pblendw
2640 {TTI::SK_Select, MVT::v8f16, {1,1,1,1}}, // pblendw
2641 {TTI::SK_Select, MVT::v16i8, {1,1,1,1}} // pblendvb
2642 };
2643
2644 if (ST->hasSSE41())
2645 if (const auto *Entry = CostTableLookup(SSE41ShuffleTbl, Kind, LT.second))
2646 if (auto KindCost = Entry->Cost[CostKind])
2647 return LT.first * *KindCost;
2648
2649 static const CostKindTblEntry SSSE3ShuffleTbl[] = {
2650 {TTI::SK_Broadcast, MVT::v8i16, {1, 3, 2, 2}}, // pshufb
2651 {TTI::SK_Broadcast, MVT::v8f16, {1, 3, 2, 2}}, // pshufb
2652 {TTI::SK_Broadcast, MVT::v16i8, {1, 3, 2, 2}}, // pshufb
2653
2654 {TTI::SK_Reverse, MVT::v8i16, {1, 2, 1, 2}}, // pshufb
2655 {TTI::SK_Reverse, MVT::v8f16, {1, 2, 1, 2}}, // pshufb
2656 {TTI::SK_Reverse, MVT::v16i8, {1, 2, 1, 2}}, // pshufb
2657
2658 {TTI::SK_Splice, MVT::v4i32, {1, 1, 1, 1}}, // palignr
2659 {TTI::SK_Splice, MVT::v4f32, {1, 1, 1, 1}}, // palignr
2660 {TTI::SK_Splice, MVT::v8i16, {1, 1, 1, 1}}, // palignr
2661 {TTI::SK_Splice, MVT::v8f16, {1, 1, 1, 1}}, // palignr
2662 {TTI::SK_Splice, MVT::v16i8, {1, 1, 1, 1}}, // palignr
2663
2664 {TTI::SK_PermuteSingleSrc, MVT::v8i16, {1, 1, 1, 1}}, // pshufb
2665 {TTI::SK_PermuteSingleSrc, MVT::v8f16, {1, 1, 1, 1}}, // pshufb
2666 {TTI::SK_PermuteSingleSrc, MVT::v16i8, {1, 1, 1, 1}}, // pshufb
2667
2668 {TTI::SK_PermuteTwoSrc, MVT::v8i16, {3, 3, 3, 3}}, // 2*pshufb + por
2669 {TTI::SK_PermuteTwoSrc, MVT::v8f16, {3, 3, 3, 3}}, // 2*pshufb + por
2670 {TTI::SK_PermuteTwoSrc, MVT::v16i8, {3, 3, 3, 3}}, // 2*pshufb + por
2671 };
2672
2673 if (ST->hasSSSE3())
2674 if (const auto *Entry = CostTableLookup(SSSE3ShuffleTbl, Kind, LT.second))
2675 if (auto KindCost = Entry->Cost[CostKind])
2676 return LT.first * *KindCost;
2677
2678 static const CostKindTblEntry SSE2ShuffleTbl[] = {
2679 {TTI::SK_Broadcast, MVT::v2f64, {1, 1, 1, 1}}, // shufpd
2680 {TTI::SK_Broadcast, MVT::v2i64, {1, 1, 1, 1}}, // pshufd
2681 {TTI::SK_Broadcast, MVT::v4i32, {1, 1, 1, 1}}, // pshufd
2682 {TTI::SK_Broadcast, MVT::v8i16, {1, 2, 2, 2}}, // pshuflw + pshufd
2683 {TTI::SK_Broadcast, MVT::v8f16, {1, 2, 2, 2}}, // pshuflw + pshufd
2684 {TTI::SK_Broadcast, MVT::v16i8, {2, 3, 3, 4}}, // unpck + pshuflw + pshufd
2685
2686 {TTI::SK_Reverse, MVT::v2f64, {1, 1, 1, 1}}, // shufpd
2687 {TTI::SK_Reverse, MVT::v2i64, {1, 1, 1, 1}}, // pshufd
2688 {TTI::SK_Reverse, MVT::v4i32, {1, 1, 1, 1}}, // pshufd
2689 {TTI::SK_Reverse, MVT::v8i16, {2, 3, 3, 3}}, // pshuflw + pshufhw + pshufd
2690 {TTI::SK_Reverse, MVT::v8f16, {2, 3, 3, 3}}, // pshuflw + pshufhw + pshufd
2691 {TTI::SK_Reverse, MVT::v16i8, {5, 6,11,11}}, // 2*pshuflw + 2*pshufhw
2692 // + 2*pshufd + 2*unpck + packus
2693
2694 {TTI::SK_Select, MVT::v2i64, {1, 1, 1, 1}}, // movsd
2695 {TTI::SK_Select, MVT::v2f64, {1, 1, 1, 1}}, // movsd
2696 {TTI::SK_Select, MVT::v4i32, {2, 2, 2, 2}}, // 2*shufps
2697 {TTI::SK_Select, MVT::v8i16, {2, 2, 3, 3}}, // pand + pandn + por
2698 {TTI::SK_Select, MVT::v8f16, {2, 2, 3, 3}}, // pand + pandn + por
2699 {TTI::SK_Select, MVT::v16i8, {2, 2, 3, 3}}, // pand + pandn + por
2700
2701 {TTI::SK_Splice, MVT::v2i64, {1, 1, 1, 1}}, // shufpd
2702 {TTI::SK_Splice, MVT::v2f64, {1, 1, 1, 1}}, // shufpd
2703 {TTI::SK_Splice, MVT::v4i32, {2, 2, 2, 2}}, // 2*{unpck,movsd,pshufd}
2704 {TTI::SK_Splice, MVT::v8i16, {3, 3, 3, 3}}, // psrldq + psrlldq + por
2705 {TTI::SK_Splice, MVT::v8f16, {3, 3, 3, 3}}, // psrldq + psrlldq + por
2706 {TTI::SK_Splice, MVT::v16i8, {3, 3, 3, 3}}, // psrldq + psrlldq + por
2707
2708 {TTI::SK_PermuteSingleSrc, MVT::v2f64, {1, 1, 1, 1}}, // shufpd
2709 {TTI::SK_PermuteSingleSrc, MVT::v2i64, {1, 1, 1, 1}}, // pshufd
2710 {TTI::SK_PermuteSingleSrc, MVT::v4i32, {1, 1, 1, 1}}, // pshufd
2711 {TTI::SK_PermuteSingleSrc, MVT::v8i16, {3, 5, 5, 5}}, // 2*pshuflw + 2*pshufhw
2712 // + pshufd/unpck
2713 {TTI::SK_PermuteSingleSrc, MVT::v8f16, {3, 5, 5, 5}}, // 2*pshuflw + 2*pshufhw
2714 // + pshufd/unpck
2715 {TTI::SK_PermuteSingleSrc, MVT::v16i8, {8, 10, 10, 10}}, // 2*pshuflw + 2*pshufhw
2716 // + 2*pshufd + 2*unpck + 2*packus
2717
2718 {TTI::SK_PermuteTwoSrc, MVT::v2f64, {1, 1, 1, 1}}, // shufpd
2719 {TTI::SK_PermuteTwoSrc, MVT::v2i64, {1, 1, 1, 1}}, // shufpd
2720 {TTI::SK_PermuteTwoSrc, MVT::v4i32, {2, 2, 2, 2}}, // 2*{unpck,movsd,pshufd}
2721 {TTI::SK_PermuteTwoSrc, MVT::v8i16, {6, 8, 8, 8}}, // blend+permute
2722 {TTI::SK_PermuteTwoSrc, MVT::v8f16, {6, 8, 8, 8}}, // blend+permute
2723 {TTI::SK_PermuteTwoSrc, MVT::v16i8, {11, 13, 13, 13}}, // blend+permute
2724 };
2725
2726 static const CostTblEntry SSE3BroadcastLoadTbl[] = {
2727 {TTI::SK_Broadcast, MVT::v2f64, 0}, // broadcast handled by movddup
2728 };
2729
2730 if (ST->hasSSE2()) {
2731 bool IsLoad =
2732 llvm::any_of(Args, [](const auto &V) { return isa<LoadInst>(V); });
2733 if (ST->hasSSE3() && IsLoad)
2734 if (const auto *Entry =
2735 CostTableLookup(SSE3BroadcastLoadTbl, Kind, LT.second)) {
2736 assert(isLegalBroadcastLoad(SrcTy->getElementType(),
2737 LT.second.getVectorElementCount()) &&
2738 "Table entry missing from isLegalBroadcastLoad()");
2739 return LT.first * Entry->Cost;
2740 }
2741
2742 if (const auto *Entry = CostTableLookup(SSE2ShuffleTbl, Kind, LT.second))
2743 if (auto KindCost = Entry->Cost[CostKind])
2744 return LT.first * *KindCost;
2745 }
2746
2747 static const CostKindTblEntry SSE1ShuffleTbl[] = {
2748 { TTI::SK_Broadcast, MVT::v4f32, {1,1,1,1} }, // shufps
2749 { TTI::SK_Reverse, MVT::v4f32, {1,1,1,1} }, // shufps
2750 { TTI::SK_Select, MVT::v4f32, {2,2,2,2} }, // 2*shufps
2751 { TTI::SK_Splice, MVT::v4f32, {2,2,2,2} }, // 2*shufps
2752 { TTI::SK_PermuteSingleSrc, MVT::v4f32, {1,1,1,1} }, // shufps
2753 { TTI::SK_PermuteTwoSrc, MVT::v4f32, {2,2,2,2} }, // 2*shufps
2754 };
2755
2756 if (ST->hasSSE1()) {
2757 if (LT.first == 1 && LT.second == MVT::v4f32 && Mask.size() == 4) {
2758 // SHUFPS: both pairs must come from the same source register.
2759 auto MatchSHUFPS = [](int X, int Y) {
2760 return X < 0 || Y < 0 || ((X & 4) == (Y & 4));
2761 };
2762 if (MatchSHUFPS(Mask[0], Mask[1]) && MatchSHUFPS(Mask[2], Mask[3]))
2763 return 1;
2764 }
2765 if (const auto *Entry = CostTableLookup(SSE1ShuffleTbl, Kind, LT.second))
2766 if (auto KindCost = Entry->Cost[CostKind])
2767 return LT.first * *KindCost;
2768 }
2769
2770 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
2771 SubTp);
2772}
2773
2775 Type *Src,
2777 TTI::TargetCostKind CostKind,
2778 const Instruction *I) const {
2779 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2780 assert(ISD && "Invalid opcode");
2781
2782 // A narrow (i8/i16) zero-extension used as a GEP *index* can be folded into
2783 // the addressing mode of the consuming memory op, but only if the source is
2784 // already materialised zero-extended in a full register. X86's SIB form
2785 // [base + index*scale + disp] reads the index at full width and does NOT
2786 // zero-extend a narrow index (unlike AArch64's uxtw-extended addressing), so
2787 // a "dirty" narrow source (e.g. an i16 add result used only as an index)
2788 // still needs a dedicated movzx and is not free. Price it as free only with
2789 // positive evidence that no movzx is required.
2790 if (ISD == ISD::ZERO_EXTEND && I && I->hasOneUse() && Src->isIntegerTy() &&
2791 Src->getScalarSizeInBits() < 32) {
2792 const Use &U = *I->use_begin();
2793 if (isa<GetElementPtrInst>(U.getUser()) &&
2794 U.getOperandNo() != GetElementPtrInst::getPointerOperandIndex()) {
2795 const Value *Op = I->getOperand(0);
2796 // Clean sources: an extending load, a zeroext argument, or a value whose
2797 // high bits are provably zero (e.g. from a shift/mask). These mirror the
2798 // proof-based reasoning the middle end uses elsewhere (ValueTracking and
2799 // InstCombine's canEvaluateZExtd); we intentionally do NOT treat a merely
2800 // multiply-used operand as clean, since that is a guess rather than
2801 // proof.
2802 if (isa<LoadInst>(Op))
2803 return TTI::TCC_Free;
2804 if (const auto *A = dyn_cast<Argument>(Op))
2805 if (A->hasAttribute(Attribute::ZExt))
2806 return TTI::TCC_Free;
2807 if (computeKnownBits(Op, I->getDataLayout(), /*AC=*/nullptr, I)
2808 .countMinLeadingZeros() > 0)
2809 return TTI::TCC_Free;
2810 }
2811 }
2812
2813 // The cost tables include both specific, custom (non-legal) src/dst type
2814 // conversions and generic, legalized types. We test for customs first, before
2815 // falling back to legalization.
2816 // FIXME: Need a better design of the cost table to handle non-simple types of
2817 // potential massive combinations (elem_num x src_type x dst_type).
2818 static const TypeConversionCostKindTblEntry AVX512BWConversionTbl[]{
2819 { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v32i8, { 1, 1, 1, 1 } },
2820 { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v32i8, { 1, 1, 1, 1 } },
2821
2822 // Mask sign extend has an instruction.
2823 { ISD::SIGN_EXTEND, MVT::v2i8, MVT::v2i1, { 1, 1, 1, 1 } },
2824 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v2i1, { 1, 1, 1, 1 } },
2825 { ISD::SIGN_EXTEND, MVT::v2i16, MVT::v2i1, { 1, 1, 1, 1 } },
2826 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v2i1, { 1, 1, 1, 1 } },
2827 { ISD::SIGN_EXTEND, MVT::v4i8, MVT::v4i1, { 1, 1, 1, 1 } },
2828 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v4i1, { 1, 1, 1, 1 } },
2829 { ISD::SIGN_EXTEND, MVT::v4i16, MVT::v4i1, { 1, 1, 1, 1 } },
2830 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v4i1, { 1, 1, 1, 1 } },
2831 { ISD::SIGN_EXTEND, MVT::v8i8, MVT::v8i1, { 1, 1, 1, 1 } },
2832 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v8i1, { 1, 1, 1, 1 } },
2833 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i1, { 1, 1, 1, 1 } },
2834 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v16i1, { 1, 1, 1, 1 } },
2835 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1, { 1, 1, 1, 1 } },
2836 { ISD::SIGN_EXTEND, MVT::v32i8, MVT::v32i1, { 1, 1, 1, 1 } },
2837 { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v32i1, { 1, 1, 1, 1 } },
2838 { ISD::SIGN_EXTEND, MVT::v64i8, MVT::v64i1, { 1, 1, 1, 1 } },
2839 { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v64i1, { 1, 1, 1, 1 } },
2840
2841 // Mask zero extend is a sext + shift.
2842 { ISD::ZERO_EXTEND, MVT::v2i8, MVT::v2i1, { 2, 1, 1, 1 } },
2843 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v2i1, { 2, 1, 1, 1 } },
2844 { ISD::ZERO_EXTEND, MVT::v2i16, MVT::v2i1, { 2, 1, 1, 1 } },
2845 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v2i1, { 2, 1, 1, 1 } },
2846 { ISD::ZERO_EXTEND, MVT::v4i8, MVT::v4i1, { 2, 1, 1, 1 } },
2847 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v4i1, { 2, 1, 1, 1 } },
2848 { ISD::ZERO_EXTEND, MVT::v4i16, MVT::v4i1, { 2, 1, 1, 1 } },
2849 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v4i1, { 2, 1, 1, 1 } },
2850 { ISD::ZERO_EXTEND, MVT::v8i8, MVT::v8i1, { 2, 1, 1, 1 } },
2851 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v8i1, { 2, 1, 1, 1 } },
2852 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i1, { 2, 1, 1, 1 } },
2853 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v16i1, { 2, 1, 1, 1 } },
2854 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1, { 2, 1, 1, 1 } },
2855 { ISD::ZERO_EXTEND, MVT::v32i8, MVT::v32i1, { 2, 1, 1, 1 } },
2856 { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v32i1, { 2, 1, 1, 1 } },
2857 { ISD::ZERO_EXTEND, MVT::v64i8, MVT::v64i1, { 2, 1, 1, 1 } },
2858 { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v64i1, { 2, 1, 1, 1 } },
2859
2860 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i8, { 2, 1, 1, 1 } },
2861 { ISD::TRUNCATE, MVT::v2i1, MVT::v16i8, { 2, 1, 1, 1 } },
2862 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i16, { 2, 1, 1, 1 } },
2863 { ISD::TRUNCATE, MVT::v2i1, MVT::v8i16, { 2, 1, 1, 1 } },
2864 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i8, { 2, 1, 1, 1 } },
2865 { ISD::TRUNCATE, MVT::v4i1, MVT::v16i8, { 2, 1, 1, 1 } },
2866 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i16, { 2, 1, 1, 1 } },
2867 { ISD::TRUNCATE, MVT::v4i1, MVT::v8i16, { 2, 1, 1, 1 } },
2868 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i8, { 2, 1, 1, 1 } },
2869 { ISD::TRUNCATE, MVT::v8i1, MVT::v16i8, { 2, 1, 1, 1 } },
2870 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i16, { 2, 1, 1, 1 } },
2871 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i8, { 2, 1, 1, 1 } },
2872 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i16, { 2, 1, 1, 1 } },
2873 { ISD::TRUNCATE, MVT::v32i1, MVT::v32i8, { 2, 1, 1, 1 } },
2874 { ISD::TRUNCATE, MVT::v32i1, MVT::v32i16, { 2, 1, 1, 1 } },
2875 { ISD::TRUNCATE, MVT::v64i1, MVT::v64i8, { 2, 1, 1, 1 } },
2876 { ISD::TRUNCATE, MVT::v64i1, MVT::v32i16, { 2, 1, 1, 1 } },
2877
2878 { ISD::TRUNCATE, MVT::v32i8, MVT::v32i16, { 2, 1, 1, 1 } },
2879 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, { 2, 1, 1, 1 } }, // widen to zmm
2880 { ISD::TRUNCATE, MVT::v2i8, MVT::v2i16, { 2, 1, 1, 1 } }, // vpmovwb
2881 { ISD::TRUNCATE, MVT::v4i8, MVT::v4i16, { 2, 1, 1, 1 } }, // vpmovwb
2882 { ISD::TRUNCATE, MVT::v8i8, MVT::v8i16, { 2, 1, 1, 1 } }, // vpmovwb
2883 };
2884
2885 static const TypeConversionCostKindTblEntry AVX512DQConversionTbl[] = {
2886 // Mask sign extend has an instruction.
2887 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i1, { 1, 1, 1, 1 } },
2888 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v2i1, { 1, 1, 1, 1 } },
2889 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i1, { 1, 1, 1, 1 } },
2890 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i1, { 1, 1, 1, 1 } },
2891 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1, { 1, 1, 1, 1 } },
2892 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v16i1, { 1, 1, 1, 1 } },
2893 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i1, { 1, 1, 1, 1 } },
2894 { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i1, { 1, 1, 1, 1 } },
2895
2896 // Mask zero extend is a sext + shift.
2897 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i1, { 2, 1, 1, 1, } },
2898 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v2i1, { 2, 1, 1, 1, } },
2899 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i1, { 2, 1, 1, 1, } },
2900 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i1, { 2, 1, 1, 1, } },
2901 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1, { 2, 1, 1, 1, } },
2902 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v16i1, { 2, 1, 1, 1, } },
2903 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i1, { 2, 1, 1, 1, } },
2904 { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i1, { 2, 1, 1, 1, } },
2905
2906 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i64, { 2, 1, 1, 1 } },
2907 { ISD::TRUNCATE, MVT::v2i1, MVT::v4i32, { 2, 1, 1, 1 } },
2908 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i32, { 2, 1, 1, 1 } },
2909 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i64, { 2, 1, 1, 1 } },
2910 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i32, { 2, 1, 1, 1 } },
2911 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i64, { 2, 1, 1, 1 } },
2912 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i32, { 2, 1, 1, 1 } },
2913 { ISD::TRUNCATE, MVT::v16i1, MVT::v8i64, { 2, 1, 1, 1 } },
2914
2915 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i64, { 1, 1, 1, 1 } },
2916 { ISD::SINT_TO_FP, MVT::v8f64, MVT::v8i64, { 1, 1, 1, 1 } },
2917
2918 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i64, { 1, 1, 1, 1 } },
2919 { ISD::UINT_TO_FP, MVT::v8f64, MVT::v8i64, { 1, 1, 1, 1 } },
2920
2921 { ISD::FP_TO_SINT, MVT::v8i64, MVT::v8f32, { 1, 1, 1, 1 } },
2922 { ISD::FP_TO_SINT, MVT::v8i64, MVT::v8f64, { 1, 1, 1, 1 } },
2923
2924 { ISD::FP_TO_UINT, MVT::v8i64, MVT::v8f32, { 1, 1, 1, 1 } },
2925 { ISD::FP_TO_UINT, MVT::v8i64, MVT::v8f64, { 1, 1, 1, 1 } },
2926 };
2927
2928 // TODO: For AVX512DQ + AVX512VL, we also have cheap casts for 128-bit and
2929 // 256-bit wide vectors.
2930
2931 static const TypeConversionCostKindTblEntry AVX512FConversionTbl[] = {
2932 { ISD::FP_EXTEND, MVT::v8f64, MVT::v8f32, { 1, 1, 1, 1 } },
2933 { ISD::FP_EXTEND, MVT::v8f64, MVT::v16f32, { 3, 1, 1, 1 } },
2934 { ISD::FP_EXTEND, MVT::v16f64, MVT::v16f32, { 4, 1, 1, 1 } }, // 2*vcvtps2pd+vextractf64x4
2935 { ISD::FP_EXTEND, MVT::v16f32, MVT::v16f16, { 1, 1, 1, 1 } }, // vcvtph2ps
2936 { ISD::FP_EXTEND, MVT::v8f64, MVT::v8f16, { 2, 1, 1, 1 } }, // vcvtph2ps+vcvtps2pd
2937 { ISD::FP_ROUND, MVT::v8f32, MVT::v8f64, { 1, 1, 1, 1 } },
2938 { ISD::FP_ROUND, MVT::v16f16, MVT::v16f32, { 1, 1, 1, 1 } }, // vcvtps2ph
2939
2940 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i8, { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
2941 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i8, { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
2942 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i8, { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
2943 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i8, { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
2944 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i16, { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
2945 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i16, { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
2946 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i16, { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
2947 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i16, { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
2948 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i32, { 2, 1, 1, 1 } }, // zmm vpslld+vptestmd
2949 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i32, { 2, 1, 1, 1 } }, // zmm vpslld+vptestmd
2950 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i32, { 2, 1, 1, 1 } }, // zmm vpslld+vptestmd
2951 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i32, { 2, 1, 1, 1 } }, // vpslld+vptestmd
2952 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i64, { 2, 1, 1, 1 } }, // zmm vpsllq+vptestmq
2953 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i64, { 2, 1, 1, 1 } }, // zmm vpsllq+vptestmq
2954 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i64, { 2, 1, 1, 1 } }, // vpsllq+vptestmq
2955 { ISD::TRUNCATE, MVT::v2i8, MVT::v2i32, { 2, 1, 1, 1 } }, // vpmovdb
2956 { ISD::TRUNCATE, MVT::v4i8, MVT::v4i32, { 2, 1, 1, 1 } }, // vpmovdb
2957 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, { 2, 1, 1, 1 } }, // vpmovdb
2958 { ISD::TRUNCATE, MVT::v32i8, MVT::v16i32, { 2, 1, 1, 1 } }, // vpmovdb
2959 { ISD::TRUNCATE, MVT::v64i8, MVT::v16i32, { 2, 1, 1, 1 } }, // vpmovdb
2960 { ISD::TRUNCATE, MVT::v16i16, MVT::v16i32, { 2, 1, 1, 1 } }, // vpmovdw
2961 { ISD::TRUNCATE, MVT::v32i16, MVT::v16i32, { 2, 1, 1, 1 } }, // vpmovdw
2962 { ISD::TRUNCATE, MVT::v2i8, MVT::v2i64, { 2, 1, 1, 1 } }, // vpmovqb
2963 { ISD::TRUNCATE, MVT::v2i16, MVT::v2i64, { 1, 1, 1, 1 } }, // vpshufb
2964 { ISD::TRUNCATE, MVT::v8i8, MVT::v8i64, { 2, 1, 1, 1 } }, // vpmovqb
2965 { ISD::TRUNCATE, MVT::v16i8, MVT::v8i64, { 2, 1, 1, 1 } }, // vpmovqb
2966 { ISD::TRUNCATE, MVT::v32i8, MVT::v8i64, { 2, 1, 1, 1 } }, // vpmovqb
2967 { ISD::TRUNCATE, MVT::v64i8, MVT::v8i64, { 2, 1, 1, 1 } }, // vpmovqb
2968 { ISD::TRUNCATE, MVT::v8i16, MVT::v8i64, { 2, 1, 1, 1 } }, // vpmovqw
2969 { ISD::TRUNCATE, MVT::v16i16, MVT::v8i64, { 2, 1, 1, 1 } }, // vpmovqw
2970 { ISD::TRUNCATE, MVT::v32i16, MVT::v8i64, { 2, 1, 1, 1 } }, // vpmovqw
2971 { ISD::TRUNCATE, MVT::v8i32, MVT::v8i64, { 1, 1, 1, 1 } }, // vpmovqd
2972 { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, { 1, 1, 1, 1 } }, // zmm vpmovqd
2973 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i64, { 5, 1, 1, 1 } },// 2*vpmovqd+concat+vpmovdb
2974
2975 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, { 3, 1, 1, 1 } }, // extend to v16i32
2976 { ISD::TRUNCATE, MVT::v32i8, MVT::v32i16, { 8, 1, 1, 1 } },
2977 { ISD::TRUNCATE, MVT::v64i8, MVT::v32i16, { 8, 1, 1, 1 } },
2978
2979 // Sign extend is zmm vpternlogd+vptruncdb.
2980 // Zero extend is zmm broadcast load+vptruncdw.
2981 { ISD::SIGN_EXTEND, MVT::v2i8, MVT::v2i1, { 3, 1, 1, 1 } },
2982 { ISD::ZERO_EXTEND, MVT::v2i8, MVT::v2i1, { 4, 1, 1, 1 } },
2983 { ISD::SIGN_EXTEND, MVT::v4i8, MVT::v4i1, { 3, 1, 1, 1 } },
2984 { ISD::ZERO_EXTEND, MVT::v4i8, MVT::v4i1, { 4, 1, 1, 1 } },
2985 { ISD::SIGN_EXTEND, MVT::v8i8, MVT::v8i1, { 3, 1, 1, 1 } },
2986 { ISD::ZERO_EXTEND, MVT::v8i8, MVT::v8i1, { 4, 1, 1, 1 } },
2987 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v16i1, { 3, 1, 1, 1 } },
2988 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v16i1, { 4, 1, 1, 1 } },
2989
2990 // Sign extend is zmm vpternlogd+vptruncdw.
2991 // Zero extend is zmm vpternlogd+vptruncdw+vpsrlw.
2992 { ISD::SIGN_EXTEND, MVT::v2i16, MVT::v2i1, { 3, 1, 1, 1 } },
2993 { ISD::ZERO_EXTEND, MVT::v2i16, MVT::v2i1, { 4, 1, 1, 1 } },
2994 { ISD::SIGN_EXTEND, MVT::v4i16, MVT::v4i1, { 3, 1, 1, 1 } },
2995 { ISD::ZERO_EXTEND, MVT::v4i16, MVT::v4i1, { 4, 1, 1, 1 } },
2996 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i1, { 3, 1, 1, 1 } },
2997 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i1, { 4, 1, 1, 1 } },
2998 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1, { 3, 1, 1, 1 } },
2999 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1, { 4, 1, 1, 1 } },
3000
3001 { ISD::SIGN_EXTEND, MVT::v2i32, MVT::v2i1, { 1, 1, 1, 1 } }, // zmm vpternlogd
3002 { ISD::ZERO_EXTEND, MVT::v2i32, MVT::v2i1, { 2, 1, 1, 1 } }, // zmm vpternlogd+psrld
3003 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i1, { 1, 1, 1, 1 } }, // zmm vpternlogd
3004 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i1, { 2, 1, 1, 1 } }, // zmm vpternlogd+psrld
3005 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1, { 1, 1, 1, 1 } }, // zmm vpternlogd
3006 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1, { 2, 1, 1, 1 } }, // zmm vpternlogd+psrld
3007 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i1, { 1, 1, 1, 1 } }, // zmm vpternlogq
3008 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i1, { 2, 1, 1, 1 } }, // zmm vpternlogq+psrlq
3009 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i1, { 1, 1, 1, 1 } }, // zmm vpternlogq
3010 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i1, { 2, 1, 1, 1 } }, // zmm vpternlogq+psrlq
3011
3012 { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i1, { 1, 1, 1, 1 } }, // vpternlogd
3013 { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i1, { 2, 1, 1, 1 } }, // vpternlogd+psrld
3014 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i1, { 1, 1, 1, 1 } }, // vpternlogq
3015 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i1, { 2, 1, 1, 1 } }, // vpternlogq+psrlq
3016
3017 { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, { 1, 1, 1, 1 } },
3018 { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, { 1, 1, 1, 1 } },
3019 { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i16, { 1, 1, 1, 1 } },
3020 { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i16, { 1, 1, 1, 1 } },
3021 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i8, { 1, 1, 1, 1 } },
3022 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i8, { 1, 1, 1, 1 } },
3023 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i16, { 1, 1, 1, 1 } },
3024 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i16, { 1, 1, 1, 1 } },
3025 { ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i32, { 1, 1, 1, 1 } },
3026 { ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i32, { 1, 1, 1, 1 } },
3027
3028 { ISD::SIGN_EXTEND, MVT::v32i16, MVT::v32i8, { 3, 1, 1, 1 } }, // FIXME: May not be right
3029 { ISD::ZERO_EXTEND, MVT::v32i16, MVT::v32i8, { 3, 1, 1, 1 } }, // FIXME: May not be right
3030
3031 { ISD::SINT_TO_FP, MVT::v8f64, MVT::v8i1, { 4, 1, 1, 1 } },
3032 { ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i1, { 3, 1, 1, 1 } },
3033 { ISD::SINT_TO_FP, MVT::v8f64, MVT::v16i8, { 2, 1, 1, 1 } },
3034 { ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i8, { 1, 1, 1, 1 } },
3035 { ISD::SINT_TO_FP, MVT::v8f64, MVT::v8i16, { 2, 1, 1, 1 } },
3036 { ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i16, { 1, 1, 1, 1 } },
3037 { ISD::SINT_TO_FP, MVT::v8f64, MVT::v8i32, { 1, 1, 1, 1 } },
3038 { ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i32, { 1, 1, 1, 1 } },
3039
3040 { ISD::UINT_TO_FP, MVT::v8f64, MVT::v8i1, { 4, 1, 1, 1 } },
3041 { ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i1, { 3, 1, 1, 1 } },
3042 { ISD::UINT_TO_FP, MVT::v8f64, MVT::v16i8, { 2, 1, 1, 1 } },
3043 { ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i8, { 1, 1, 1, 1 } },
3044 { ISD::UINT_TO_FP, MVT::v8f64, MVT::v8i16, { 2, 1, 1, 1 } },
3045 { ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i16, { 1, 1, 1, 1 } },
3046 { ISD::UINT_TO_FP, MVT::v8f64, MVT::v8i32, { 1, 1, 1, 1 } },
3047 { ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i32, { 1, 1, 1, 1 } },
3048 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i64, {26, 1, 1, 1 } },
3049 { ISD::UINT_TO_FP, MVT::v8f64, MVT::v8i64, { 5, 1, 1, 1 } },
3050
3051 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v16f32, { 2, 1, 1, 1 } },
3052 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v16f64, { 7, 1, 1, 1 } },
3053 { ISD::FP_TO_SINT, MVT::v32i8, MVT::v32f64, {15, 1, 1, 1 } },
3054 { ISD::FP_TO_SINT, MVT::v64i8, MVT::v64f32, {11, 1, 1, 1 } },
3055 { ISD::FP_TO_SINT, MVT::v64i8, MVT::v64f64, {31, 1, 1, 1 } },
3056 { ISD::FP_TO_SINT, MVT::v8i16, MVT::v8f64, { 3, 1, 1, 1 } },
3057 { ISD::FP_TO_SINT, MVT::v16i16, MVT::v16f64, { 7, 1, 1, 1 } },
3058 { ISD::FP_TO_SINT, MVT::v32i16, MVT::v32f32, { 5, 1, 1, 1 } },
3059 { ISD::FP_TO_SINT, MVT::v32i16, MVT::v32f64, {15, 1, 1, 1 } },
3060 { ISD::FP_TO_SINT, MVT::v8i32, MVT::v8f64, { 1, 1, 1, 1 } },
3061 { ISD::FP_TO_SINT, MVT::v16i32, MVT::v16f64, { 3, 1, 1, 1 } },
3062
3063 { ISD::FP_TO_UINT, MVT::v8i32, MVT::v8f64, { 1, 1, 1, 1 } },
3064 { ISD::FP_TO_UINT, MVT::v8i16, MVT::v8f64, { 3, 1, 1, 1 } },
3065 { ISD::FP_TO_UINT, MVT::v8i8, MVT::v8f64, { 3, 1, 1, 1 } },
3066 { ISD::FP_TO_UINT, MVT::v16i32, MVT::v16f32, { 1, 1, 1, 1 } },
3067 { ISD::FP_TO_UINT, MVT::v16i16, MVT::v16f32, { 3, 1, 1, 1 } },
3068 { ISD::FP_TO_UINT, MVT::v16i8, MVT::v16f32, { 3, 1, 1, 1 } },
3069 };
3070
3071 static const TypeConversionCostKindTblEntry AVX512BWVLConversionTbl[] {
3072 // Mask sign extend has an instruction.
3073 { ISD::SIGN_EXTEND, MVT::v2i8, MVT::v2i1, { 1, 1, 1, 1 } },
3074 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v2i1, { 1, 1, 1, 1 } },
3075 { ISD::SIGN_EXTEND, MVT::v2i16, MVT::v2i1, { 1, 1, 1, 1 } },
3076 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v2i1, { 1, 1, 1, 1 } },
3077 { ISD::SIGN_EXTEND, MVT::v4i16, MVT::v4i1, { 1, 1, 1, 1 } },
3078 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v4i1, { 1, 1, 1, 1 } },
3079 { ISD::SIGN_EXTEND, MVT::v4i8, MVT::v4i1, { 1, 1, 1, 1 } },
3080 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v4i1, { 1, 1, 1, 1 } },
3081 { ISD::SIGN_EXTEND, MVT::v8i8, MVT::v8i1, { 1, 1, 1, 1 } },
3082 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v8i1, { 1, 1, 1, 1 } },
3083 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i1, { 1, 1, 1, 1 } },
3084 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v16i1, { 1, 1, 1, 1 } },
3085 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1, { 1, 1, 1, 1 } },
3086 { ISD::SIGN_EXTEND, MVT::v32i8, MVT::v32i1, { 1, 1, 1, 1 } },
3087 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v32i1, { 1, 1, 1, 1 } },
3088 { ISD::SIGN_EXTEND, MVT::v32i8, MVT::v64i1, { 1, 1, 1, 1 } },
3089 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v64i1, { 1, 1, 1, 1 } },
3090
3091 // Mask zero extend is a sext + shift.
3092 { ISD::ZERO_EXTEND, MVT::v2i8, MVT::v2i1, { 2, 1, 1, 1 } },
3093 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v2i1, { 2, 1, 1, 1 } },
3094 { ISD::ZERO_EXTEND, MVT::v2i16, MVT::v2i1, { 2, 1, 1, 1 } },
3095 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v2i1, { 2, 1, 1, 1 } },
3096 { ISD::ZERO_EXTEND, MVT::v4i8, MVT::v4i1, { 2, 1, 1, 1 } },
3097 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v4i1, { 2, 1, 1, 1 } },
3098 { ISD::ZERO_EXTEND, MVT::v4i16, MVT::v4i1, { 2, 1, 1, 1 } },
3099 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v4i1, { 2, 1, 1, 1 } },
3100 { ISD::ZERO_EXTEND, MVT::v8i8, MVT::v8i1, { 2, 1, 1, 1 } },
3101 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v8i1, { 2, 1, 1, 1 } },
3102 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i1, { 2, 1, 1, 1 } },
3103 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v16i1, { 2, 1, 1, 1 } },
3104 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1, { 2, 1, 1, 1 } },
3105 { ISD::ZERO_EXTEND, MVT::v32i8, MVT::v32i1, { 2, 1, 1, 1 } },
3106 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v32i1, { 2, 1, 1, 1 } },
3107 { ISD::ZERO_EXTEND, MVT::v32i8, MVT::v64i1, { 2, 1, 1, 1 } },
3108 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v64i1, { 2, 1, 1, 1 } },
3109
3110 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i8, { 2, 1, 1, 1 } },
3111 { ISD::TRUNCATE, MVT::v2i1, MVT::v16i8, { 2, 1, 1, 1 } },
3112 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i16, { 2, 1, 1, 1 } },
3113 { ISD::TRUNCATE, MVT::v2i1, MVT::v8i16, { 2, 1, 1, 1 } },
3114 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i8, { 2, 1, 1, 1 } },
3115 { ISD::TRUNCATE, MVT::v4i1, MVT::v16i8, { 2, 1, 1, 1 } },
3116 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i16, { 2, 1, 1, 1 } },
3117 { ISD::TRUNCATE, MVT::v4i1, MVT::v8i16, { 2, 1, 1, 1 } },
3118 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i8, { 2, 1, 1, 1 } },
3119 { ISD::TRUNCATE, MVT::v8i1, MVT::v16i8, { 2, 1, 1, 1 } },
3120 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i16, { 2, 1, 1, 1 } },
3121 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i8, { 2, 1, 1, 1 } },
3122 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i16, { 2, 1, 1, 1 } },
3123 { ISD::TRUNCATE, MVT::v32i1, MVT::v32i8, { 2, 1, 1, 1 } },
3124 { ISD::TRUNCATE, MVT::v32i1, MVT::v16i16, { 2, 1, 1, 1 } },
3125 { ISD::TRUNCATE, MVT::v64i1, MVT::v32i8, { 2, 1, 1, 1 } },
3126 { ISD::TRUNCATE, MVT::v64i1, MVT::v16i16, { 2, 1, 1, 1 } },
3127
3128 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, { 2, 1, 1, 1 } },
3129 };
3130
3131 static const TypeConversionCostKindTblEntry AVX512DQVLConversionTbl[] = {
3132 // Mask sign extend has an instruction.
3133 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i1, { 1, 1, 1, 1 } },
3134 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v2i1, { 1, 1, 1, 1 } },
3135 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i1, { 1, 1, 1, 1 } },
3136 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v16i1, { 1, 1, 1, 1 } },
3137 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i1, { 1, 1, 1, 1 } },
3138 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v8i1, { 1, 1, 1, 1 } },
3139 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v16i1, { 1, 1, 1, 1 } },
3140 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1, { 1, 1, 1, 1 } },
3141
3142 // Mask zero extend is a sext + shift.
3143 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i1, { 2, 1, 1, 1 } },
3144 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v2i1, { 2, 1, 1, 1 } },
3145 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i1, { 2, 1, 1, 1 } },
3146 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v16i1, { 2, 1, 1, 1 } },
3147 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i1, { 2, 1, 1, 1 } },
3148 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v8i1, { 2, 1, 1, 1 } },
3149 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v16i1, { 2, 1, 1, 1 } },
3150 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1, { 2, 1, 1, 1 } },
3151
3152 { ISD::TRUNCATE, MVT::v16i1, MVT::v4i64, { 2, 1, 1, 1 } },
3153 { ISD::TRUNCATE, MVT::v16i1, MVT::v8i32, { 2, 1, 1, 1 } },
3154 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i64, { 2, 1, 1, 1 } },
3155 { ISD::TRUNCATE, MVT::v2i1, MVT::v4i32, { 2, 1, 1, 1 } },
3156 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i32, { 2, 1, 1, 1 } },
3157 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i64, { 2, 1, 1, 1 } },
3158 { ISD::TRUNCATE, MVT::v8i1, MVT::v4i64, { 2, 1, 1, 1 } },
3159 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i32, { 2, 1, 1, 1 } },
3160
3161 { ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i64, { 1, 1, 1, 1 } },
3162 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, { 1, 1, 1, 1 } },
3163 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i64, { 1, 1, 1, 1 } },
3164 { ISD::SINT_TO_FP, MVT::v4f64, MVT::v4i64, { 1, 1, 1, 1 } },
3165
3166 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i64, { 1, 1, 1, 1 } },
3167 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, { 1, 1, 1, 1 } },
3168 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i64, { 1, 1, 1, 1 } },
3169 { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i64, { 1, 1, 1, 1 } },
3170
3171 { ISD::FP_TO_SINT, MVT::v2i64, MVT::v4f32, { 1, 1, 1, 1 } },
3172 { ISD::FP_TO_SINT, MVT::v4i64, MVT::v4f32, { 1, 1, 1, 1 } },
3173 { ISD::FP_TO_SINT, MVT::v2i64, MVT::v2f64, { 1, 1, 1, 1 } },
3174 { ISD::FP_TO_SINT, MVT::v4i64, MVT::v4f64, { 1, 1, 1, 1 } },
3175
3176 { ISD::FP_TO_UINT, MVT::v2i64, MVT::v4f32, { 1, 1, 1, 1 } },
3177 { ISD::FP_TO_UINT, MVT::v4i64, MVT::v4f32, { 1, 1, 1, 1 } },
3178 { ISD::FP_TO_UINT, MVT::v2i64, MVT::v2f64, { 1, 1, 1, 1 } },
3179 { ISD::FP_TO_UINT, MVT::v4i64, MVT::v4f64, { 1, 1, 1, 1 } },
3180 };
3181
3182 static const TypeConversionCostKindTblEntry AVX512VLConversionTbl[] = {
3183 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i8, { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
3184 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i8, { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
3185 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i8, { 3, 1, 1, 1 } }, // sext+vpslld+vptestmd
3186 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i8, { 8, 1, 1, 1 } }, // split+2*v8i8
3187 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i16, { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
3188 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i16, { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
3189 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i16, { 3, 1, 1, 1 } }, // sext+vpsllq+vptestmq
3190 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i16, { 8, 1, 1, 1 } }, // split+2*v8i16
3191 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i32, { 2, 1, 1, 1 } }, // vpslld+vptestmd
3192 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i32, { 2, 1, 1, 1 } }, // vpslld+vptestmd
3193 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i32, { 2, 1, 1, 1 } }, // vpslld+vptestmd
3194 { ISD::TRUNCATE, MVT::v16i1, MVT::v8i32, { 2, 1, 1, 1 } }, // vpslld+vptestmd
3195 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i64, { 2, 1, 1, 1 } }, // vpsllq+vptestmq
3196 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i64, { 2, 1, 1, 1 } }, // vpsllq+vptestmq
3197 { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, { 1, 1, 1, 1 } }, // vpmovqd
3198 { ISD::TRUNCATE, MVT::v4i8, MVT::v4i64, { 2, 1, 1, 1 } }, // vpmovqb
3199 { ISD::TRUNCATE, MVT::v4i16, MVT::v4i64, { 2, 1, 1, 1 } }, // vpmovqw
3200 { ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, { 2, 1, 1, 1 } }, // vpmovwb
3201
3202 // sign extend is vpcmpeq+maskedmove+vpmovdw+vpacksswb
3203 // zero extend is vpcmpeq+maskedmove+vpmovdw+vpsrlw+vpackuswb
3204 { ISD::SIGN_EXTEND, MVT::v2i8, MVT::v2i1, { 5, 1, 1, 1 } },
3205 { ISD::ZERO_EXTEND, MVT::v2i8, MVT::v2i1, { 6, 1, 1, 1 } },
3206 { ISD::SIGN_EXTEND, MVT::v4i8, MVT::v4i1, { 5, 1, 1, 1 } },
3207 { ISD::ZERO_EXTEND, MVT::v4i8, MVT::v4i1, { 6, 1, 1, 1 } },
3208 { ISD::SIGN_EXTEND, MVT::v8i8, MVT::v8i1, { 5, 1, 1, 1 } },
3209 { ISD::ZERO_EXTEND, MVT::v8i8, MVT::v8i1, { 6, 1, 1, 1 } },
3210 { ISD::SIGN_EXTEND, MVT::v16i8, MVT::v16i1, {10, 1, 1, 1 } },
3211 { ISD::ZERO_EXTEND, MVT::v16i8, MVT::v16i1, {12, 1, 1, 1 } },
3212
3213 // sign extend is vpcmpeq+maskedmove+vpmovdw
3214 // zero extend is vpcmpeq+maskedmove+vpmovdw+vpsrlw
3215 { ISD::SIGN_EXTEND, MVT::v2i16, MVT::v2i1, { 4, 1, 1, 1 } },
3216 { ISD::ZERO_EXTEND, MVT::v2i16, MVT::v2i1, { 5, 1, 1, 1 } },
3217 { ISD::SIGN_EXTEND, MVT::v4i16, MVT::v4i1, { 4, 1, 1, 1 } },
3218 { ISD::ZERO_EXTEND, MVT::v4i16, MVT::v4i1, { 5, 1, 1, 1 } },
3219 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v8i1, { 4, 1, 1, 1 } },
3220 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v8i1, { 5, 1, 1, 1 } },
3221 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1, {10, 1, 1, 1 } },
3222 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1, {12, 1, 1, 1 } },
3223
3224 { ISD::SIGN_EXTEND, MVT::v2i32, MVT::v2i1, { 1, 1, 1, 1 } }, // vpternlogd
3225 { ISD::ZERO_EXTEND, MVT::v2i32, MVT::v2i1, { 2, 1, 1, 1 } }, // vpternlogd+psrld
3226 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i1, { 1, 1, 1, 1 } }, // vpternlogd
3227 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i1, { 2, 1, 1, 1 } }, // vpternlogd+psrld
3228 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1, { 1, 1, 1, 1 } }, // vpternlogd
3229 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1, { 2, 1, 1, 1 } }, // vpternlogd+psrld
3230 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v16i1, { 1, 1, 1, 1 } }, // vpternlogd
3231 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v16i1, { 2, 1, 1, 1 } }, // vpternlogd+psrld
3232
3233 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i1, { 1, 1, 1, 1 } }, // vpternlogq
3234 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i1, { 2, 1, 1, 1 } }, // vpternlogq+psrlq
3235 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i1, { 1, 1, 1, 1 } }, // vpternlogq
3236 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i1, { 2, 1, 1, 1 } }, // vpternlogq+psrlq
3237
3238 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v16i8, { 1, 1, 1, 1 } },
3239 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v16i8, { 1, 1, 1, 1 } },
3240 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v16i8, { 1, 1, 1, 1 } },
3241 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v16i8, { 1, 1, 1, 1 } },
3242 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, { 1, 1, 1, 1 } },
3243 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, { 1, 1, 1, 1 } },
3244 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v8i16, { 1, 1, 1, 1 } },
3245 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v8i16, { 1, 1, 1, 1 } },
3246 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, { 1, 1, 1, 1 } },
3247 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, { 1, 1, 1, 1 } },
3248 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, { 1, 1, 1, 1 } },
3249 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, { 1, 1, 1, 1 } },
3250
3251 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v16i8, { 1, 1, 1, 1 } },
3252 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v16i8, { 1, 1, 1, 1 } },
3253 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v8i16, { 1, 1, 1, 1 } },
3254 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, { 1, 1, 1, 1 } },
3255
3256 { ISD::UINT_TO_FP, MVT::f32, MVT::i64, { 1, 1, 1, 1 } },
3257 { ISD::UINT_TO_FP, MVT::f64, MVT::i64, { 1, 1, 1, 1 } },
3258 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v16i8, { 1, 1, 1, 1 } },
3259 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v16i8, { 1, 1, 1, 1 } },
3260 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v8i16, { 1, 1, 1, 1 } },
3261 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, { 1, 1, 1, 1 } },
3262 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, { 1, 1, 1, 1 } },
3263 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, { 1, 1, 1, 1 } },
3264 { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i32, { 1, 1, 1, 1 } },
3265 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i32, { 1, 1, 1, 1 } },
3266 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i64, { 5, 1, 1, 1 } },
3267 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, { 5, 1, 1, 1 } },
3268 { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i64, { 5, 1, 1, 1 } },
3269
3270 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v8f32, { 2, 1, 1, 1 } },
3271 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v16f32, { 2, 1, 1, 1 } },
3272 { ISD::FP_TO_SINT, MVT::v32i8, MVT::v32f32, { 5, 1, 1, 1 } },
3273
3274 { ISD::FP_TO_UINT, MVT::i64, MVT::f32, { 1, 1, 1, 1 } },
3275 { ISD::FP_TO_UINT, MVT::i64, MVT::f64, { 1, 1, 1, 1 } },
3276 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, { 1, 1, 1, 1 } },
3277 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v2f64, { 1, 1, 1, 1 } },
3278 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f64, { 1, 1, 1, 1 } },
3279 { ISD::FP_TO_UINT, MVT::v8i32, MVT::v8f32, { 1, 1, 1, 1 } },
3280 { ISD::FP_TO_UINT, MVT::v8i32, MVT::v8f64, { 1, 1, 1, 1 } },
3281 };
3282
3283 static const TypeConversionCostKindTblEntry AVX2ConversionTbl[] = {
3284 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i1, { 3, 1, 1, 1 } },
3285 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i1, { 3, 1, 1, 1 } },
3286 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1, { 3, 1, 1, 1 } },
3287 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1, { 3, 1, 1, 1 } },
3288 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1, { 1, 1, 1, 1 } },
3289 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1, { 1, 1, 1, 1 } },
3290
3291 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v16i8, { 2, 1, 1, 1 } },
3292 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v16i8, { 2, 1, 1, 1 } },
3293 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v16i8, { 2, 1, 1, 1 } },
3294 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v16i8, { 2, 1, 1, 1 } },
3295 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, { 2, 1, 1, 1 } },
3296 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, { 2, 1, 1, 1 } },
3297 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v8i16, { 2, 1, 1, 1 } },
3298 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v8i16, { 2, 1, 1, 1 } },
3299 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, { 2, 1, 1, 1 } },
3300 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, { 2, 1, 1, 1 } },
3301 { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i16, { 3, 1, 1, 1 } },
3302 { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i16, { 3, 1, 1, 1 } },
3303 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, { 2, 1, 1, 1 } },
3304 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, { 2, 1, 1, 1 } },
3305
3306 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i32, { 2, 1, 1, 1 } },
3307
3308 { ISD::TRUNCATE, MVT::v16i16, MVT::v16i32, { 4, 1, 1, 1 } },
3309 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, { 4, 1, 1, 1 } },
3310 { ISD::TRUNCATE, MVT::v16i8, MVT::v8i16, { 1, 1, 1, 1 } },
3311 { ISD::TRUNCATE, MVT::v16i8, MVT::v4i32, { 1, 1, 1, 1 } },
3312 { ISD::TRUNCATE, MVT::v16i8, MVT::v2i64, { 1, 1, 1, 1 } },
3313 { ISD::TRUNCATE, MVT::v16i8, MVT::v8i32, { 4, 1, 1, 1 } },
3314 { ISD::TRUNCATE, MVT::v16i8, MVT::v4i64, { 4, 1, 1, 1 } },
3315 { ISD::TRUNCATE, MVT::v8i16, MVT::v4i32, { 1, 1, 1, 1 } },
3316 { ISD::TRUNCATE, MVT::v8i16, MVT::v2i64, { 1, 1, 1, 1 } },
3317 { ISD::TRUNCATE, MVT::v8i16, MVT::v4i64, { 5, 1, 1, 1 } },
3318 { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, { 1, 1, 1, 1 } },
3319 { ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, { 2, 1, 1, 1 } },
3320
3321 { ISD::FP_EXTEND, MVT::v8f64, MVT::v8f32, { 3, 1, 1, 1 } },
3322 { ISD::FP_ROUND, MVT::v8f32, MVT::v8f64, { 3, 1, 1, 1 } },
3323
3324 { ISD::FP_TO_SINT, MVT::v16i16, MVT::v8f32, { 1, 1, 1, 1 } },
3325 { ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f64, { 1, 1, 1, 1 } },
3326 { ISD::FP_TO_SINT, MVT::v8i32, MVT::v8f32, { 1, 1, 1, 1 } },
3327 { ISD::FP_TO_SINT, MVT::v8i32, MVT::v8f64, { 3, 1, 1, 1 } },
3328
3329 { ISD::FP_TO_UINT, MVT::i64, MVT::f32, { 3, 1, 1, 1 } },
3330 { ISD::FP_TO_UINT, MVT::i64, MVT::f64, { 3, 1, 1, 1 } },
3331 { ISD::FP_TO_UINT, MVT::v16i16, MVT::v8f32, { 1, 1, 1, 1 } },
3332 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, { 3, 1, 1, 1 } },
3333 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v2f64, { 4, 1, 1, 1 } },
3334 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f64, { 4, 1, 1, 1 } },
3335 { ISD::FP_TO_UINT, MVT::v8i32, MVT::v8f32, { 3, 1, 1, 1 } },
3336 { ISD::FP_TO_UINT, MVT::v8i32, MVT::v4f64, { 4, 1, 1, 1 } },
3337
3338 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v16i8, { 2, 1, 1, 1 } },
3339 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v16i8, { 2, 1, 1, 1 } },
3340 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v8i16, { 2, 1, 1, 1 } },
3341 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, { 2, 1, 1, 1 } },
3342 { ISD::SINT_TO_FP, MVT::v4f64, MVT::v4i32, { 1, 1, 1, 1 } },
3343 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i32, { 1, 1, 1, 1 } },
3344 { ISD::SINT_TO_FP, MVT::v8f64, MVT::v8i32, { 3, 1, 1, 1 } },
3345
3346 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v16i8, { 2, 1, 1, 1 } },
3347 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v16i8, { 2, 1, 1, 1 } },
3348 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v8i16, { 2, 1, 1, 1 } },
3349 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, { 2, 1, 1, 1 } },
3350 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, { 2, 1, 1, 1 } },
3351 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, { 1, 1, 1, 1 } },
3352 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, { 2, 1, 1, 1 } },
3353 { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i32, { 2, 1, 1, 1 } },
3354 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i32, { 2, 1, 1, 1 } },
3355 { ISD::UINT_TO_FP, MVT::v8f64, MVT::v8i32, { 4, 1, 1, 1 } },
3356 };
3357
3358 static const TypeConversionCostKindTblEntry AVXConversionTbl[] = {
3359 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i1, { 4, 1, 1, 1 } },
3360 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i1, { 4, 1, 1, 1 } },
3361 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1, { 4, 1, 1, 1 } },
3362 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1, { 4, 1, 1, 1 } },
3363 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i1, { 4, 1, 1, 1 } },
3364 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i1, { 4, 1, 1, 1 } },
3365
3366 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v16i8, { 3, 1, 1, 1 } },
3367 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v16i8, { 3, 1, 1, 1 } },
3368 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v16i8, { 3, 1, 1, 1 } },
3369 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v16i8, { 3, 1, 1, 1 } },
3370 { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, { 3, 1, 1, 1 } },
3371 { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, { 3, 1, 1, 1 } },
3372 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v8i16, { 3, 1, 1, 1 } },
3373 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v8i16, { 3, 1, 1, 1 } },
3374 { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, { 3, 1, 1, 1 } },
3375 { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, { 3, 1, 1, 1 } },
3376 { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, { 3, 1, 1, 1 } },
3377 { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, { 3, 1, 1, 1 } },
3378
3379 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i64, { 4, 1, 1, 1 } },
3380 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i32, { 5, 1, 1, 1 } },
3381 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i16, { 4, 1, 1, 1 } },
3382 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i64, { 9, 1, 1, 1 } },
3383 { ISD::TRUNCATE, MVT::v16i1, MVT::v16i64, {11, 1, 1, 1 } },
3384
3385 { ISD::TRUNCATE, MVT::v16i16, MVT::v16i32, { 6, 1, 1, 1 } },
3386 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, { 6, 1, 1, 1 } },
3387 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, { 2, 1, 1, 1 } }, // and+extract+packuswb
3388 { ISD::TRUNCATE, MVT::v16i8, MVT::v8i32, { 5, 1, 1, 1 } },
3389 { ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, { 5, 1, 1, 1 } },
3390 { ISD::TRUNCATE, MVT::v16i8, MVT::v4i64, { 5, 1, 1, 1 } },
3391 { ISD::TRUNCATE, MVT::v8i16, MVT::v4i64, { 3, 1, 1, 1 } }, // and+extract+2*packusdw
3392 { ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, { 2, 1, 1, 1 } },
3393
3394 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i1, { 3, 1, 1, 1 } },
3395 { ISD::SINT_TO_FP, MVT::v4f64, MVT::v4i1, { 3, 1, 1, 1 } },
3396 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i1, { 8, 1, 1, 1 } },
3397 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v16i8, { 4, 1, 1, 1 } },
3398 { ISD::SINT_TO_FP, MVT::v4f64, MVT::v16i8, { 2, 1, 1, 1 } },
3399 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, { 4, 1, 1, 1 } },
3400 { ISD::SINT_TO_FP, MVT::v4f64, MVT::v8i16, { 2, 1, 1, 1 } },
3401 { ISD::SINT_TO_FP, MVT::v4f64, MVT::v4i32, { 2, 1, 1, 1 } },
3402 { ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i32, { 2, 1, 1, 1 } },
3403 { ISD::SINT_TO_FP, MVT::v8f64, MVT::v8i32, { 4, 1, 1, 1 } },
3404 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v2i64, { 5, 1, 1, 1 } },
3405 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i64, { 8, 1, 1, 1 } },
3406
3407 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i1, { 7, 1, 1, 1 } },
3408 { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i1, { 7, 1, 1, 1 } },
3409 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i1, { 6, 1, 1, 1 } },
3410 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v16i8, { 4, 1, 1, 1 } },
3411 { ISD::UINT_TO_FP, MVT::v4f64, MVT::v16i8, { 2, 1, 1, 1 } },
3412 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, { 4, 1, 1, 1 } },
3413 { ISD::UINT_TO_FP, MVT::v4f64, MVT::v8i16, { 2, 1, 1, 1 } },
3414 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, { 4, 1, 1, 1 } },
3415 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, { 4, 1, 1, 1 } },
3416 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, { 5, 1, 1, 1 } },
3417 { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i32, { 6, 1, 1, 1 } },
3418 { ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i32, { 8, 1, 1, 1 } },
3419 { ISD::UINT_TO_FP, MVT::v8f64, MVT::v8i32, {10, 1, 1, 1 } },
3420 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i64, {11, 1, 1, 1 } },
3421 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i64, {18, 1, 1, 1 } },
3422 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, { 5, 1, 1, 1 } },
3423 { ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i64, {10, 1, 1, 1 } },
3424
3425 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v8f32, { 2, 1, 1, 1 } },
3426 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v4f64, { 2, 1, 1, 1 } },
3427 { ISD::FP_TO_SINT, MVT::v32i8, MVT::v8f32, { 2, 1, 1, 1 } },
3428 { ISD::FP_TO_SINT, MVT::v32i8, MVT::v4f64, { 2, 1, 1, 1 } },
3429 { ISD::FP_TO_SINT, MVT::v8i16, MVT::v8f32, { 2, 1, 1, 1 } },
3430 { ISD::FP_TO_SINT, MVT::v8i16, MVT::v4f64, { 2, 1, 1, 1 } },
3431 { ISD::FP_TO_SINT, MVT::v16i16, MVT::v8f32, { 2, 1, 1, 1 } },
3432 { ISD::FP_TO_SINT, MVT::v16i16, MVT::v4f64, { 2, 1, 1, 1 } },
3433 { ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f64, { 2, 1, 1, 1 } },
3434 { ISD::FP_TO_SINT, MVT::v8i32, MVT::v8f32, { 2, 1, 1, 1 } },
3435 { ISD::FP_TO_SINT, MVT::v8i32, MVT::v8f64, { 5, 1, 1, 1 } },
3436
3437 { ISD::FP_TO_UINT, MVT::v16i8, MVT::v8f32, { 2, 1, 1, 1 } },
3438 { ISD::FP_TO_UINT, MVT::v16i8, MVT::v4f64, { 2, 1, 1, 1 } },
3439 { ISD::FP_TO_UINT, MVT::v32i8, MVT::v8f32, { 2, 1, 1, 1 } },
3440 { ISD::FP_TO_UINT, MVT::v32i8, MVT::v4f64, { 2, 1, 1, 1 } },
3441 { ISD::FP_TO_UINT, MVT::v8i16, MVT::v8f32, { 2, 1, 1, 1 } },
3442 { ISD::FP_TO_UINT, MVT::v8i16, MVT::v4f64, { 2, 1, 1, 1 } },
3443 { ISD::FP_TO_UINT, MVT::v16i16, MVT::v8f32, { 2, 1, 1, 1 } },
3444 { ISD::FP_TO_UINT, MVT::v16i16, MVT::v4f64, { 2, 1, 1, 1 } },
3445 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, { 3, 1, 1, 1 } },
3446 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v2f64, { 4, 1, 1, 1 } },
3447 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f64, { 6, 1, 1, 1 } },
3448 { ISD::FP_TO_UINT, MVT::v8i32, MVT::v8f32, { 7, 1, 1, 1 } },
3449 { ISD::FP_TO_UINT, MVT::v8i32, MVT::v4f64, { 7, 1, 1, 1 } },
3450
3451 { ISD::FP_EXTEND, MVT::v4f64, MVT::v4f32, { 1, 1, 1, 1 } },
3452 { ISD::FP_ROUND, MVT::v4f32, MVT::v4f64, { 1, 1, 1, 1 } },
3453 };
3454
3455 static const TypeConversionCostKindTblEntry SSE41ConversionTbl[] = {
3456 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v16i8, { 1, 1, 1, 1 } },
3457 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v16i8, { 1, 1, 1, 1 } },
3458 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v16i8, { 1, 1, 1, 1 } },
3459 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v16i8, { 1, 1, 1, 1 } },
3460 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v16i8, { 1, 1, 1, 1 } },
3461 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v16i8, { 1, 1, 1, 1 } },
3462 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v8i16, { 1, 1, 1, 1 } },
3463 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v8i16, { 1, 1, 1, 1 } },
3464 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v8i16, { 1, 1, 1, 1 } },
3465 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v8i16, { 1, 1, 1, 1 } },
3466 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v4i32, { 1, 1, 1, 1 } },
3467 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v4i32, { 1, 1, 1, 1 } },
3468
3469 // These truncates end up widening elements.
3470 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i8, { 1, 1, 1, 1 } }, // PMOVXZBQ
3471 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i16, { 1, 1, 1, 1 } }, // PMOVXZWQ
3472 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i8, { 1, 1, 1, 1 } }, // PMOVXZBD
3473
3474 { ISD::TRUNCATE, MVT::v16i8, MVT::v4i32, { 2, 1, 1, 1 } },
3475 { ISD::TRUNCATE, MVT::v8i16, MVT::v4i32, { 2, 1, 1, 1 } },
3476 { ISD::TRUNCATE, MVT::v16i8, MVT::v2i64, { 2, 1, 1, 1 } },
3477
3478 { ISD::SINT_TO_FP, MVT::f32, MVT::i32, { 1, 1, 1, 1 } },
3479 { ISD::SINT_TO_FP, MVT::f64, MVT::i32, { 1, 1, 1, 1 } },
3480 { ISD::SINT_TO_FP, MVT::f32, MVT::i64, { 1, 1, 1, 1 } },
3481 { ISD::SINT_TO_FP, MVT::f64, MVT::i64, { 1, 1, 1, 1 } },
3482 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v16i8, { 1, 1, 1, 1 } },
3483 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v16i8, { 1, 1, 1, 1 } },
3484 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v8i16, { 1, 1, 1, 1 } },
3485 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v8i16, { 1, 1, 1, 1 } },
3486 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, { 1, 1, 1, 1 } },
3487 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v4i32, { 1, 1, 1, 1 } },
3488 { ISD::SINT_TO_FP, MVT::v4f64, MVT::v4i32, { 2, 1, 1, 1 } },
3489
3490 { ISD::UINT_TO_FP, MVT::f32, MVT::i32, { 1, 1, 1, 1 } },
3491 { ISD::UINT_TO_FP, MVT::f64, MVT::i32, { 1, 1, 1, 1 } },
3492 { ISD::UINT_TO_FP, MVT::f32, MVT::i64, { 4, 1, 1, 1 } },
3493 { ISD::UINT_TO_FP, MVT::f64, MVT::i64, { 4, 1, 1, 1 } },
3494 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v16i8, { 1, 1, 1, 1 } },
3495 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v16i8, { 1, 1, 1, 1 } },
3496 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v8i16, { 1, 1, 1, 1 } },
3497 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v8i16, { 1, 1, 1, 1 } },
3498 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, { 3, 1, 1, 1 } },
3499 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, { 3, 1, 1, 1 } },
3500 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v4i32, { 2, 1, 1, 1 } },
3501 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v2i64, {12, 1, 1, 1 } },
3502 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i64, {22, 1, 1, 1 } },
3503 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, { 4, 1, 1, 1 } },
3504
3505 { ISD::FP_TO_SINT, MVT::i32, MVT::f32, { 1, 1, 1, 1 } },
3506 { ISD::FP_TO_SINT, MVT::i64, MVT::f32, { 1, 1, 1, 1 } },
3507 { ISD::FP_TO_SINT, MVT::i32, MVT::f64, { 1, 1, 1, 1 } },
3508 { ISD::FP_TO_SINT, MVT::i64, MVT::f64, { 1, 1, 1, 1 } },
3509 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v4f32, { 2, 1, 1, 1 } },
3510 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v2f64, { 2, 1, 1, 1 } },
3511 { ISD::FP_TO_SINT, MVT::v8i16, MVT::v4f32, { 1, 1, 1, 1 } },
3512 { ISD::FP_TO_SINT, MVT::v8i16, MVT::v2f64, { 1, 1, 1, 1 } },
3513 { ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, { 1, 1, 1, 1 } },
3514 { ISD::FP_TO_SINT, MVT::v4i32, MVT::v2f64, { 1, 1, 1, 1 } },
3515
3516 { ISD::FP_TO_UINT, MVT::i32, MVT::f32, { 1, 1, 1, 1 } },
3517 { ISD::FP_TO_UINT, MVT::i64, MVT::f32, { 4, 1, 1, 1 } },
3518 { ISD::FP_TO_UINT, MVT::i32, MVT::f64, { 1, 1, 1, 1 } },
3519 { ISD::FP_TO_UINT, MVT::i64, MVT::f64, { 4, 1, 1, 1 } },
3520 { ISD::FP_TO_UINT, MVT::v16i8, MVT::v4f32, { 2, 1, 1, 1 } },
3521 { ISD::FP_TO_UINT, MVT::v16i8, MVT::v2f64, { 2, 1, 1, 1 } },
3522 { ISD::FP_TO_UINT, MVT::v8i16, MVT::v4f32, { 1, 1, 1, 1 } },
3523 { ISD::FP_TO_UINT, MVT::v8i16, MVT::v2f64, { 1, 1, 1, 1 } },
3524 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, { 4, 1, 1, 1 } },
3525 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v2f64, { 4, 1, 1, 1 } },
3526 };
3527
3528 static const TypeConversionCostKindTblEntry SSE2ConversionTbl[] = {
3529 // These are somewhat magic numbers justified by comparing the
3530 // output of llvm-mca for our various supported scheduler models
3531 // and basing it off the worst case scenario.
3532 { ISD::SINT_TO_FP, MVT::f32, MVT::i32, { 3, 1, 1, 1 } },
3533 { ISD::SINT_TO_FP, MVT::f64, MVT::i32, { 3, 1, 1, 1 } },
3534 { ISD::SINT_TO_FP, MVT::f32, MVT::i64, { 3, 1, 1, 1 } },
3535 { ISD::SINT_TO_FP, MVT::f64, MVT::i64, { 3, 1, 1, 1 } },
3536 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v16i8, { 3, 1, 1, 1 } },
3537 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v16i8, { 4, 1, 1, 1 } },
3538 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v8i16, { 3, 1, 1, 1 } },
3539 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v8i16, { 4, 1, 1, 1 } },
3540 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, { 3, 1, 1, 1 } },
3541 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v4i32, { 4, 1, 1, 1 } },
3542 { ISD::SINT_TO_FP, MVT::v4f32, MVT::v2i64, { 8, 1, 1, 1 } },
3543 { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, { 8, 1, 1, 1 } },
3544
3545 { ISD::UINT_TO_FP, MVT::f32, MVT::i32, { 3, 1, 1, 1 } },
3546 { ISD::UINT_TO_FP, MVT::f64, MVT::i32, { 3, 1, 1, 1 } },
3547 { ISD::UINT_TO_FP, MVT::f32, MVT::i64, { 8, 1, 1, 1 } },
3548 { ISD::UINT_TO_FP, MVT::f64, MVT::i64, { 9, 1, 1, 1 } },
3549 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v16i8, { 4, 1, 1, 1 } },
3550 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v16i8, { 4, 1, 1, 1 } },
3551 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v8i16, { 4, 1, 1, 1 } },
3552 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v8i16, { 4, 1, 1, 1 } },
3553 { ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, { 7, 1, 1, 1 } },
3554 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v4i32, { 7, 1, 1, 1 } },
3555 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, { 5, 1, 1, 1 } },
3556 { ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, {15, 1, 1, 1 } },
3557 { ISD::UINT_TO_FP, MVT::v4f32, MVT::v2i64, {18, 1, 1, 1 } },
3558
3559 { ISD::FP_TO_SINT, MVT::i32, MVT::f32, { 4, 1, 1, 1 } },
3560 { ISD::FP_TO_SINT, MVT::i64, MVT::f32, { 4, 1, 1, 1 } },
3561 { ISD::FP_TO_SINT, MVT::i32, MVT::f64, { 4, 1, 1, 1 } },
3562 { ISD::FP_TO_SINT, MVT::i64, MVT::f64, { 4, 1, 1, 1 } },
3563 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v4f32, { 6, 1, 1, 1 } },
3564 { ISD::FP_TO_SINT, MVT::v16i8, MVT::v2f64, { 6, 1, 1, 1 } },
3565 { ISD::FP_TO_SINT, MVT::v8i16, MVT::v4f32, { 5, 1, 1, 1 } },
3566 { ISD::FP_TO_SINT, MVT::v8i16, MVT::v2f64, { 5, 1, 1, 1 } },
3567 { ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, { 4, 1, 1, 1 } },
3568 { ISD::FP_TO_SINT, MVT::v4i32, MVT::v2f64, { 4, 1, 1, 1 } },
3569
3570 { ISD::FP_TO_UINT, MVT::i32, MVT::f32, { 4, 1, 1, 1 } },
3571 { ISD::FP_TO_UINT, MVT::i64, MVT::f32, { 4, 1, 1, 1 } },
3572 { ISD::FP_TO_UINT, MVT::i32, MVT::f64, { 4, 1, 1, 1 } },
3573 { ISD::FP_TO_UINT, MVT::i64, MVT::f64, {15, 1, 1, 1 } },
3574 { ISD::FP_TO_UINT, MVT::v16i8, MVT::v4f32, { 6, 1, 1, 1 } },
3575 { ISD::FP_TO_UINT, MVT::v16i8, MVT::v2f64, { 6, 1, 1, 1 } },
3576 { ISD::FP_TO_UINT, MVT::v8i16, MVT::v4f32, { 5, 1, 1, 1 } },
3577 { ISD::FP_TO_UINT, MVT::v8i16, MVT::v2f64, { 5, 1, 1, 1 } },
3578 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, { 8, 1, 1, 1 } },
3579 { ISD::FP_TO_UINT, MVT::v4i32, MVT::v2f64, { 8, 1, 1, 1 } },
3580
3581 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v16i8, { 4, 1, 1, 1 } },
3582 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v16i8, { 4, 1, 1, 1 } },
3583 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v16i8, { 2, 1, 1, 1 } },
3584 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v16i8, { 3, 1, 1, 1 } },
3585 { ISD::ZERO_EXTEND, MVT::v8i16, MVT::v16i8, { 1, 1, 1, 1 } },
3586 { ISD::SIGN_EXTEND, MVT::v8i16, MVT::v16i8, { 2, 1, 1, 1 } },
3587 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v8i16, { 2, 1, 1, 1 } },
3588 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v8i16, { 3, 1, 1, 1 } },
3589 { ISD::ZERO_EXTEND, MVT::v4i32, MVT::v8i16, { 1, 1, 1, 1 } },
3590 { ISD::SIGN_EXTEND, MVT::v4i32, MVT::v8i16, { 2, 1, 1, 1 } },
3591 { ISD::ZERO_EXTEND, MVT::v2i64, MVT::v4i32, { 1, 1, 1, 1 } },
3592 { ISD::SIGN_EXTEND, MVT::v2i64, MVT::v4i32, { 2, 1, 1, 1 } },
3593
3594 // These truncates are really widening elements.
3595 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i32, { 1, 1, 1, 1 } }, // PSHUFD
3596 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i16, { 2, 1, 1, 1 } }, // PUNPCKLWD+DQ
3597 { ISD::TRUNCATE, MVT::v2i1, MVT::v2i8, { 3, 1, 1, 1 } }, // PUNPCKLBW+WD+PSHUFD
3598 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i16, { 1, 1, 1, 1 } }, // PUNPCKLWD
3599 { ISD::TRUNCATE, MVT::v4i1, MVT::v4i8, { 2, 1, 1, 1 } }, // PUNPCKLBW+WD
3600 { ISD::TRUNCATE, MVT::v8i1, MVT::v8i8, { 1, 1, 1, 1 } }, // PUNPCKLBW
3601
3602 { ISD::TRUNCATE, MVT::v16i8, MVT::v8i16, { 2, 1, 1, 1 } }, // PAND+PACKUSWB
3603 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i16, { 3, 1, 1, 1 } },
3604 { ISD::TRUNCATE, MVT::v16i8, MVT::v4i32, { 3, 1, 1, 1 } }, // PAND+2*PACKUSWB
3605 { ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, { 7, 1, 1, 1 } },
3606 { ISD::TRUNCATE, MVT::v2i16, MVT::v2i32, { 1, 1, 1, 1 } },
3607 { ISD::TRUNCATE, MVT::v8i16, MVT::v4i32, { 3, 1, 1, 1 } },
3608 { ISD::TRUNCATE, MVT::v8i16, MVT::v8i32, { 5, 1, 1, 1 } },
3609 { ISD::TRUNCATE, MVT::v16i16, MVT::v16i32, {10, 1, 1, 1 } },
3610 { ISD::TRUNCATE, MVT::v16i8, MVT::v2i64, { 4, 1, 1, 1 } }, // PAND+3*PACKUSWB
3611 { ISD::TRUNCATE, MVT::v8i16, MVT::v2i64, { 2, 1, 1, 1 } }, // PSHUFD+PSHUFLW
3612 { ISD::TRUNCATE, MVT::v4i32, MVT::v2i64, { 1, 1, 1, 1 } }, // PSHUFD
3613 };
3614
3615 static const TypeConversionCostKindTblEntry F16ConversionTbl[] = {
3616 { ISD::FP_ROUND, MVT::f16, MVT::f32, { 1, 1, 1, 1 } },
3617 { ISD::FP_ROUND, MVT::v8f16, MVT::v8f32, { 1, 1, 1, 1 } },
3618 { ISD::FP_ROUND, MVT::v4f16, MVT::v4f32, { 1, 1, 1, 1 } },
3619 { ISD::FP_EXTEND, MVT::f32, MVT::f16, { 1, 1, 1, 1 } },
3620 { ISD::FP_EXTEND, MVT::f64, MVT::f16, { 2, 1, 1, 1 } }, // vcvtph2ps+vcvtps2pd
3621 { ISD::FP_EXTEND, MVT::v8f32, MVT::v8f16, { 1, 1, 1, 1 } },
3622 { ISD::FP_EXTEND, MVT::v4f32, MVT::v4f16, { 1, 1, 1, 1 } },
3623 { ISD::FP_EXTEND, MVT::v4f64, MVT::v4f16, { 2, 1, 1, 1 } }, // vcvtph2ps+vcvtps2pd
3624 };
3625
3626 // Attempt to map directly to (simple) MVT types to let us match custom entries.
3627 EVT SrcTy = TLI->getValueType(DL, Src);
3628 EVT DstTy = TLI->getValueType(DL, Dst);
3629
3630 // If we're sign-extending a vector comparison result back to the comparison
3631 // width, this will be free without AVX512 (or for 8/16-bit types without
3632 // BWI).
3633 if (!ST->hasAVX512() || (!ST->hasBWI() && DstTy.getScalarSizeInBits() < 32)) {
3634 if (I && Opcode == Instruction::CastOps::SExt &&
3635 SrcTy.isFixedLengthVectorOf(MVT::i1)) {
3636 if (auto *CmpI = dyn_cast<CmpInst>(I->getOperand(0))) {
3637 Type *CmpTy = CmpI->getOperand(0)->getType();
3638 if (CmpTy->getScalarSizeInBits() == DstTy.getScalarSizeInBits())
3639 return TTI::TCC_Free;
3640 }
3641 }
3642 }
3643
3644 // The function getSimpleVT only handles simple value types.
3645 if (SrcTy.isSimple() && DstTy.isSimple()) {
3646 MVT SimpleSrcTy = SrcTy.getSimpleVT();
3647 MVT SimpleDstTy = DstTy.getSimpleVT();
3648
3649 if (ST->useAVX512Regs()) {
3650 if (ST->hasBWI())
3651 if (const auto *Entry = ConvertCostTableLookup(
3652 AVX512BWConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
3653 if (auto KindCost = Entry->Cost[CostKind])
3654 return *KindCost;
3655
3656 if (ST->hasDQI())
3657 if (const auto *Entry = ConvertCostTableLookup(
3658 AVX512DQConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
3659 if (auto KindCost = Entry->Cost[CostKind])
3660 return *KindCost;
3661
3662 if (ST->hasAVX512())
3663 if (const auto *Entry = ConvertCostTableLookup(
3664 AVX512FConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
3665 if (auto KindCost = Entry->Cost[CostKind])
3666 return *KindCost;
3667 }
3668
3669 if (ST->hasBWI())
3670 if (const auto *Entry = ConvertCostTableLookup(
3671 AVX512BWVLConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
3672 if (auto KindCost = Entry->Cost[CostKind])
3673 return *KindCost;
3674
3675 if (ST->hasDQI())
3676 if (const auto *Entry = ConvertCostTableLookup(
3677 AVX512DQVLConversionTbl, ISD, SimpleDstTy, SimpleSrcTy))
3678 if (auto KindCost = Entry->Cost[CostKind])
3679 return *KindCost;
3680
3681 if (ST->hasAVX512())
3682 if (const auto *Entry = ConvertCostTableLookup(AVX512VLConversionTbl, ISD,
3683 SimpleDstTy, SimpleSrcTy))
3684 if (auto KindCost = Entry->Cost[CostKind])
3685 return *KindCost;
3686
3687 if (ST->hasAVX2()) {
3688 if (const auto *Entry = ConvertCostTableLookup(AVX2ConversionTbl, ISD,
3689 SimpleDstTy, SimpleSrcTy))
3690 if (auto KindCost = Entry->Cost[CostKind])
3691 return *KindCost;
3692 }
3693
3694 if (ST->hasAVX()) {
3695 if (const auto *Entry = ConvertCostTableLookup(AVXConversionTbl, ISD,
3696 SimpleDstTy, SimpleSrcTy))
3697 if (auto KindCost = Entry->Cost[CostKind])
3698 return *KindCost;
3699 }
3700
3701 if (ST->hasF16C()) {
3702 if (const auto *Entry = ConvertCostTableLookup(F16ConversionTbl, ISD,
3703 SimpleDstTy, SimpleSrcTy))
3704 if (auto KindCost = Entry->Cost[CostKind])
3705 return *KindCost;
3706 }
3707
3708 if (ST->hasSSE41()) {
3709 if (const auto *Entry = ConvertCostTableLookup(SSE41ConversionTbl, ISD,
3710 SimpleDstTy, SimpleSrcTy))
3711 if (auto KindCost = Entry->Cost[CostKind])
3712 return *KindCost;
3713 }
3714
3715 if (ST->hasSSE2()) {
3716 if (const auto *Entry = ConvertCostTableLookup(SSE2ConversionTbl, ISD,
3717 SimpleDstTy, SimpleSrcTy))
3718 if (auto KindCost = Entry->Cost[CostKind])
3719 return *KindCost;
3720 }
3721
3722 if ((ISD == ISD::FP_ROUND && SimpleDstTy == MVT::f16) ||
3723 (ISD == ISD::FP_EXTEND && SimpleSrcTy == MVT::f16)) {
3724 // fp16 conversions not covered by any table entries require a libcall.
3725 // Return a large (arbitrary) number to model this.
3726 return InstructionCost(64);
3727 }
3728 }
3729
3730 // Fall back to legalized types.
3731 std::pair<InstructionCost, MVT> LTSrc = getTypeLegalizationCost(Src);
3732 std::pair<InstructionCost, MVT> LTDest = getTypeLegalizationCost(Dst);
3733
3734 // If we're truncating to the same legalized type - just assume its free.
3735 if (ISD == ISD::TRUNCATE && LTSrc.second == LTDest.second)
3736 return TTI::TCC_Free;
3737
3738 if (ST->useAVX512Regs()) {
3739 if (ST->hasBWI())
3740 if (const auto *Entry = ConvertCostTableLookup(
3741 AVX512BWConversionTbl, ISD, LTDest.second, LTSrc.second))
3742 if (auto KindCost = Entry->Cost[CostKind])
3743 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3744
3745 if (ST->hasDQI())
3746 if (const auto *Entry = ConvertCostTableLookup(
3747 AVX512DQConversionTbl, ISD, LTDest.second, LTSrc.second))
3748 if (auto KindCost = Entry->Cost[CostKind])
3749 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3750
3751 if (ST->hasAVX512())
3752 if (const auto *Entry = ConvertCostTableLookup(
3753 AVX512FConversionTbl, ISD, LTDest.second, LTSrc.second))
3754 if (auto KindCost = Entry->Cost[CostKind])
3755 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3756 }
3757
3758 if (ST->hasBWI())
3759 if (const auto *Entry = ConvertCostTableLookup(AVX512BWVLConversionTbl, ISD,
3760 LTDest.second, LTSrc.second))
3761 if (auto KindCost = Entry->Cost[CostKind])
3762 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3763
3764 if (ST->hasDQI())
3765 if (const auto *Entry = ConvertCostTableLookup(AVX512DQVLConversionTbl, ISD,
3766 LTDest.second, LTSrc.second))
3767 if (auto KindCost = Entry->Cost[CostKind])
3768 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3769
3770 if (ST->hasAVX512())
3771 if (const auto *Entry = ConvertCostTableLookup(AVX512VLConversionTbl, ISD,
3772 LTDest.second, LTSrc.second))
3773 if (auto KindCost = Entry->Cost[CostKind])
3774 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3775
3776 if (ST->hasAVX2())
3777 if (const auto *Entry = ConvertCostTableLookup(AVX2ConversionTbl, ISD,
3778 LTDest.second, LTSrc.second))
3779 if (auto KindCost = Entry->Cost[CostKind])
3780 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3781
3782 if (ST->hasAVX())
3783 if (const auto *Entry = ConvertCostTableLookup(AVXConversionTbl, ISD,
3784 LTDest.second, LTSrc.second))
3785 if (auto KindCost = Entry->Cost[CostKind])
3786 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3787
3788 if (ST->hasF16C()) {
3789 if (const auto *Entry = ConvertCostTableLookup(F16ConversionTbl, ISD,
3790 LTDest.second, LTSrc.second))
3791 if (auto KindCost = Entry->Cost[CostKind])
3792 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3793 }
3794
3795 if (ST->hasSSE41())
3796 if (const auto *Entry = ConvertCostTableLookup(SSE41ConversionTbl, ISD,
3797 LTDest.second, LTSrc.second))
3798 if (auto KindCost = Entry->Cost[CostKind])
3799 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3800
3801 if (ST->hasSSE2())
3802 if (const auto *Entry = ConvertCostTableLookup(SSE2ConversionTbl, ISD,
3803 LTDest.second, LTSrc.second))
3804 if (auto KindCost = Entry->Cost[CostKind])
3805 return std::max(LTSrc.first, LTDest.first) * *KindCost;
3806
3807 // Fallback, for i8/i16 sitofp/uitofp cases we need to extend to i32 for
3808 // sitofp.
3809 if ((ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP) &&
3810 1 < Src->getScalarSizeInBits() && Src->getScalarSizeInBits() < 32) {
3811 Type *ExtSrc = Src->getWithNewBitWidth(32);
3812 unsigned ExtOpc =
3813 (ISD == ISD::SINT_TO_FP) ? Instruction::SExt : Instruction::ZExt;
3814
3815 // For scalar loads the extend would be free.
3816 InstructionCost ExtCost = 0;
3817 if (!(Src->isIntegerTy() && I && isa<LoadInst>(I->getOperand(0))))
3818 ExtCost = getCastInstrCost(ExtOpc, ExtSrc, Src, CCH, CostKind);
3819
3820 return ExtCost + getCastInstrCost(Instruction::SIToFP, Dst, ExtSrc,
3821 TTI::CastContextHint::None, CostKind);
3822 }
3823
3824 // Fallback for fptosi/fptoui i8/i16 cases we need to truncate from fptosi
3825 // i32.
3826 if ((ISD == ISD::FP_TO_SINT || ISD == ISD::FP_TO_UINT) &&
3827 1 < Dst->getScalarSizeInBits() && Dst->getScalarSizeInBits() < 32) {
3828 Type *TruncDst = Dst->getWithNewBitWidth(32);
3829 return getCastInstrCost(Instruction::FPToSI, TruncDst, Src, CCH, CostKind) +
3830 getCastInstrCost(Instruction::Trunc, Dst, TruncDst,
3831 TTI::CastContextHint::None, CostKind);
3832 }
3833
3834 // TODO: Allow non-throughput costs that aren't binary.
3835 auto AdjustCost = [&CostKind](InstructionCost Cost,
3837 if (CostKind != TTI::TCK_RecipThroughput)
3838 return Cost == 0 ? 0 : N;
3839 return Cost * N;
3840 };
3841 return AdjustCost(
3842 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
3843}
3844
3845// Additive cost for a predicated op's "predicate fanout": a select, masked
3846// load/store or gather/scatter keeps a compact <N x i1> mask, but if its data
3847// legalizes into more parts than the mask, codegen builds a sub-mask per extra
3848// part (kshiftr on AVX-512, unpack/extend on SSE/AVX) the per-part tables miss.
3849// FanoutCostPerPart is a tie-break: it beats the vectorizer's widest-VF bias.
3851 Type *DataVTy, Type *MaskVTy) {
3852 constexpr unsigned FanoutCostPerPart = 4;
3853 unsigned DataParts = TTI.getNumberOfParts(DataVTy);
3854 unsigned MaskParts = TTI.getNumberOfParts(MaskVTy);
3855 if (DataParts > MaskParts && MaskParts > 0)
3856 return InstructionCost((DataParts - MaskParts) * FanoutCostPerPart);
3857 return InstructionCost(0);
3858}
3859
3861 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
3863 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
3864 // Early out if this type isn't scalar/vector integer/float.
3865 if (!(ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy()))
3866 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
3867 Op1Info, Op2Info, I);
3868
3869 // Legalize the type.
3870 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
3871
3872 MVT MTy = LT.second;
3873
3874 int ISD = TLI->InstructionOpcodeToISD(Opcode);
3875 assert(ISD && "Invalid opcode");
3876
3877 // Predicate fanout (see getPredicateFanoutCost), AVX-512 only: on SSE/AVX a
3878 // select blend is already priced per part, so charging it there shrinks VF.
3879 InstructionCost MaskExpCost = 0;
3880 if (Opcode == Instruction::Select && ST->hasAVX512() &&
3882 MaskExpCost = getPredicateFanoutCost(*this, ValTy, CondTy);
3883
3884 InstructionCost ExtraCost = 0;
3885 if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) {
3886 // Some vector comparison predicates cost extra instructions.
3887 // TODO: Adjust ExtraCost based on CostKind?
3888 // TODO: Should we invert this and assume worst case cmp costs
3889 // and reduce for particular predicates?
3890 if (MTy.isVector() &&
3891 !((ST->hasXOP() && (!ST->hasAVX2() || MTy.is128BitVector())) ||
3892 (ST->hasAVX512() && 32 <= MTy.getScalarSizeInBits()) ||
3893 ST->hasBWI())) {
3894 // Fallback to I if a specific predicate wasn't specified.
3895 CmpInst::Predicate Pred = VecPred;
3896 if (I && (Pred == CmpInst::BAD_ICMP_PREDICATE ||
3898 Pred = cast<CmpInst>(I)->getPredicate();
3899
3900 bool CmpWithConstant = false;
3901 if (auto *CmpInstr = dyn_cast_or_null<CmpInst>(I))
3902 CmpWithConstant = isa<Constant>(CmpInstr->getOperand(1));
3903
3904 switch (Pred) {
3906 // xor(cmpeq(x,y),-1)
3907 ExtraCost = CmpWithConstant ? 0 : 1;
3908 break;
3911 // xor(cmpgt(x,y),-1)
3912 ExtraCost = CmpWithConstant ? 0 : 1;
3913 break;
3916 // cmpgt(xor(x,signbit),xor(y,signbit))
3917 // xor(cmpeq(pmaxu(x,y),x),-1)
3918 ExtraCost = CmpWithConstant ? 1 : 2;
3919 break;
3922 if ((ST->hasSSE41() && MTy.getScalarSizeInBits() == 32) ||
3923 (ST->hasSSE2() && MTy.getScalarSizeInBits() < 32)) {
3924 // cmpeq(psubus(x,y),0)
3925 // cmpeq(pminu(x,y),x)
3926 ExtraCost = 1;
3927 } else {
3928 // xor(cmpgt(xor(x,signbit),xor(y,signbit)),-1)
3929 ExtraCost = CmpWithConstant ? 2 : 3;
3930 }
3931 break;
3934 // Without AVX we need to expand FCMP_ONE/FCMP_UEQ cases.
3935 // Use FCMP_UEQ expansion - FCMP_ONE should be the same.
3936 if (CondTy && !ST->hasAVX())
3937 return getCmpSelInstrCost(Opcode, ValTy, CondTy,
3939 Op1Info, Op2Info) +
3940 getCmpSelInstrCost(Opcode, ValTy, CondTy,
3942 Op1Info, Op2Info) +
3943 getArithmeticInstrCost(Instruction::Or, CondTy, CostKind);
3944
3945 break;
3948 // Assume worst case scenario and add the maximum extra cost.
3949 ExtraCost = 3;
3950 break;
3951 default:
3952 break;
3953 }
3954 }
3955 }
3956
3957 static const CostKindTblEntry SLMCostTbl[] = {
3958 // slm pcmpeq/pcmpgt throughput is 2
3959 { ISD::SETCC, MVT::v2i64, { 2, 5, 1, 2 } },
3960 // slm pblendvb/blendvpd/blendvps throughput is 4
3961 { ISD::SELECT, MVT::v2f64, { 4, 4, 1, 3 } }, // vblendvpd
3962 { ISD::SELECT, MVT::v4f32, { 4, 4, 1, 3 } }, // vblendvps
3963 { ISD::SELECT, MVT::v2i64, { 4, 4, 1, 3 } }, // pblendvb
3964 { ISD::SELECT, MVT::v8i32, { 4, 4, 1, 3 } }, // pblendvb
3965 { ISD::SELECT, MVT::v8i16, { 4, 4, 1, 3 } }, // pblendvb
3966 { ISD::SELECT, MVT::v16i8, { 4, 4, 1, 3 } }, // pblendvb
3967 };
3968
3969 static const CostKindTblEntry AVX512BWCostTbl[] = {
3970 { ISD::SETCC, MVT::v32i16, { 1, 1, 1, 1 } },
3971 { ISD::SETCC, MVT::v16i16, { 1, 1, 1, 1 } },
3972 { ISD::SETCC, MVT::v64i8, { 1, 1, 1, 1 } },
3973 { ISD::SETCC, MVT::v32i8, { 1, 1, 1, 1 } },
3974
3975 { ISD::SELECT, MVT::v32i16, { 1, 1, 1, 1 } },
3976 { ISD::SELECT, MVT::v64i8, { 1, 1, 1, 1 } },
3977 };
3978
3979 static const CostKindTblEntry AVX512CostTbl[] = {
3980 { ISD::SETCC, MVT::v8f64, { 1, 4, 1, 1 } },
3981 { ISD::SETCC, MVT::v4f64, { 1, 4, 1, 1 } },
3982 { ISD::SETCC, MVT::v16f32, { 1, 4, 1, 1 } },
3983 { ISD::SETCC, MVT::v8f32, { 1, 4, 1, 1 } },
3984
3985 { ISD::SETCC, MVT::v8i64, { 1, 1, 1, 1 } },
3986 { ISD::SETCC, MVT::v4i64, { 1, 1, 1, 1 } },
3987 { ISD::SETCC, MVT::v2i64, { 1, 1, 1, 1 } },
3988 { ISD::SETCC, MVT::v16i32, { 1, 1, 1, 1 } },
3989 { ISD::SETCC, MVT::v8i32, { 1, 1, 1, 1 } },
3990 { ISD::SETCC, MVT::v32i16, { 3, 7, 5, 5 } },
3991 { ISD::SETCC, MVT::v64i8, { 3, 7, 5, 5 } },
3992
3993 { ISD::SELECT, MVT::v8i64, { 1, 1, 1, 1 } },
3994 { ISD::SELECT, MVT::v4i64, { 1, 1, 1, 1 } },
3995 { ISD::SELECT, MVT::v2i64, { 1, 1, 1, 1 } },
3996 { ISD::SELECT, MVT::v16i32, { 1, 1, 1, 1 } },
3997 { ISD::SELECT, MVT::v8i32, { 1, 1, 1, 1 } },
3998 { ISD::SELECT, MVT::v4i32, { 1, 1, 1, 1 } },
3999 { ISD::SELECT, MVT::v8f64, { 1, 1, 1, 1 } },
4000 { ISD::SELECT, MVT::v4f64, { 1, 1, 1, 1 } },
4001 { ISD::SELECT, MVT::v2f64, { 1, 1, 1, 1 } },
4002 { ISD::SELECT, MVT::f64, { 1, 1, 1, 1 } },
4003 { ISD::SELECT, MVT::v16f32, { 1, 1, 1, 1 } },
4004 { ISD::SELECT, MVT::v8f32 , { 1, 1, 1, 1 } },
4005 { ISD::SELECT, MVT::v4f32, { 1, 1, 1, 1 } },
4006 { ISD::SELECT, MVT::f32 , { 1, 1, 1, 1 } },
4007
4008 { ISD::SELECT, MVT::v32i16, { 2, 2, 4, 4 } },
4009 { ISD::SELECT, MVT::v16i16, { 1, 1, 1, 1 } },
4010 { ISD::SELECT, MVT::v8i16, { 1, 1, 1, 1 } },
4011 { ISD::SELECT, MVT::v64i8, { 2, 2, 4, 4 } },
4012 { ISD::SELECT, MVT::v32i8, { 1, 1, 1, 1 } },
4013 { ISD::SELECT, MVT::v16i8, { 1, 1, 1, 1 } },
4014 };
4015
4016 static const CostKindTblEntry AVX2CostTbl[] = {
4017 { ISD::SETCC, MVT::v4f64, { 1, 4, 1, 2 } },
4018 { ISD::SETCC, MVT::v2f64, { 1, 4, 1, 1 } },
4019 { ISD::SETCC, MVT::f64, { 1, 4, 1, 1 } },
4020 { ISD::SETCC, MVT::v8f32, { 1, 4, 1, 2 } },
4021 { ISD::SETCC, MVT::v4f32, { 1, 4, 1, 1 } },
4022 { ISD::SETCC, MVT::f32, { 1, 4, 1, 1 } },
4023
4024 { ISD::SETCC, MVT::v4i64, { 1, 1, 1, 2 } },
4025 { ISD::SETCC, MVT::v8i32, { 1, 1, 1, 2 } },
4026 { ISD::SETCC, MVT::v16i16, { 1, 1, 1, 2 } },
4027 { ISD::SETCC, MVT::v32i8, { 1, 1, 1, 2 } },
4028
4029 { ISD::SELECT, MVT::v4f64, { 2, 2, 1, 2 } }, // vblendvpd
4030 { ISD::SELECT, MVT::v8f32, { 2, 2, 1, 2 } }, // vblendvps
4031 { ISD::SELECT, MVT::v4i64, { 2, 2, 1, 2 } }, // pblendvb
4032 { ISD::SELECT, MVT::v8i32, { 2, 2, 1, 2 } }, // pblendvb
4033 { ISD::SELECT, MVT::v16i16, { 2, 2, 1, 2 } }, // pblendvb
4034 { ISD::SELECT, MVT::v32i8, { 2, 2, 1, 2 } }, // pblendvb
4035 };
4036
4037 static const CostKindTblEntry XOPCostTbl[] = {
4038 { ISD::SETCC, MVT::v4i64, { 4, 2, 5, 6 } },
4039 { ISD::SETCC, MVT::v2i64, { 1, 1, 1, 1 } },
4040 };
4041
4042 static const CostKindTblEntry AVX1CostTbl[] = {
4043 { ISD::SETCC, MVT::v4f64, { 2, 3, 1, 2 } },
4044 { ISD::SETCC, MVT::v2f64, { 1, 3, 1, 1 } },
4045 { ISD::SETCC, MVT::f64, { 1, 3, 1, 1 } },
4046 { ISD::SETCC, MVT::v8f32, { 2, 3, 1, 2 } },
4047 { ISD::SETCC, MVT::v4f32, { 1, 3, 1, 1 } },
4048 { ISD::SETCC, MVT::f32, { 1, 3, 1, 1 } },
4049
4050 // AVX1 does not support 8-wide integer compare.
4051 { ISD::SETCC, MVT::v4i64, { 4, 2, 5, 6 } },
4052 { ISD::SETCC, MVT::v8i32, { 4, 2, 5, 6 } },
4053 { ISD::SETCC, MVT::v16i16, { 4, 2, 5, 6 } },
4054 { ISD::SETCC, MVT::v32i8, { 4, 2, 5, 6 } },
4055
4056 { ISD::SELECT, MVT::v4f64, { 3, 3, 1, 2 } }, // vblendvpd
4057 { ISD::SELECT, MVT::v8f32, { 3, 3, 1, 2 } }, // vblendvps
4058 { ISD::SELECT, MVT::v4i64, { 3, 3, 1, 2 } }, // vblendvpd
4059 { ISD::SELECT, MVT::v8i32, { 3, 3, 1, 2 } }, // vblendvps
4060 { ISD::SELECT, MVT::v16i16, { 3, 3, 3, 3 } }, // vandps + vandnps + vorps
4061 { ISD::SELECT, MVT::v32i8, { 3, 3, 3, 3 } }, // vandps + vandnps + vorps
4062 };
4063
4064 static const CostKindTblEntry SSE42CostTbl[] = {
4065 { ISD::SETCC, MVT::v2i64, { 1, 2, 1, 2 } },
4066 };
4067
4068 static const CostKindTblEntry SSE41CostTbl[] = {
4069 { ISD::SETCC, MVT::v2f64, { 1, 5, 1, 1 } },
4070 { ISD::SETCC, MVT::v4f32, { 1, 5, 1, 1 } },
4071
4072 { ISD::SELECT, MVT::v2f64, { 2, 2, 1, 2 } }, // blendvpd
4073 { ISD::SELECT, MVT::f64, { 2, 2, 1, 2 } }, // blendvpd
4074 { ISD::SELECT, MVT::v4f32, { 2, 2, 1, 2 } }, // blendvps
4075 { ISD::SELECT, MVT::f32 , { 2, 2, 1, 2 } }, // blendvps
4076 { ISD::SELECT, MVT::v2i64, { 2, 2, 1, 2 } }, // pblendvb
4077 { ISD::SELECT, MVT::v4i32, { 2, 2, 1, 2 } }, // pblendvb
4078 { ISD::SELECT, MVT::v8i16, { 2, 2, 1, 2 } }, // pblendvb
4079 { ISD::SELECT, MVT::v16i8, { 2, 2, 1, 2 } }, // pblendvb
4080 };
4081
4082 static const CostKindTblEntry SSE2CostTbl[] = {
4083 { ISD::SETCC, MVT::v2f64, { 2, 5, 1, 1 } },
4084 { ISD::SETCC, MVT::f64, { 1, 5, 1, 1 } },
4085
4086 { ISD::SETCC, MVT::v2i64, { 5, 4, 5, 5 } }, // pcmpeqd/pcmpgtd expansion
4087 { ISD::SETCC, MVT::v4i32, { 1, 1, 1, 1 } },
4088 { ISD::SETCC, MVT::v8i16, { 1, 1, 1, 1 } },
4089 { ISD::SETCC, MVT::v16i8, { 1, 1, 1, 1 } },
4090
4091 { ISD::SELECT, MVT::v2f64, { 2, 2, 3, 3 } }, // andpd + andnpd + orpd
4092 { ISD::SELECT, MVT::f64, { 2, 2, 3, 3 } }, // andpd + andnpd + orpd
4093 { ISD::SELECT, MVT::v2i64, { 2, 2, 3, 3 } }, // pand + pandn + por
4094 { ISD::SELECT, MVT::v4i32, { 2, 2, 3, 3 } }, // pand + pandn + por
4095 { ISD::SELECT, MVT::v8i16, { 2, 2, 3, 3 } }, // pand + pandn + por
4096 { ISD::SELECT, MVT::v16i8, { 2, 2, 3, 3 } }, // pand + pandn + por
4097 };
4098
4099 static const CostKindTblEntry SSE1CostTbl[] = {
4100 { ISD::SETCC, MVT::v4f32, { 2, 5, 1, 1 } },
4101 { ISD::SETCC, MVT::f32, { 1, 5, 1, 1 } },
4102
4103 { ISD::SELECT, MVT::v4f32, { 2, 2, 3, 3 } }, // andps + andnps + orps
4104 { ISD::SELECT, MVT::f32, { 2, 2, 3, 3 } }, // andps + andnps + orps
4105 };
4106
4107 if (ST->useSLMArithCosts())
4108 if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
4109 if (auto KindCost = Entry->Cost[CostKind])
4110 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4111
4112 if (ST->hasBWI())
4113 if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
4114 if (auto KindCost = Entry->Cost[CostKind])
4115 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4116
4117 if (ST->hasAVX512())
4118 if (const auto *Entry = CostTableLookup(AVX512CostTbl, ISD, MTy))
4119 if (auto KindCost = Entry->Cost[CostKind])
4120 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4121
4122 if (ST->hasAVX2())
4123 if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
4124 if (auto KindCost = Entry->Cost[CostKind])
4125 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4126
4127 if (ST->hasXOP())
4128 if (const auto *Entry = CostTableLookup(XOPCostTbl, ISD, MTy))
4129 if (auto KindCost = Entry->Cost[CostKind])
4130 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4131
4132 if (ST->hasAVX())
4133 if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
4134 if (auto KindCost = Entry->Cost[CostKind])
4135 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4136
4137 if (ST->hasSSE42())
4138 if (const auto *Entry = CostTableLookup(SSE42CostTbl, ISD, MTy))
4139 if (auto KindCost = Entry->Cost[CostKind])
4140 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4141
4142 if (ST->hasSSE41())
4143 if (const auto *Entry = CostTableLookup(SSE41CostTbl, ISD, MTy))
4144 if (auto KindCost = Entry->Cost[CostKind])
4145 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4146
4147 if (ST->hasSSE2())
4148 if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
4149 if (auto KindCost = Entry->Cost[CostKind])
4150 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4151
4152 if (ST->hasSSE1())
4153 if (const auto *Entry = CostTableLookup(SSE1CostTbl, ISD, MTy))
4154 if (auto KindCost = Entry->Cost[CostKind])
4155 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4156
4157 // Assume a 3cy latency for fp select ops.
4158 if (CostKind == TTI::TCK_Latency && Opcode == Instruction::Select)
4159 if (ValTy->getScalarType()->isFloatingPointTy())
4160 return 3;
4161
4162 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
4163 Op1Info, Op2Info, I) +
4164 MaskExpCost;
4165}
4166
4168
4169/// Returns true if \p V is the square root, divided into with the reciprocal
4170/// allowed, and the target replaces the pair with the rsqrt* based estimate,
4171/// costed with the division. The estimate is formed for the legalized type
4172/// \p VT.
4174 const X86TargetLowering &TLI, MVT VT,
4175 TTI::TargetCostKind CostKind) {
4176 using namespace PatternMatch;
4177 if (CostKind != TTI::TCK_RecipThroughput && CostKind != TTI::TCK_Latency)
4178 return false;
4180 if (!II || II->getIntrinsicID() != Intrinsic::sqrt || !II->hasOneUse() ||
4181 !match(II->user_back(), m_FDiv(m_Value(), m_Specific(II))) ||
4182 !cast<FPMathOperator>(II->user_back())->hasAllowReciprocal())
4183 return false;
4184 return TLI.hasSqrtEstimate(VT, /*Reciprocal=*/true) &&
4185 TLI.getRecipEstimateSqrtEnabled(VT, *II->getFunction()) !=
4187}
4188
4191 TTI::TargetCostKind CostKind) const {
4192 // Costs should match the codegen from:
4193 // BITREVERSE: llvm\test\CodeGen\X86\vector-bitreverse.ll
4194 // BSWAP: llvm\test\CodeGen\X86\bswap-vector.ll
4195 // CTLZ: llvm\test\CodeGen\X86\vector-lzcnt-*.ll
4196 // CTPOP: llvm\test\CodeGen\X86\vector-popcnt-*.ll
4197 // CTTZ: llvm\test\CodeGen\X86\vector-tzcnt-*.ll
4198
4199 // TODO: Overflow intrinsics (*ADDO, *SUBO, *MULO) with vector types are not
4200 // specialized in these tables yet.
4201 static const CostKindTblEntry AVX512VBMI2CostTbl[] = {
4202 { ISD::FSHL, MVT::v8i64, { 1, 1, 1, 1 } },
4203 { ISD::FSHL, MVT::v4i64, { 1, 1, 1, 1 } },
4204 { ISD::FSHL, MVT::v2i64, { 1, 1, 1, 1 } },
4205 { ISD::FSHL, MVT::v16i32, { 1, 1, 1, 1 } },
4206 { ISD::FSHL, MVT::v8i32, { 1, 1, 1, 1 } },
4207 { ISD::FSHL, MVT::v4i32, { 1, 1, 1, 1 } },
4208 { ISD::FSHL, MVT::v32i16, { 1, 1, 1, 1 } },
4209 { ISD::FSHL, MVT::v16i16, { 1, 1, 1, 1 } },
4210 { ISD::FSHL, MVT::v8i16, { 1, 1, 1, 1 } },
4211 { ISD::ROTL, MVT::v32i16, { 1, 1, 1, 1 } },
4212 { ISD::ROTL, MVT::v16i16, { 1, 1, 1, 1 } },
4213 { ISD::ROTL, MVT::v8i16, { 1, 1, 1, 1 } },
4214 { ISD::ROTR, MVT::v32i16, { 1, 1, 1, 1 } },
4215 { ISD::ROTR, MVT::v16i16, { 1, 1, 1, 1 } },
4216 { ISD::ROTR, MVT::v8i16, { 1, 1, 1, 1 } },
4217 { X86ISD::VROTLI, MVT::v32i16, { 1, 1, 1, 1 } },
4218 { X86ISD::VROTLI, MVT::v16i16, { 1, 1, 1, 1 } },
4219 { X86ISD::VROTLI, MVT::v8i16, { 1, 1, 1, 1 } },
4220 };
4221 static const CostKindTblEntry AVX512BITALGCostTbl[] = {
4222 { ISD::CTPOP, MVT::v32i16, { 1, 1, 1, 1 } },
4223 { ISD::CTPOP, MVT::v64i8, { 1, 1, 1, 1 } },
4224 { ISD::CTPOP, MVT::v16i16, { 1, 1, 1, 1 } },
4225 { ISD::CTPOP, MVT::v32i8, { 1, 1, 1, 1 } },
4226 { ISD::CTPOP, MVT::v8i16, { 1, 1, 1, 1 } },
4227 { ISD::CTPOP, MVT::v16i8, { 1, 1, 1, 1 } },
4228 };
4229 static const CostKindTblEntry AVX512VPOPCNTDQCostTbl[] = {
4230 { ISD::CTPOP, MVT::v8i64, { 1, 1, 1, 1 } },
4231 { ISD::CTPOP, MVT::v16i32, { 1, 1, 1, 1 } },
4232 { ISD::CTPOP, MVT::v4i64, { 1, 1, 1, 1 } },
4233 { ISD::CTPOP, MVT::v8i32, { 1, 1, 1, 1 } },
4234 { ISD::CTPOP, MVT::v2i64, { 1, 1, 1, 1 } },
4235 { ISD::CTPOP, MVT::v4i32, { 1, 1, 1, 1 } },
4236 };
4237 static const CostKindTblEntry AVX512CDCostTbl[] = {
4238 { ISD::CTLZ, MVT::v8i64, { 1, 5, 1, 1 } },
4239 { ISD::CTLZ, MVT::v16i32, { 1, 5, 1, 1 } },
4240 { ISD::CTLZ, MVT::v32i16, { 18, 27, 23, 27 } },
4241 { ISD::CTLZ, MVT::v64i8, { 3, 16, 9, 11 } },
4242 { ISD::CTLZ, MVT::v4i64, { 1, 5, 1, 1 } },
4243 { ISD::CTLZ, MVT::v8i32, { 1, 5, 1, 1 } },
4244 { ISD::CTLZ, MVT::v16i16, { 8, 19, 11, 13 } },
4245 { ISD::CTLZ, MVT::v32i8, { 2, 11, 9, 10 } },
4246 { ISD::CTLZ, MVT::v2i64, { 1, 5, 1, 1 } },
4247 { ISD::CTLZ, MVT::v4i32, { 1, 5, 1, 1 } },
4248 { ISD::CTLZ, MVT::v8i16, { 3, 15, 4, 6 } },
4249 { ISD::CTLZ, MVT::v16i8, { 2, 10, 9, 10 } },
4250
4251 { ISD::CTTZ, MVT::v8i64, { 2, 8, 6, 7 } },
4252 { ISD::CTTZ, MVT::v16i32, { 2, 8, 6, 7 } },
4253 { ISD::CTTZ, MVT::v4i64, { 1, 8, 6, 6 } },
4254 { ISD::CTTZ, MVT::v8i32, { 1, 8, 6, 6 } },
4255 { ISD::CTTZ, MVT::v2i64, { 1, 8, 6, 6 } },
4256 { ISD::CTTZ, MVT::v4i32, { 1, 8, 6, 6 } },
4257 };
4258 static const CostKindTblEntry AVX512BWCostTbl[] = {
4259 { ISD::ABS, MVT::v32i16, { 1, 1, 1, 1 } },
4260 { ISD::ABS, MVT::v64i8, { 1, 1, 1, 1 } },
4261 { ISD::BITREVERSE, MVT::v2i64, { 3, 10, 10, 11 } },
4262 { ISD::BITREVERSE, MVT::v4i64, { 3, 11, 10, 11 } },
4263 { ISD::BITREVERSE, MVT::v8i64, { 3, 12, 10, 14 } },
4264 { ISD::BITREVERSE, MVT::v4i32, { 3, 10, 10, 11 } },
4265 { ISD::BITREVERSE, MVT::v8i32, { 3, 11, 10, 11 } },
4266 { ISD::BITREVERSE, MVT::v16i32, { 3, 12, 10, 14 } },
4267 { ISD::BITREVERSE, MVT::v8i16, { 3, 10, 10, 11 } },
4268 { ISD::BITREVERSE, MVT::v16i16, { 3, 11, 10, 11 } },
4269 { ISD::BITREVERSE, MVT::v32i16, { 3, 12, 10, 14 } },
4270 { ISD::BITREVERSE, MVT::v16i8, { 2, 5, 9, 9 } },
4271 { ISD::BITREVERSE, MVT::v32i8, { 2, 5, 9, 9 } },
4272 { ISD::BITREVERSE, MVT::v64i8, { 2, 5, 9, 12 } },
4273 { ISD::BSWAP, MVT::v2i64, { 1, 1, 1, 2 } },
4274 { ISD::BSWAP, MVT::v4i64, { 1, 1, 1, 2 } },
4275 { ISD::BSWAP, MVT::v8i64, { 1, 1, 1, 2 } },
4276 { ISD::BSWAP, MVT::v4i32, { 1, 1, 1, 2 } },
4277 { ISD::BSWAP, MVT::v8i32, { 1, 1, 1, 2 } },
4278 { ISD::BSWAP, MVT::v16i32, { 1, 1, 1, 2 } },
4279 { ISD::BSWAP, MVT::v8i16, { 1, 1, 1, 2 } },
4280 { ISD::BSWAP, MVT::v16i16, { 1, 1, 1, 2 } },
4281 { ISD::BSWAP, MVT::v32i16, { 1, 1, 1, 2 } },
4282 { ISD::CTLZ, MVT::v8i64, { 8, 22, 23, 23 } },
4283 { ISD::CTLZ, MVT::v16i32, { 8, 23, 25, 25 } },
4284 { ISD::CTLZ, MVT::v32i16, { 4, 15, 15, 16 } },
4285 { ISD::CTLZ, MVT::v64i8, { 3, 12, 10, 9 } },
4286 { ISD::CTPOP, MVT::v2i64, { 3, 7, 10, 10 } },
4287 { ISD::CTPOP, MVT::v4i64, { 3, 7, 10, 10 } },
4288 { ISD::CTPOP, MVT::v8i64, { 3, 8, 10, 12 } },
4289 { ISD::CTPOP, MVT::v4i32, { 7, 11, 14, 14 } },
4290 { ISD::CTPOP, MVT::v8i32, { 7, 11, 14, 14 } },
4291 { ISD::CTPOP, MVT::v16i32, { 7, 12, 14, 16 } },
4292 { ISD::CTPOP, MVT::v8i16, { 2, 7, 11, 11 } },
4293 { ISD::CTPOP, MVT::v16i16, { 2, 7, 11, 11 } },
4294 { ISD::CTPOP, MVT::v32i16, { 3, 7, 11, 13 } },
4295 { ISD::CTPOP, MVT::v16i8, { 2, 4, 8, 8 } },
4296 { ISD::CTPOP, MVT::v32i8, { 2, 4, 8, 8 } },
4297 { ISD::CTPOP, MVT::v64i8, { 2, 5, 8, 10 } },
4298 { ISD::CTTZ, MVT::v8i16, { 3, 9, 14, 14 } },
4299 { ISD::CTTZ, MVT::v16i16, { 3, 9, 14, 14 } },
4300 { ISD::CTTZ, MVT::v32i16, { 3, 10, 14, 16 } },
4301 { ISD::CTTZ, MVT::v16i8, { 2, 6, 11, 11 } },
4302 { ISD::CTTZ, MVT::v32i8, { 2, 6, 11, 11 } },
4303 { ISD::CTTZ, MVT::v64i8, { 3, 7, 11, 13 } },
4304 { ISD::MASK_BEFOREFIRST, MVT::v32i1, { 4, 8, 4, 8 } }, // kmov seq
4305 { ISD::MASK_BEFOREFIRST, MVT::v64i1, { 4, 8, 4, 8 } }, // kmov seq
4306 { ISD::MULHS, MVT::v32i16, { 1, 5, 1, 1 } },
4307 { ISD::MULHU, MVT::v32i16, { 1, 5, 1, 1 } },
4308 { ISD::ROTL, MVT::v32i16, { 2, 8, 6, 8 } },
4309 { ISD::ROTL, MVT::v16i16, { 2, 8, 6, 7 } },
4310 { ISD::ROTL, MVT::v8i16, { 2, 7, 6, 7 } },
4311 { ISD::ROTL, MVT::v64i8, { 5, 6, 11, 12 } },
4312 { ISD::ROTL, MVT::v32i8, { 5, 15, 7, 10 } },
4313 { ISD::ROTL, MVT::v16i8, { 5, 15, 7, 10 } },
4314 { ISD::ROTR, MVT::v32i16, { 2, 8, 6, 8 } },
4315 { ISD::ROTR, MVT::v16i16, { 2, 8, 6, 7 } },
4316 { ISD::ROTR, MVT::v8i16, { 2, 7, 6, 7 } },
4317 { ISD::ROTR, MVT::v64i8, { 5, 6, 12, 14 } },
4318 { ISD::ROTR, MVT::v32i8, { 5, 14, 6, 9 } },
4319 { ISD::ROTR, MVT::v16i8, { 5, 14, 6, 9 } },
4320 { X86ISD::VROTLI, MVT::v32i16, { 2, 5, 3, 3 } },
4321 { X86ISD::VROTLI, MVT::v16i16, { 1, 5, 3, 3 } },
4322 { X86ISD::VROTLI, MVT::v8i16, { 1, 5, 3, 3 } },
4323 { X86ISD::VROTLI, MVT::v64i8, { 2, 9, 3, 4 } },
4324 { X86ISD::VROTLI, MVT::v32i8, { 1, 9, 3, 4 } },
4325 { X86ISD::VROTLI, MVT::v16i8, { 1, 8, 3, 4 } },
4326 { ISD::SADDSAT, MVT::v32i16, { 1, 1, 1, 1 } },
4327 { ISD::SADDSAT, MVT::v64i8, { 1, 1, 1, 1 } },
4328 { ISD::SMAX, MVT::v32i16, { 1, 1, 1, 1 } },
4329 { ISD::SMAX, MVT::v64i8, { 1, 1, 1, 1 } },
4330 { ISD::SMIN, MVT::v32i16, { 1, 1, 1, 1 } },
4331 { ISD::SMIN, MVT::v64i8, { 1, 1, 1, 1 } },
4332 { ISD::SMULO, MVT::v32i16, { 3, 6, 4, 4 } },
4333 { ISD::SMULO, MVT::v64i8, { 8, 21, 17, 18 } },
4334 { ISD::UMULO, MVT::v32i16, { 2, 5, 3, 3 } },
4335 { ISD::UMULO, MVT::v64i8, { 8, 15, 15, 16 } },
4336 { ISD::SSUBSAT, MVT::v32i16, { 1, 1, 1, 1 } },
4337 { ISD::SSUBSAT, MVT::v64i8, { 1, 1, 1, 1 } },
4338 { ISD::UADDSAT, MVT::v32i16, { 1, 1, 1, 1 } },
4339 { ISD::UADDSAT, MVT::v64i8, { 1, 1, 1, 1 } },
4340 { ISD::UMAX, MVT::v32i16, { 1, 1, 1, 1 } },
4341 { ISD::UMAX, MVT::v64i8, { 1, 1, 1, 1 } },
4342 { ISD::UMIN, MVT::v32i16, { 1, 1, 1, 1 } },
4343 { ISD::UMIN, MVT::v64i8, { 1, 1, 1, 1 } },
4344 { ISD::USUBSAT, MVT::v32i16, { 1, 1, 1, 1 } },
4345 { ISD::USUBSAT, MVT::v64i8, { 1, 1, 1, 1 } },
4346 };
4347 static const CostKindTblEntry AVX512CostTbl[] = {
4348 { ISD::ABS, MVT::v8i64, { 1, 1, 1, 1 } },
4349 { ISD::ABS, MVT::v4i64, { 1, 1, 1, 1 } },
4350 { ISD::ABS, MVT::v2i64, { 1, 1, 1, 1 } },
4351 { ISD::ABS, MVT::v16i32, { 1, 1, 1, 1 } },
4352 { ISD::ABS, MVT::v8i32, { 1, 1, 1, 1 } },
4353 { ISD::ABS, MVT::v32i16, { 2, 7, 4, 4 } },
4354 { ISD::ABS, MVT::v16i16, { 1, 1, 1, 1 } },
4355 { ISD::ABS, MVT::v64i8, { 2, 7, 4, 4 } },
4356 { ISD::ABS, MVT::v32i8, { 1, 1, 1, 1 } },
4357 { ISD::BITREVERSE, MVT::v8i64, { 9, 13, 20, 20 } },
4358 { ISD::BITREVERSE, MVT::v16i32, { 9, 13, 20, 20 } },
4359 { ISD::BITREVERSE, MVT::v32i16, { 9, 13, 20, 20 } },
4360 { ISD::BITREVERSE, MVT::v64i8, { 6, 11, 17, 17 } },
4361 { ISD::BSWAP, MVT::v8i64, { 4, 7, 5, 5 } },
4362 { ISD::BSWAP, MVT::v16i32, { 4, 7, 5, 5 } },
4363 { ISD::BSWAP, MVT::v32i16, { 4, 7, 5, 5 } },
4364 { ISD::CTLZ, MVT::v8i64, { 10, 28, 32, 32 } },
4365 { ISD::CTLZ, MVT::v16i32, { 12, 30, 38, 38 } },
4366 { ISD::CTLZ, MVT::v32i16, { 8, 15, 29, 29 } },
4367 { ISD::CTLZ, MVT::v64i8, { 6, 11, 19, 19 } },
4368 { ISD::CTPOP, MVT::v8i64, { 16, 16, 19, 19 } },
4369 { ISD::CTPOP, MVT::v16i32, { 24, 19, 27, 27 } },
4370 { ISD::CTPOP, MVT::v32i16, { 18, 15, 22, 22 } },
4371 { ISD::CTPOP, MVT::v64i8, { 12, 11, 16, 16 } },
4372 { ISD::CTTZ, MVT::v8i64, { 2, 8, 6, 7 } },
4373 { ISD::CTTZ, MVT::v16i32, { 2, 8, 6, 7 } },
4374 { ISD::CTTZ, MVT::v32i16, { 7, 17, 27, 27 } },
4375 { ISD::CTTZ, MVT::v64i8, { 6, 13, 21, 21 } },
4376 { ISD::MASK_BEFOREFIRST, MVT::v2i1, { 4, 8, 4, 8 } }, // kmov seq
4377 { ISD::MASK_BEFOREFIRST, MVT::v4i1, { 4, 8, 4, 8 } }, // kmov seq
4378 { ISD::MASK_BEFOREFIRST, MVT::v8i1, { 4, 8, 4, 8 } }, // kmov seq
4379 { ISD::MASK_BEFOREFIRST, MVT::v16i1, { 4, 8, 4, 8 } }, // kmov seq
4380 { ISD::MULHS, MVT::v16i32, { 3, 10, 6, 7 } },
4381 { ISD::MULHS, MVT::v8i32, { 3, 9, 6, 6 } },
4382 { ISD::MULHS, MVT::v32i16, { 3, 7, 5, 5 } },
4383 { ISD::MULHS, MVT::v16i16, { 1, 5, 1, 1 } },
4384 { ISD::MULHU, MVT::v16i32, { 3, 10, 6, 7 } },
4385 { ISD::MULHU, MVT::v8i32, { 3, 9, 6, 6 } },
4386 { ISD::MULHU, MVT::v32i16, { 3, 7, 5, 5 } },
4387 { ISD::MULHU, MVT::v16i16, { 1, 5, 1, 1 } },
4388 { ISD::ROTL, MVT::v8i64, { 1, 1, 1, 1 } },
4389 { ISD::ROTL, MVT::v4i64, { 1, 1, 1, 1 } },
4390 { ISD::ROTL, MVT::v2i64, { 1, 1, 1, 1 } },
4391 { ISD::ROTL, MVT::v16i32, { 1, 1, 1, 1 } },
4392 { ISD::ROTL, MVT::v8i32, { 1, 1, 1, 1 } },
4393 { ISD::ROTL, MVT::v4i32, { 1, 1, 1, 1 } },
4394 { ISD::ROTR, MVT::v8i64, { 1, 1, 1, 1 } },
4395 { ISD::ROTR, MVT::v4i64, { 1, 1, 1, 1 } },
4396 { ISD::ROTR, MVT::v2i64, { 1, 1, 1, 1 } },
4397 { ISD::ROTR, MVT::v16i32, { 1, 1, 1, 1 } },
4398 { ISD::ROTR, MVT::v8i32, { 1, 1, 1, 1 } },
4399 { ISD::ROTR, MVT::v4i32, { 1, 1, 1, 1 } },
4400 { X86ISD::VROTLI, MVT::v8i64, { 1, 1, 1, 1 } },
4401 { X86ISD::VROTLI, MVT::v4i64, { 1, 1, 1, 1 } },
4402 { X86ISD::VROTLI, MVT::v2i64, { 1, 1, 1, 1 } },
4403 { X86ISD::VROTLI, MVT::v16i32, { 1, 1, 1, 1 } },
4404 { X86ISD::VROTLI, MVT::v8i32, { 1, 1, 1, 1 } },
4405 { X86ISD::VROTLI, MVT::v4i32, { 1, 1, 1, 1 } },
4406 { ISD::SADDSAT, MVT::v2i64, { 3, 3, 8, 9 } },
4407 { ISD::SADDSAT, MVT::v4i64, { 2, 2, 6, 7 } },
4408 { ISD::SADDSAT, MVT::v8i64, { 3, 3, 6, 7 } },
4409 { ISD::SADDSAT, MVT::v4i32, { 2, 2, 6, 7 } },
4410 { ISD::SADDSAT, MVT::v8i32, { 2, 2, 6, 7 } },
4411 { ISD::SADDSAT, MVT::v16i32, { 3, 3, 6, 7 } },
4412 { ISD::SADDSAT, MVT::v32i16, { 2, 2, 2, 2 } },
4413 { ISD::SADDSAT, MVT::v64i8, { 2, 2, 2, 2 } },
4414 { ISD::SMAX, MVT::v8i64, { 1, 3, 1, 1 } },
4415 { ISD::SMAX, MVT::v16i32, { 1, 1, 1, 1 } },
4416 { ISD::SMAX, MVT::v32i16, { 3, 7, 5, 5 } },
4417 { ISD::SMAX, MVT::v64i8, { 3, 7, 5, 5 } },
4418 { ISD::SMAX, MVT::v4i64, { 1, 3, 1, 1 } },
4419 { ISD::SMAX, MVT::v2i64, { 1, 3, 1, 1 } },
4420 { ISD::SMIN, MVT::v8i64, { 1, 3, 1, 1 } },
4421 { ISD::SMIN, MVT::v16i32, { 1, 1, 1, 1 } },
4422 { ISD::SMIN, MVT::v32i16, { 3, 7, 5, 5 } },
4423 { ISD::SMIN, MVT::v64i8, { 3, 7, 5, 5 } },
4424 { ISD::SMIN, MVT::v4i64, { 1, 3, 1, 1 } },
4425 { ISD::SMIN, MVT::v2i64, { 1, 3, 1, 1 } },
4426 { ISD::SMULO, MVT::v8i64, { 44, 44, 81, 93 } },
4427 { ISD::SMULO, MVT::v16i32, { 5, 12, 9, 11 } },
4428 { ISD::SMULO, MVT::v32i16, { 6, 12, 17, 17 } },
4429 { ISD::SMULO, MVT::v64i8, { 22, 28, 42, 42 } },
4430 { ISD::SSUBSAT, MVT::v2i64, { 2, 13, 9, 10 } },
4431 { ISD::SSUBSAT, MVT::v4i64, { 2, 15, 7, 8 } },
4432 { ISD::SSUBSAT, MVT::v8i64, { 2, 14, 7, 8 } },
4433 { ISD::SSUBSAT, MVT::v4i32, { 2, 14, 7, 8 } },
4434 { ISD::SSUBSAT, MVT::v8i32, { 2, 15, 7, 8 } },
4435 { ISD::SSUBSAT, MVT::v16i32, { 2, 14, 7, 8 } },
4436 { ISD::SSUBSAT, MVT::v32i16, { 2, 2, 2, 2 } },
4437 { ISD::SSUBSAT, MVT::v64i8, { 2, 2, 2, 2 } },
4438 { ISD::UMAX, MVT::v8i64, { 1, 3, 1, 1 } },
4439 { ISD::UMAX, MVT::v16i32, { 1, 1, 1, 1 } },
4440 { ISD::UMAX, MVT::v32i16, { 3, 7, 5, 5 } },
4441 { ISD::UMAX, MVT::v64i8, { 3, 7, 5, 5 } },
4442 { ISD::UMAX, MVT::v4i64, { 1, 3, 1, 1 } },
4443 { ISD::UMAX, MVT::v2i64, { 1, 3, 1, 1 } },
4444 { ISD::UMIN, MVT::v8i64, { 1, 3, 1, 1 } },
4445 { ISD::UMIN, MVT::v16i32, { 1, 1, 1, 1 } },
4446 { ISD::UMIN, MVT::v32i16, { 3, 7, 5, 5 } },
4447 { ISD::UMIN, MVT::v64i8, { 3, 7, 5, 5 } },
4448 { ISD::UMIN, MVT::v4i64, { 1, 3, 1, 1 } },
4449 { ISD::UMIN, MVT::v2i64, { 1, 3, 1, 1 } },
4450 { ISD::UMULO, MVT::v8i64, { 52, 52, 95, 104} },
4451 { ISD::UMULO, MVT::v16i32, { 5, 12, 8, 10 } },
4452 { ISD::UMULO, MVT::v32i16, { 5, 13, 16, 16 } },
4453 { ISD::UMULO, MVT::v64i8, { 18, 24, 30, 30 } },
4454 { ISD::UADDSAT, MVT::v2i64, { 1, 4, 4, 4 } },
4455 { ISD::UADDSAT, MVT::v4i64, { 1, 4, 4, 4 } },
4456 { ISD::UADDSAT, MVT::v8i64, { 1, 4, 4, 4 } },
4457 { ISD::UADDSAT, MVT::v4i32, { 1, 2, 4, 4 } },
4458 { ISD::UADDSAT, MVT::v8i32, { 1, 2, 4, 4 } },
4459 { ISD::UADDSAT, MVT::v16i32, { 2, 2, 4, 4 } },
4460 { ISD::UADDSAT, MVT::v32i16, { 2, 2, 2, 2 } },
4461 { ISD::UADDSAT, MVT::v64i8, { 2, 2, 2, 2 } },
4462 { ISD::USUBSAT, MVT::v2i64, { 1, 4, 2, 2 } },
4463 { ISD::USUBSAT, MVT::v4i64, { 1, 4, 2, 2 } },
4464 { ISD::USUBSAT, MVT::v8i64, { 1, 4, 2, 2 } },
4465 { ISD::USUBSAT, MVT::v8i32, { 1, 2, 2, 2 } },
4466 { ISD::USUBSAT, MVT::v16i32, { 1, 2, 2, 2 } },
4467 { ISD::USUBSAT, MVT::v32i16, { 2, 2, 2, 2 } },
4468 { ISD::USUBSAT, MVT::v64i8, { 2, 2, 2, 2 } },
4469 { ISD::FMAXNUM, MVT::f32, { 2, 2, 3, 3 } },
4470 { ISD::FMAXNUM, MVT::v4f32, { 1, 1, 3, 3 } },
4471 { ISD::FMAXNUM, MVT::v8f32, { 2, 2, 3, 3 } },
4472 { ISD::FMAXNUM, MVT::v16f32, { 4, 4, 3, 3 } },
4473 { ISD::FMAXNUM, MVT::f64, { 2, 2, 3, 3 } },
4474 { ISD::FMAXNUM, MVT::v2f64, { 1, 1, 3, 3 } },
4475 { ISD::FMAXNUM, MVT::v4f64, { 2, 2, 3, 3 } },
4476 { ISD::FMAXNUM, MVT::v8f64, { 3, 3, 3, 3 } },
4477 { ISD::FSQRT, MVT::f32, { 3, 12, 1, 1 } }, // Skylake from http://www.agner.org/
4478 { ISD::FSQRT, MVT::v4f32, { 3, 12, 1, 1 } }, // Skylake from http://www.agner.org/
4479 { ISD::FSQRT, MVT::v8f32, { 6, 12, 1, 1 } }, // Skylake from http://www.agner.org/
4480 { ISD::FSQRT, MVT::v16f32, { 12, 20, 1, 3 } }, // Skylake from http://www.agner.org/
4481 { ISD::FSQRT, MVT::f64, { 6, 18, 1, 1 } }, // Skylake from http://www.agner.org/
4482 { ISD::FSQRT, MVT::v2f64, { 6, 18, 1, 1 } }, // Skylake from http://www.agner.org/
4483 { ISD::FSQRT, MVT::v4f64, { 12, 18, 1, 1 } }, // Skylake from http://www.agner.org/
4484 { ISD::FSQRT, MVT::v8f64, { 24, 32, 1, 3 } }, // Skylake from http://www.agner.org/
4485 };
4486 static const CostKindTblEntry XOPCostTbl[] = {
4487 { ISD::BITREVERSE, MVT::v4i64, { 3, 6, 5, 6 } },
4488 { ISD::BITREVERSE, MVT::v8i32, { 3, 6, 5, 6 } },
4489 { ISD::BITREVERSE, MVT::v16i16, { 3, 6, 5, 6 } },
4490 { ISD::BITREVERSE, MVT::v32i8, { 3, 6, 5, 6 } },
4491 { ISD::BITREVERSE, MVT::v2i64, { 2, 7, 1, 1 } },
4492 { ISD::BITREVERSE, MVT::v4i32, { 2, 7, 1, 1 } },
4493 { ISD::BITREVERSE, MVT::v8i16, { 2, 7, 1, 1 } },
4494 { ISD::BITREVERSE, MVT::v16i8, { 2, 7, 1, 1 } },
4495 { ISD::BITREVERSE, MVT::i64, { 2, 2, 3, 4 } },
4496 { ISD::BITREVERSE, MVT::i32, { 2, 2, 3, 4 } },
4497 { ISD::BITREVERSE, MVT::i16, { 2, 2, 3, 4 } },
4498 { ISD::BITREVERSE, MVT::i8, { 2, 2, 3, 4 } },
4499 // XOP: ROTL = VPROT(X,Y), ROTR = VPROT(X,SUB(0,Y))
4500 { ISD::ROTL, MVT::v4i64, { 4, 7, 5, 6 } },
4501 { ISD::ROTL, MVT::v8i32, { 4, 7, 5, 6 } },
4502 { ISD::ROTL, MVT::v16i16, { 4, 7, 5, 6 } },
4503 { ISD::ROTL, MVT::v32i8, { 4, 7, 5, 6 } },
4504 { ISD::ROTL, MVT::v2i64, { 1, 3, 1, 1 } },
4505 { ISD::ROTL, MVT::v4i32, { 1, 3, 1, 1 } },
4506 { ISD::ROTL, MVT::v8i16, { 1, 3, 1, 1 } },
4507 { ISD::ROTL, MVT::v16i8, { 1, 3, 1, 1 } },
4508 { ISD::ROTR, MVT::v4i64, { 4, 7, 8, 9 } },
4509 { ISD::ROTR, MVT::v8i32, { 4, 7, 8, 9 } },
4510 { ISD::ROTR, MVT::v16i16, { 4, 7, 8, 9 } },
4511 { ISD::ROTR, MVT::v32i8, { 4, 7, 8, 9 } },
4512 { ISD::ROTR, MVT::v2i64, { 1, 3, 3, 3 } },
4513 { ISD::ROTR, MVT::v4i32, { 1, 3, 3, 3 } },
4514 { ISD::ROTR, MVT::v8i16, { 1, 3, 3, 3 } },
4515 { ISD::ROTR, MVT::v16i8, { 1, 3, 3, 3 } },
4516 { X86ISD::VROTLI, MVT::v4i64, { 4, 7, 5, 6 } },
4517 { X86ISD::VROTLI, MVT::v8i32, { 4, 7, 5, 6 } },
4518 { X86ISD::VROTLI, MVT::v16i16, { 4, 7, 5, 6 } },
4519 { X86ISD::VROTLI, MVT::v32i8, { 4, 7, 5, 6 } },
4520 { X86ISD::VROTLI, MVT::v2i64, { 1, 3, 1, 1 } },
4521 { X86ISD::VROTLI, MVT::v4i32, { 1, 3, 1, 1 } },
4522 { X86ISD::VROTLI, MVT::v8i16, { 1, 3, 1, 1 } },
4523 { X86ISD::VROTLI, MVT::v16i8, { 1, 3, 1, 1 } },
4524 };
4525 static const CostKindTblEntry AVX2CostTbl[] = {
4526 { ISD::ABS, MVT::v2i64, { 2, 4, 3, 5 } }, // VBLENDVPD(X,VPSUBQ(0,X),X)
4527 { ISD::ABS, MVT::v4i64, { 2, 4, 3, 5 } }, // VBLENDVPD(X,VPSUBQ(0,X),X)
4528 { ISD::ABS, MVT::v4i32, { 1, 1, 1, 1 } },
4529 { ISD::ABS, MVT::v8i32, { 1, 1, 1, 2 } },
4530 { ISD::ABS, MVT::v8i16, { 1, 1, 1, 1 } },
4531 { ISD::ABS, MVT::v16i16, { 1, 1, 1, 2 } },
4532 { ISD::ABS, MVT::v16i8, { 1, 1, 1, 1 } },
4533 { ISD::ABS, MVT::v32i8, { 1, 1, 1, 2 } },
4534 { ISD::BITREVERSE, MVT::v2i64, { 3, 11, 10, 11 } },
4535 { ISD::BITREVERSE, MVT::v4i64, { 5, 11, 10, 17 } },
4536 { ISD::BITREVERSE, MVT::v4i32, { 3, 11, 10, 11 } },
4537 { ISD::BITREVERSE, MVT::v8i32, { 5, 11, 10, 17 } },
4538 { ISD::BITREVERSE, MVT::v8i16, { 3, 11, 10, 11 } },
4539 { ISD::BITREVERSE, MVT::v16i16, { 5, 11, 10, 17 } },
4540 { ISD::BITREVERSE, MVT::v16i8, { 3, 6, 9, 9 } },
4541 { ISD::BITREVERSE, MVT::v32i8, { 4, 5, 9, 15 } },
4542 { ISD::BSWAP, MVT::v2i64, { 1, 2, 1, 2 } },
4543 { ISD::BSWAP, MVT::v4i64, { 1, 3, 1, 2 } },
4544 { ISD::BSWAP, MVT::v4i32, { 1, 2, 1, 2 } },
4545 { ISD::BSWAP, MVT::v8i32, { 1, 3, 1, 2 } },
4546 { ISD::BSWAP, MVT::v8i16, { 1, 2, 1, 2 } },
4547 { ISD::BSWAP, MVT::v16i16, { 1, 3, 1, 2 } },
4548 { ISD::CTLZ, MVT::v2i64, { 7, 18, 24, 25 } },
4549 { ISD::CTLZ, MVT::v4i64, { 14, 18, 24, 44 } },
4550 { ISD::CTLZ, MVT::v4i32, { 5, 16, 19, 20 } },
4551 { ISD::CTLZ, MVT::v8i32, { 10, 16, 19, 34 } },
4552 { ISD::CTLZ, MVT::v8i16, { 4, 13, 14, 15 } },
4553 { ISD::CTLZ, MVT::v16i16, { 6, 14, 14, 24 } },
4554 { ISD::CTLZ, MVT::v16i8, { 3, 12, 9, 10 } },
4555 { ISD::CTLZ, MVT::v32i8, { 4, 12, 9, 14 } },
4556 { ISD::CTPOP, MVT::v2i64, { 3, 9, 10, 10 } },
4557 { ISD::CTPOP, MVT::v4i64, { 4, 9, 10, 14 } },
4558 { ISD::CTPOP, MVT::v4i32, { 7, 12, 14, 14 } },
4559 { ISD::CTPOP, MVT::v8i32, { 7, 12, 14, 18 } },
4560 { ISD::CTPOP, MVT::v8i16, { 3, 7, 11, 11 } },
4561 { ISD::CTPOP, MVT::v16i16, { 6, 8, 11, 18 } },
4562 { ISD::CTPOP, MVT::v16i8, { 2, 5, 8, 8 } },
4563 { ISD::CTPOP, MVT::v32i8, { 3, 5, 8, 12 } },
4564 { ISD::CTTZ, MVT::v2i64, { 4, 11, 13, 13 } },
4565 { ISD::CTTZ, MVT::v4i64, { 5, 11, 13, 20 } },
4566 { ISD::CTTZ, MVT::v4i32, { 7, 14, 17, 17 } },
4567 { ISD::CTTZ, MVT::v8i32, { 7, 15, 17, 24 } },
4568 { ISD::CTTZ, MVT::v8i16, { 4, 9, 14, 14 } },
4569 { ISD::CTTZ, MVT::v16i16, { 6, 9, 14, 24 } },
4570 { ISD::CTTZ, MVT::v16i8, { 3, 7, 11, 11 } },
4571 { ISD::CTTZ, MVT::v32i8, { 5, 7, 11, 18 } },
4572 { ISD::MASK_BEFOREFIRST, MVT::v2i64, { 7, 13, 8, 13 } }, // movmsk seq
4573 { ISD::MASK_BEFOREFIRST, MVT::v4i32, { 7, 13, 8, 13 } }, // movmsk seq
4574 { ISD::MASK_BEFOREFIRST, MVT::v8i16, { 8, 14, 9, 14 } }, // movmsk seq
4575 { ISD::MASK_BEFOREFIRST, MVT::v16i8, { 7, 11, 8, 11 } }, // movmsk seq
4576 { ISD::MASK_BEFOREFIRST, MVT::v32i8, { 8, 14, 9, 14 } }, // movmsk seq
4577 { ISD::MULHS, MVT::v8i32, { 4, 9, 6, 12 } },
4578 { ISD::MULHS, MVT::v16i16, { 2, 5, 1, 2 } },
4579 { ISD::MULHU, MVT::v8i32, { 4, 9, 6, 12 } },
4580 { ISD::MULHU, MVT::v16i16, { 2, 5, 1, 2 } },
4581 { ISD::SADDSAT, MVT::v2i64, { 4, 13, 8, 11 } },
4582 { ISD::SADDSAT, MVT::v4i64, { 3, 10, 8, 12 } },
4583 { ISD::SADDSAT, MVT::v4i32, { 2, 6, 7, 9 } },
4584 { ISD::SADDSAT, MVT::v8i32, { 4, 6, 7, 13 } },
4585 { ISD::SADDSAT, MVT::v16i16, { 1, 1, 1, 2 } },
4586 { ISD::SADDSAT, MVT::v32i8, { 1, 1, 1, 2 } },
4587 { ISD::SMAX, MVT::v2i64, { 2, 7, 2, 3 } },
4588 { ISD::SMAX, MVT::v4i64, { 2, 7, 2, 3 } },
4589 { ISD::SMAX, MVT::v8i32, { 1, 1, 1, 2 } },
4590 { ISD::SMAX, MVT::v16i16, { 1, 1, 1, 2 } },
4591 { ISD::SMAX, MVT::v32i8, { 1, 1, 1, 2 } },
4592 { ISD::SMIN, MVT::v2i64, { 2, 7, 2, 3 } },
4593 { ISD::SMIN, MVT::v4i64, { 2, 7, 2, 3 } },
4594 { ISD::SMIN, MVT::v8i32, { 1, 1, 1, 2 } },
4595 { ISD::SMIN, MVT::v16i16, { 1, 1, 1, 2 } },
4596 { ISD::SMIN, MVT::v32i8, { 1, 1, 1, 2 } },
4597 { ISD::SMULO, MVT::v4i64, { 20, 20, 33, 37 } },
4598 { ISD::SMULO, MVT::v2i64, { 8, 8, 13, 15 } },
4599 { ISD::SMULO, MVT::v8i32, { 8, 20, 13, 24 } },
4600 { ISD::SMULO, MVT::v4i32, { 5, 15, 11, 12 } },
4601 { ISD::SMULO, MVT::v16i16, { 4, 14, 8, 14 } },
4602 { ISD::SMULO, MVT::v8i16, { 3, 9, 6, 6 } },
4603 { ISD::SMULO, MVT::v32i8, { 9, 15, 18, 35 } },
4604 { ISD::SMULO, MVT::v16i8, { 6, 22, 14, 21 } },
4605 { ISD::SSUBSAT, MVT::v2i64, { 4, 13, 9, 13 } },
4606 { ISD::SSUBSAT, MVT::v4i64, { 4, 15, 9, 13 } },
4607 { ISD::SSUBSAT, MVT::v4i32, { 3, 14, 9, 11 } },
4608 { ISD::SSUBSAT, MVT::v8i32, { 4, 15, 9, 16 } },
4609 { ISD::SSUBSAT, MVT::v16i16, { 1, 1, 1, 2 } },
4610 { ISD::SSUBSAT, MVT::v32i8, { 1, 1, 1, 2 } },
4611 { ISD::UADDSAT, MVT::v2i64, { 2, 8, 6, 6 } },
4612 { ISD::UADDSAT, MVT::v4i64, { 3, 8, 6, 10 } },
4613 { ISD::UADDSAT, MVT::v8i32, { 2, 2, 4, 8 } },
4614 { ISD::UADDSAT, MVT::v16i16, { 1, 1, 1, 2 } },
4615 { ISD::UADDSAT, MVT::v32i8, { 1, 1, 1, 2 } },
4616 { ISD::UMAX, MVT::v2i64, { 2, 8, 5, 6 } },
4617 { ISD::UMAX, MVT::v4i64, { 2, 8, 5, 8 } },
4618 { ISD::UMAX, MVT::v8i32, { 1, 1, 1, 2 } },
4619 { ISD::UMAX, MVT::v16i16, { 1, 1, 1, 2 } },
4620 { ISD::UMAX, MVT::v32i8, { 1, 1, 1, 2 } },
4621 { ISD::UMIN, MVT::v2i64, { 2, 8, 5, 6 } },
4622 { ISD::UMIN, MVT::v4i64, { 2, 8, 5, 8 } },
4623 { ISD::UMIN, MVT::v8i32, { 1, 1, 1, 2 } },
4624 { ISD::UMIN, MVT::v16i16, { 1, 1, 1, 2 } },
4625 { ISD::UMIN, MVT::v32i8, { 1, 1, 1, 2 } },
4626 { ISD::UMULO, MVT::v4i64, { 24, 24, 39, 43 } },
4627 { ISD::UMULO, MVT::v2i64, { 10, 10, 15, 19 } },
4628 { ISD::UMULO, MVT::v8i32, { 8, 11, 13, 23 } },
4629 { ISD::UMULO, MVT::v4i32, { 5, 12, 11, 12 } },
4630 { ISD::UMULO, MVT::v16i16, { 4, 6, 8, 13 } },
4631 { ISD::UMULO, MVT::v8i16, { 2, 8, 6, 6 } },
4632 { ISD::UMULO, MVT::v32i8, { 9, 13, 17, 33 } },
4633 { ISD::UMULO, MVT::v16i8, { 6, 19, 13, 20 } },
4634 { ISD::USUBSAT, MVT::v2i64, { 2, 7, 6, 6 } },
4635 { ISD::USUBSAT, MVT::v4i64, { 3, 7, 6, 10 } },
4636 { ISD::USUBSAT, MVT::v8i32, { 2, 2, 2, 4 } },
4637 { ISD::USUBSAT, MVT::v16i16, { 1, 1, 1, 2 } },
4638 { ISD::USUBSAT, MVT::v32i8, { 1, 1, 1, 2 } },
4639 { ISD::FMAXNUM, MVT::f32, { 2, 7, 3, 5 } }, // MAXSS + CMPUNORDSS + BLENDVPS
4640 { ISD::FMAXNUM, MVT::v4f32, { 2, 7, 3, 5 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4641 { ISD::FMAXNUM, MVT::v8f32, { 3, 7, 3, 6 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4642 { ISD::FMAXNUM, MVT::f64, { 2, 7, 3, 5 } }, // MAXSD + CMPUNORDSD + BLENDVPD
4643 { ISD::FMAXNUM, MVT::v2f64, { 2, 7, 3, 5 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4644 { ISD::FMAXNUM, MVT::v4f64, { 3, 7, 3, 6 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4645 { ISD::FSQRT, MVT::f32, { 7, 15, 1, 1 } }, // vsqrtss
4646 { ISD::FSQRT, MVT::v4f32, { 7, 15, 1, 1 } }, // vsqrtps
4647 { ISD::FSQRT, MVT::v8f32, { 14, 21, 1, 3 } }, // vsqrtps
4648 { ISD::FSQRT, MVT::f64, { 14, 21, 1, 1 } }, // vsqrtsd
4649 { ISD::FSQRT, MVT::v2f64, { 14, 21, 1, 1 } }, // vsqrtpd
4650 { ISD::FSQRT, MVT::v4f64, { 28, 35, 1, 3 } }, // vsqrtpd
4651 };
4652 static const CostKindTblEntry AVX1CostTbl[] = {
4653 { ISD::ABS, MVT::v4i64, { 6, 8, 6, 12 } }, // VBLENDVPD(X,VPSUBQ(0,X),X)
4654 { ISD::ABS, MVT::v8i32, { 3, 6, 4, 5 } },
4655 { ISD::ABS, MVT::v16i16, { 3, 6, 4, 5 } },
4656 { ISD::ABS, MVT::v32i8, { 3, 6, 4, 5 } },
4657 { ISD::BITREVERSE, MVT::v4i64, { 17, 20, 20, 33 } }, // 2 x 128-bit Op + extract/insert
4658 { ISD::BITREVERSE, MVT::v2i64, { 8, 13, 10, 16 } },
4659 { ISD::BITREVERSE, MVT::v8i32, { 17, 20, 20, 33 } }, // 2 x 128-bit Op + extract/insert
4660 { ISD::BITREVERSE, MVT::v4i32, { 8, 13, 10, 16 } },
4661 { ISD::BITREVERSE, MVT::v16i16, { 17, 20, 20, 33 } }, // 2 x 128-bit Op + extract/insert
4662 { ISD::BITREVERSE, MVT::v8i16, { 8, 13, 10, 16 } },
4663 { ISD::BITREVERSE, MVT::v32i8, { 13, 15, 17, 26 } }, // 2 x 128-bit Op + extract/insert
4664 { ISD::BITREVERSE, MVT::v16i8, { 7, 7, 9, 13 } },
4665 { ISD::BSWAP, MVT::v4i64, { 5, 6, 5, 10 } },
4666 { ISD::BSWAP, MVT::v2i64, { 2, 2, 1, 3 } },
4667 { ISD::BSWAP, MVT::v8i32, { 5, 6, 5, 10 } },
4668 { ISD::BSWAP, MVT::v4i32, { 2, 2, 1, 3 } },
4669 { ISD::BSWAP, MVT::v16i16, { 5, 6, 5, 10 } },
4670 { ISD::BSWAP, MVT::v8i16, { 2, 2, 1, 3 } },
4671 { ISD::CTLZ, MVT::v4i64, { 29, 33, 49, 58 } }, // 2 x 128-bit Op + extract/insert
4672 { ISD::CTLZ, MVT::v2i64, { 14, 24, 24, 28 } },
4673 { ISD::CTLZ, MVT::v8i32, { 24, 28, 39, 48 } }, // 2 x 128-bit Op + extract/insert
4674 { ISD::CTLZ, MVT::v4i32, { 12, 20, 19, 23 } },
4675 { ISD::CTLZ, MVT::v16i16, { 19, 22, 29, 38 } }, // 2 x 128-bit Op + extract/insert
4676 { ISD::CTLZ, MVT::v8i16, { 9, 16, 14, 18 } },
4677 { ISD::CTLZ, MVT::v32i8, { 14, 15, 19, 28 } }, // 2 x 128-bit Op + extract/insert
4678 { ISD::CTLZ, MVT::v16i8, { 7, 12, 9, 13 } },
4679 { ISD::CTPOP, MVT::v4i64, { 14, 18, 19, 28 } }, // 2 x 128-bit Op + extract/insert
4680 { ISD::CTPOP, MVT::v2i64, { 7, 14, 10, 14 } },
4681 { ISD::CTPOP, MVT::v8i32, { 18, 24, 27, 36 } }, // 2 x 128-bit Op + extract/insert
4682 { ISD::CTPOP, MVT::v4i32, { 9, 20, 14, 18 } },
4683 { ISD::CTPOP, MVT::v16i16, { 16, 21, 22, 31 } }, // 2 x 128-bit Op + extract/insert
4684 { ISD::CTPOP, MVT::v8i16, { 8, 18, 11, 15 } },
4685 { ISD::CTPOP, MVT::v32i8, { 13, 15, 16, 25 } }, // 2 x 128-bit Op + extract/insert
4686 { ISD::CTPOP, MVT::v16i8, { 6, 12, 8, 12 } },
4687 { ISD::CTTZ, MVT::v4i64, { 17, 22, 24, 33 } }, // 2 x 128-bit Op + extract/insert
4688 { ISD::CTTZ, MVT::v2i64, { 9, 19, 13, 17 } },
4689 { ISD::CTTZ, MVT::v8i32, { 21, 27, 32, 41 } }, // 2 x 128-bit Op + extract/insert
4690 { ISD::CTTZ, MVT::v4i32, { 11, 24, 17, 21 } },
4691 { ISD::CTTZ, MVT::v16i16, { 18, 24, 27, 36 } }, // 2 x 128-bit Op + extract/insert
4692 { ISD::CTTZ, MVT::v8i16, { 9, 21, 14, 18 } },
4693 { ISD::CTTZ, MVT::v32i8, { 15, 18, 21, 30 } }, // 2 x 128-bit Op + extract/insert
4694 { ISD::CTTZ, MVT::v16i8, { 8, 16, 11, 15 } },
4695 { ISD::MASK_BEFOREFIRST, MVT::v2i64, { 8, 11, 9, 11 } }, // movmsk seq
4696 { ISD::MASK_BEFOREFIRST, MVT::v4i32, { 8, 11, 9, 11 } }, // movmsk seq
4697 { ISD::MASK_BEFOREFIRST, MVT::v8i16, { 10, 13, 11, 13 } }, // movmsk seq
4698 { ISD::MASK_BEFOREFIRST, MVT::v16i8, { 8, 11, 9, 11 } }, // movmsk seq
4699 { ISD::MASK_BEFOREFIRST, MVT::v32i8, { 18, 24, 19, 24 } }, // movmsk seq
4700 { ISD::MULHS, MVT::v8i32, { 9, 11, 14, 18 } },
4701 { ISD::MULHS, MVT::v16i16, { 3, 7, 5, 6 } },
4702 { ISD::MULHU, MVT::v8i32, { 9, 11, 14, 18 } },
4703 { ISD::MULHU, MVT::v16i16, { 3, 7, 5, 6 } },
4704 { ISD::SADDSAT, MVT::v2i64, { 6, 13, 8, 11 } },
4705 { ISD::SADDSAT, MVT::v4i64, { 13, 20, 15, 25 } }, // 2 x 128-bit Op + extract/insert
4706 { ISD::SADDSAT, MVT::v8i32, { 12, 18, 14, 24 } }, // 2 x 128-bit Op + extract/insert
4707 { ISD::SADDSAT, MVT::v16i16, { 3, 3, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4708 { ISD::SADDSAT, MVT::v32i8, { 3, 3, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4709 { ISD::SMAX, MVT::v4i64, { 6, 9, 6, 12 } }, // 2 x 128-bit Op + extract/insert
4710 { ISD::SMAX, MVT::v2i64, { 3, 7, 2, 4 } },
4711 { ISD::SMAX, MVT::v8i32, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4712 { ISD::SMAX, MVT::v16i16, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4713 { ISD::SMAX, MVT::v32i8, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4714 { ISD::SMIN, MVT::v4i64, { 6, 9, 6, 12 } }, // 2 x 128-bit Op + extract/insert
4715 { ISD::SMIN, MVT::v2i64, { 3, 7, 2, 3 } },
4716 { ISD::SMIN, MVT::v8i32, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4717 { ISD::SMIN, MVT::v16i16, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4718 { ISD::SMIN, MVT::v32i8, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4719 { ISD::SMULO, MVT::v4i64, { 20, 20, 33, 37 } },
4720 { ISD::SMULO, MVT::v2i64, { 9, 9, 13, 17 } },
4721 { ISD::SMULO, MVT::v8i32, { 15, 20, 24, 29 } },
4722 { ISD::SMULO, MVT::v4i32, { 7, 15, 11, 13 } },
4723 { ISD::SMULO, MVT::v16i16, { 8, 14, 14, 15 } },
4724 { ISD::SMULO, MVT::v8i16, { 3, 9, 6, 6 } },
4725 { ISD::SMULO, MVT::v32i8, { 20, 20, 37, 39 } },
4726 { ISD::SMULO, MVT::v16i8, { 9, 22, 18, 21 } },
4727 { ISD::SSUBSAT, MVT::v2i64, { 7, 13, 9, 13 } },
4728 { ISD::SSUBSAT, MVT::v4i64, { 15, 21, 18, 29 } }, // 2 x 128-bit Op + extract/insert
4729 { ISD::SSUBSAT, MVT::v8i32, { 15, 19, 18, 29 } }, // 2 x 128-bit Op + extract/insert
4730 { ISD::SSUBSAT, MVT::v16i16, { 3, 3, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4731 { ISD::SSUBSAT, MVT::v32i8, { 3, 3, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4732 { ISD::UADDSAT, MVT::v2i64, { 3, 8, 6, 6 } },
4733 { ISD::UADDSAT, MVT::v4i64, { 8, 11, 14, 15 } }, // 2 x 128-bit Op + extract/insert
4734 { ISD::UADDSAT, MVT::v8i32, { 6, 6, 10, 11 } }, // 2 x 128-bit Op + extract/insert
4735 { ISD::UADDSAT, MVT::v16i16, { 3, 3, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4736 { ISD::UADDSAT, MVT::v32i8, { 3, 3, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4737 { ISD::UMAX, MVT::v4i64, { 9, 10, 11, 17 } }, // 2 x 128-bit Op + extract/insert
4738 { ISD::UMAX, MVT::v2i64, { 4, 8, 5, 7 } },
4739 { ISD::UMAX, MVT::v8i32, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4740 { ISD::UMAX, MVT::v16i16, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4741 { ISD::UMAX, MVT::v32i8, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4742 { ISD::UMIN, MVT::v4i64, { 9, 10, 11, 17 } }, // 2 x 128-bit Op + extract/insert
4743 { ISD::UMIN, MVT::v2i64, { 4, 8, 5, 7 } },
4744 { ISD::UMIN, MVT::v8i32, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4745 { ISD::UMIN, MVT::v16i16, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4746 { ISD::UMIN, MVT::v32i8, { 4, 6, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4747 { ISD::UMULO, MVT::v4i64, { 24, 26, 39, 45 } },
4748 { ISD::UMULO, MVT::v2i64, { 10, 12, 15, 20 } },
4749 { ISD::UMULO, MVT::v8i32, { 14, 15, 23, 28 } },
4750 { ISD::UMULO, MVT::v4i32, { 7, 12, 11, 13 } },
4751 { ISD::UMULO, MVT::v16i16, { 7, 11, 13, 14 } },
4752 { ISD::UMULO, MVT::v8i16, { 3, 8, 6, 6 } },
4753 { ISD::UMULO, MVT::v32i8, { 19, 19, 35, 37 } },
4754 { ISD::UMULO, MVT::v16i8, { 9, 19, 17, 20 } },
4755 { ISD::USUBSAT, MVT::v2i64, { 3, 7, 6, 6 } },
4756 { ISD::USUBSAT, MVT::v4i64, { 8, 10, 14, 15 } }, // 2 x 128-bit Op + extract/insert
4757 { ISD::USUBSAT, MVT::v8i32, { 4, 4, 7, 8 } }, // 2 x 128-bit Op + extract/insert
4758 { ISD::USUBSAT, MVT::v8i32, { 3, 3, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4759 { ISD::USUBSAT, MVT::v16i16, { 3, 3, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4760 { ISD::USUBSAT, MVT::v32i8, { 3, 3, 5, 6 } }, // 2 x 128-bit Op + extract/insert
4761 { ISD::FMAXNUM, MVT::f32, { 3, 6, 3, 5 } }, // MAXSS + CMPUNORDSS + BLENDVPS
4762 { ISD::FMAXNUM, MVT::v4f32, { 3, 6, 3, 5 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4763 { ISD::FMAXNUM, MVT::v8f32, { 5, 7, 3, 10 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4764 { ISD::FMAXNUM, MVT::f64, { 3, 6, 3, 5 } }, // MAXSD + CMPUNORDSD + BLENDVPD
4765 { ISD::FMAXNUM, MVT::v2f64, { 3, 6, 3, 5 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4766 { ISD::FMAXNUM, MVT::v4f64, { 5, 7, 3, 10 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4767 { ISD::FSQRT, MVT::f32, { 21, 21, 1, 1 } }, // vsqrtss
4768 { ISD::FSQRT, MVT::v4f32, { 21, 21, 1, 1 } }, // vsqrtps
4769 { ISD::FSQRT, MVT::v8f32, { 42, 42, 1, 3 } }, // vsqrtps
4770 { ISD::FSQRT, MVT::f64, { 27, 27, 1, 1 } }, // vsqrtsd
4771 { ISD::FSQRT, MVT::v2f64, { 27, 27, 1, 1 } }, // vsqrtpd
4772 { ISD::FSQRT, MVT::v4f64, { 54, 54, 1, 3 } }, // vsqrtpd
4773 };
4774 static const CostKindTblEntry GFNICostTbl[] = {
4775 { ISD::BITREVERSE, MVT::i8, { 3, 3, 3, 4 } }, // gf2p8affineqb
4776 { ISD::BITREVERSE, MVT::i16, { 3, 3, 4, 6 } }, // gf2p8affineqb
4777 { ISD::BITREVERSE, MVT::i32, { 3, 3, 4, 5 } }, // gf2p8affineqb
4778 { ISD::BITREVERSE, MVT::i64, { 3, 3, 4, 6 } }, // gf2p8affineqb
4779 { ISD::BITREVERSE, MVT::v16i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
4780 { ISD::BITREVERSE, MVT::v32i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
4781 { ISD::BITREVERSE, MVT::v64i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
4782 { ISD::BITREVERSE, MVT::v8i16, { 1, 8, 2, 4 } }, // gf2p8affineqb
4783 { ISD::BITREVERSE, MVT::v16i16, { 1, 9, 2, 4 } }, // gf2p8affineqb
4784 { ISD::BITREVERSE, MVT::v32i16, { 1, 9, 2, 4 } }, // gf2p8affineqb
4785 { ISD::BITREVERSE, MVT::v4i32, { 1, 8, 2, 4 } }, // gf2p8affineqb
4786 { ISD::BITREVERSE, MVT::v8i32, { 1, 9, 2, 4 } }, // gf2p8affineqb
4787 { ISD::BITREVERSE, MVT::v16i32, { 1, 9, 2, 4 } }, // gf2p8affineqb
4788 { ISD::BITREVERSE, MVT::v2i64, { 1, 8, 2, 4 } }, // gf2p8affineqb
4789 { ISD::BITREVERSE, MVT::v4i64, { 1, 9, 2, 4 } }, // gf2p8affineqb
4790 { ISD::BITREVERSE, MVT::v8i64, { 1, 9, 2, 4 } }, // gf2p8affineqb
4791 { X86ISD::VROTLI, MVT::v16i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
4792 { X86ISD::VROTLI, MVT::v32i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
4793 { X86ISD::VROTLI, MVT::v64i8, { 1, 6, 1, 2 } }, // gf2p8affineqb
4794 };
4795 static const CostKindTblEntry GLMCostTbl[] = {
4796 { ISD::FSQRT, MVT::f32, { 19, 20, 1, 1 } }, // sqrtss
4797 { ISD::FSQRT, MVT::v4f32, { 37, 41, 1, 5 } }, // sqrtps
4798 { ISD::FSQRT, MVT::f64, { 34, 35, 1, 1 } }, // sqrtsd
4799 { ISD::FSQRT, MVT::v2f64, { 67, 71, 1, 5 } }, // sqrtpd
4800 };
4801 static const CostKindTblEntry SLMCostTbl[] = {
4802 { ISD::BSWAP, MVT::v2i64, { 5, 5, 1, 5 } },
4803 { ISD::BSWAP, MVT::v4i32, { 5, 5, 1, 5 } },
4804 { ISD::BSWAP, MVT::v8i16, { 5, 5, 1, 5 } },
4805 { ISD::FSQRT, MVT::f32, { 20, 20, 1, 1 } }, // sqrtss
4806 { ISD::FSQRT, MVT::v4f32, { 40, 41, 1, 5 } }, // sqrtps
4807 { ISD::FSQRT, MVT::f64, { 35, 35, 1, 1 } }, // sqrtsd
4808 { ISD::FSQRT, MVT::v2f64, { 70, 71, 1, 5 } }, // sqrtpd
4809 };
4810 static const CostKindTblEntry SSE42CostTbl[] = {
4811 { ISD::FMAXNUM, MVT::f32, { 5, 5, 7, 7 } }, // MAXSS + CMPUNORDSS + BLENDVPS
4812 { ISD::FMAXNUM, MVT::v4f32, { 4, 4, 4, 5 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4813 { ISD::FMAXNUM, MVT::f64, { 5, 5, 7, 7 } }, // MAXSD + CMPUNORDSD + BLENDVPD
4814 { ISD::FMAXNUM, MVT::v2f64, { 4, 4, 4, 5 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4815 { ISD::FSQRT, MVT::f32, { 18, 18, 1, 1 } }, // Nehalem from http://www.agner.org/
4816 { ISD::FSQRT, MVT::v4f32, { 18, 18, 1, 1 } }, // Nehalem from http://www.agner.org/
4817 };
4818 static const CostKindTblEntry SSE41CostTbl[] = {
4819 { ISD::ABS, MVT::v2i64, { 3, 4, 3, 5 } }, // BLENDVPD(X,PSUBQ(0,X),X)
4820 { ISD::MASK_BEFOREFIRST, MVT::v2i64, { 8, 11, 9, 11 } }, // movmsk seq
4821 { ISD::MULHS, MVT::v4i32, { 3, 9, 6, 7 } },
4822 { ISD::MULHU, MVT::v4i32, { 3, 9, 6, 7 } },
4823 { ISD::SADDSAT, MVT::v2i64, { 10, 14, 17, 21 } },
4824 { ISD::SADDSAT, MVT::v4i32, { 5, 11, 8, 10 } },
4825 { ISD::SSUBSAT, MVT::v2i64, { 12, 19, 25, 29 } },
4826 { ISD::SSUBSAT, MVT::v4i32, { 6, 14, 10, 12 } },
4827 { ISD::SMAX, MVT::v2i64, { 3, 7, 2, 3 } },
4828 { ISD::SMAX, MVT::v4i32, { 1, 1, 1, 1 } },
4829 { ISD::SMAX, MVT::v16i8, { 1, 1, 1, 1 } },
4830 { ISD::SMIN, MVT::v2i64, { 3, 7, 2, 3 } },
4831 { ISD::SMIN, MVT::v4i32, { 1, 1, 1, 1 } },
4832 { ISD::SMIN, MVT::v16i8, { 1, 1, 1, 1 } },
4833 { ISD::SMULO, MVT::v2i64, { 9, 11, 13, 17 } },
4834 { ISD::SMULO, MVT::v4i32, { 20, 24, 13, 19 } },
4835 { ISD::SMULO, MVT::v8i16, { 5, 9, 8, 8 } },
4836 { ISD::SMULO, MVT::v16i8, { 13, 22, 24, 25 } },
4837 { ISD::UADDSAT, MVT::v2i64, { 6, 13, 14, 14 } },
4838 { ISD::UADDSAT, MVT::v4i32, { 2, 2, 4, 4 } },
4839 { ISD::USUBSAT, MVT::v2i64, { 6, 10, 14, 14 } },
4840 { ISD::USUBSAT, MVT::v4i32, { 1, 2, 2, 2 } },
4841 { ISD::UMAX, MVT::v2i64, { 2, 11, 6, 7 } },
4842 { ISD::UMAX, MVT::v4i32, { 1, 1, 1, 1 } },
4843 { ISD::UMAX, MVT::v8i16, { 1, 1, 1, 1 } },
4844 { ISD::UMIN, MVT::v2i64, { 2, 11, 6, 7 } },
4845 { ISD::UMIN, MVT::v4i32, { 1, 1, 1, 1 } },
4846 { ISD::UMIN, MVT::v8i16, { 1, 1, 1, 1 } },
4847 { ISD::UMULO, MVT::v2i64, { 14, 20, 15, 20 } },
4848 { ISD::UMULO, MVT::v4i32, { 19, 22, 12, 18 } },
4849 { ISD::UMULO, MVT::v8i16, { 4, 9, 7, 7 } },
4850 { ISD::UMULO, MVT::v16i8, { 13, 19, 18, 20 } },
4851 };
4852 static const CostKindTblEntry SSSE3CostTbl[] = {
4853 { ISD::ABS, MVT::v4i32, { 1, 2, 1, 1 } },
4854 { ISD::ABS, MVT::v8i16, { 1, 2, 1, 1 } },
4855 { ISD::ABS, MVT::v16i8, { 1, 2, 1, 1 } },
4856 { ISD::BITREVERSE, MVT::v2i64, { 16, 20, 11, 21 } },
4857 { ISD::BITREVERSE, MVT::v4i32, { 16, 20, 11, 21 } },
4858 { ISD::BITREVERSE, MVT::v8i16, { 16, 20, 11, 21 } },
4859 { ISD::BITREVERSE, MVT::v16i8, { 11, 12, 10, 16 } },
4860 { ISD::BSWAP, MVT::v2i64, { 2, 3, 1, 5 } },
4861 { ISD::BSWAP, MVT::v4i32, { 2, 3, 1, 5 } },
4862 { ISD::BSWAP, MVT::v8i16, { 2, 3, 1, 5 } },
4863 { ISD::CTLZ, MVT::v2i64, { 18, 28, 28, 35 } },
4864 { ISD::CTLZ, MVT::v4i32, { 15, 20, 22, 28 } },
4865 { ISD::CTLZ, MVT::v8i16, { 13, 17, 16, 22 } },
4866 { ISD::CTLZ, MVT::v16i8, { 11, 15, 10, 16 } },
4867 { ISD::CTPOP, MVT::v2i64, { 13, 19, 12, 18 } },
4868 { ISD::CTPOP, MVT::v4i32, { 18, 24, 16, 22 } },
4869 { ISD::CTPOP, MVT::v8i16, { 13, 18, 14, 20 } },
4870 { ISD::CTPOP, MVT::v16i8, { 11, 12, 10, 16 } },
4871 { ISD::CTTZ, MVT::v2i64, { 13, 25, 15, 22 } },
4872 { ISD::CTTZ, MVT::v4i32, { 18, 26, 19, 25 } },
4873 { ISD::CTTZ, MVT::v8i16, { 13, 20, 17, 23 } },
4874 { ISD::CTTZ, MVT::v16i8, { 11, 16, 13, 19 } },
4875 { ISD::MASK_BEFOREFIRST, MVT::v16i8, { 8, 11, 9, 11 } }, // movmsk seq
4876 };
4877 static const CostKindTblEntry SSE2CostTbl[] = {
4878 { ISD::ABS, MVT::v2i64, { 3, 6, 5, 5 } },
4879 { ISD::ABS, MVT::v4i32, { 1, 4, 4, 4 } },
4880 { ISD::ABS, MVT::v8i16, { 1, 2, 3, 3 } },
4881 { ISD::ABS, MVT::v16i8, { 1, 2, 3, 3 } },
4882 { ISD::BITREVERSE, MVT::v2i64, { 16, 20, 32, 32 } },
4883 { ISD::BITREVERSE, MVT::v4i32, { 16, 20, 30, 30 } },
4884 { ISD::BITREVERSE, MVT::v8i16, { 16, 20, 25, 25 } },
4885 { ISD::BITREVERSE, MVT::v16i8, { 11, 12, 21, 21 } },
4886 { ISD::BSWAP, MVT::v2i64, { 5, 6, 11, 11 } },
4887 { ISD::BSWAP, MVT::v4i32, { 5, 5, 9, 9 } },
4888 { ISD::BSWAP, MVT::v8i16, { 5, 5, 4, 5 } },
4889 { ISD::CTLZ, MVT::v2i64, { 10, 45, 36, 38 } },
4890 { ISD::CTLZ, MVT::v4i32, { 10, 45, 38, 40 } },
4891 { ISD::CTLZ, MVT::v8i16, { 9, 38, 32, 34 } },
4892 { ISD::CTLZ, MVT::v16i8, { 8, 39, 29, 32 } },
4893 { ISD::CTPOP, MVT::v2i64, { 12, 26, 16, 18 } },
4894 { ISD::CTPOP, MVT::v4i32, { 15, 29, 21, 23 } },
4895 { ISD::CTPOP, MVT::v8i16, { 13, 25, 18, 20 } },
4896 { ISD::CTPOP, MVT::v16i8, { 10, 21, 14, 16 } },
4897 { ISD::CTTZ, MVT::v2i64, { 14, 28, 19, 21 } },
4898 { ISD::CTTZ, MVT::v4i32, { 18, 31, 24, 26 } },
4899 { ISD::CTTZ, MVT::v8i16, { 16, 27, 21, 23 } },
4900 { ISD::CTTZ, MVT::v16i8, { 13, 23, 17, 19 } },
4901 { ISD::MASK_BEFOREFIRST, MVT::v2i64, { 10, 13, 11, 13 } }, // movmsk seq
4902 { ISD::MASK_BEFOREFIRST, MVT::v4i32, { 8, 11, 9, 11 } }, // movmsk seq
4903 { ISD::MASK_BEFOREFIRST, MVT::v8i16, { 10, 13, 11, 13 } }, // movmsk seq
4904 { ISD::MASK_BEFOREFIRST, MVT::v16i8, { 10, 13, 11, 13 } }, // movmsk seq
4905 { ISD::MULHS, MVT::v4i32, { 5, 11, 15, 15 } },
4906 { ISD::MULHS, MVT::v8i16, { 1, 5, 1, 1 } },
4907 { ISD::MULHU, MVT::v4i32, { 3, 9, 7, 7 } },
4908 { ISD::MULHU, MVT::v8i16, { 1, 5, 1, 1 } },
4909 { ISD::SADDSAT, MVT::v2i64, { 12, 14, 24, 24 } },
4910 { ISD::SADDSAT, MVT::v4i32, { 6, 11, 11, 12 } },
4911 { ISD::SADDSAT, MVT::v8i16, { 1, 2, 1, 1 } },
4912 { ISD::SADDSAT, MVT::v16i8, { 1, 2, 1, 1 } },
4913 { ISD::SMAX, MVT::v2i64, { 4, 8, 15, 15 } },
4914 { ISD::SMAX, MVT::v4i32, { 2, 4, 5, 5 } },
4915 { ISD::SMAX, MVT::v8i16, { 1, 1, 1, 1 } },
4916 { ISD::SMAX, MVT::v16i8, { 2, 4, 5, 5 } },
4917 { ISD::SMIN, MVT::v2i64, { 4, 8, 15, 15 } },
4918 { ISD::SMIN, MVT::v4i32, { 2, 4, 5, 5 } },
4919 { ISD::SMIN, MVT::v8i16, { 1, 1, 1, 1 } },
4920 { ISD::SMIN, MVT::v16i8, { 2, 4, 5, 5 } },
4921 { ISD::SMULO, MVT::v2i64, { 30, 33, 13, 23 } },
4922 { ISD::SMULO, MVT::v4i32, { 20, 24, 23, 23 } },
4923 { ISD::SMULO, MVT::v8i16, { 5, 10, 8, 8 } },
4924 { ISD::SMULO, MVT::v16i8, { 13, 23, 24, 25 } },
4925 { ISD::SSUBSAT, MVT::v2i64, { 16, 19, 31, 31 } },
4926 { ISD::SSUBSAT, MVT::v4i32, { 6, 14, 12, 13 } },
4927 { ISD::SSUBSAT, MVT::v8i16, { 1, 2, 1, 1 } },
4928 { ISD::SSUBSAT, MVT::v16i8, { 1, 2, 1, 1 } },
4929 { ISD::UADDSAT, MVT::v2i64, { 7, 13, 14, 14 } },
4930 { ISD::UADDSAT, MVT::v4i32, { 4, 5, 7, 7 } },
4931 { ISD::UADDSAT, MVT::v8i16, { 1, 2, 1, 1 } },
4932 { ISD::UADDSAT, MVT::v16i8, { 1, 2, 1, 1 } },
4933 { ISD::UMAX, MVT::v2i64, { 4, 8, 15, 15 } },
4934 { ISD::UMAX, MVT::v4i32, { 2, 5, 8, 8 } },
4935 { ISD::UMAX, MVT::v8i16, { 1, 3, 3, 3 } },
4936 { ISD::UMAX, MVT::v16i8, { 1, 1, 1, 1 } },
4937 { ISD::UMIN, MVT::v2i64, { 4, 8, 15, 15 } },
4938 { ISD::UMIN, MVT::v4i32, { 2, 5, 8, 8 } },
4939 { ISD::UMIN, MVT::v8i16, { 1, 3, 3, 3 } },
4940 { ISD::UMIN, MVT::v16i8, { 1, 1, 1, 1 } },
4941 { ISD::UMULO, MVT::v2i64, { 30, 33, 15, 29 } },
4942 { ISD::UMULO, MVT::v4i32, { 19, 22, 14, 18 } },
4943 { ISD::UMULO, MVT::v8i16, { 4, 9, 7, 7 } },
4944 { ISD::UMULO, MVT::v16i8, { 13, 19, 20, 20 } },
4945 { ISD::USUBSAT, MVT::v2i64, { 7, 10, 14, 14 } },
4946 { ISD::USUBSAT, MVT::v4i32, { 4, 4, 7, 7 } },
4947 { ISD::USUBSAT, MVT::v8i16, { 1, 2, 1, 1 } },
4948 { ISD::USUBSAT, MVT::v16i8, { 1, 2, 1, 1 } },
4949 { ISD::FMAXNUM, MVT::f64, { 5, 5, 7, 7 } },
4950 { ISD::FMAXNUM, MVT::v2f64, { 4, 6, 6, 6 } },
4951 { ISD::FSQRT, MVT::f64, { 32, 32, 1, 1 } }, // Nehalem from http://www.agner.org/
4952 { ISD::FSQRT, MVT::v2f64, { 32, 32, 1, 1 } }, // Nehalem from http://www.agner.org/
4953 };
4954 static const CostKindTblEntry SSE1CostTbl[] = {
4955 { ISD::FMAXNUM, MVT::f32, { 5, 5, 7, 7 } },
4956 { ISD::FMAXNUM, MVT::v4f32, { 4, 6, 6, 6 } },
4957 { ISD::FSQRT, MVT::f32, { 28, 30, 1, 2 } }, // Pentium III from http://www.agner.org/
4958 { ISD::FSQRT, MVT::v4f32, { 56, 56, 1, 2 } }, // Pentium III from http://www.agner.org/
4959 };
4960 static const CostKindTblEntry BMI64CostTbl[] = { // 64-bit targets
4961 { ISD::CTTZ, MVT::i64, { 1, 1, 1, 1 } },
4962 };
4963 static const CostKindTblEntry BMI32CostTbl[] = { // 32 or 64-bit targets
4964 { ISD::CTTZ, MVT::i32, { 1, 1, 1, 1 } },
4965 { ISD::CTTZ, MVT::i16, { 2, 1, 1, 1 } },
4966 { ISD::CTTZ, MVT::i8, { 2, 1, 1, 1 } },
4967 };
4968 static const CostKindTblEntry LZCNT64CostTbl[] = { // 64-bit targets
4969 { ISD::CTLZ, MVT::i64, { 1, 1, 1, 1 } },
4970 };
4971 static const CostKindTblEntry LZCNT32CostTbl[] = { // 32 or 64-bit targets
4972 { ISD::CTLZ, MVT::i32, { 1, 1, 1, 1 } },
4973 { ISD::CTLZ, MVT::i16, { 2, 1, 1, 1 } },
4974 { ISD::CTLZ, MVT::i8, { 2, 1, 1, 1 } },
4975 };
4976 static const CostKindTblEntry POPCNT64CostTbl[] = { // 64-bit targets
4977 { ISD::CTPOP, MVT::i64, { 1, 1, 1, 1 } }, // popcnt
4978 };
4979 static const CostKindTblEntry POPCNT32CostTbl[] = { // 32 or 64-bit targets
4980 { ISD::CTPOP, MVT::i32, { 1, 1, 1, 1 } }, // popcnt
4981 { ISD::CTPOP, MVT::i16, { 1, 1, 2, 2 } }, // popcnt(zext())
4982 { ISD::CTPOP, MVT::i8, { 1, 1, 2, 2 } }, // popcnt(zext())
4983 };
4984 static const CostKindTblEntry PCLMULCostTbl[] = {
4985 { ISD::CLMUL, MVT::v2i64, { 3, 12, 4, 8 } }, // MOV+2xPCLMUL+unpack
4986 { ISD::CLMUL, MVT::v4i32, { 8, 18, 12, 16 } }, // MOV+4xPCLMUL+unpack
4987 { ISD::CLMUL, MVT::i64, { 3, 12, 4, 8 } }, // MOV+PCLMUL+MOV
4988 { ISD::CLMUL, MVT::i32, { 3, 12, 4, 8 } }, // MOV+PCLMUL+MOV
4989 { ISD::CLMUL, MVT::i16, { 3, 12, 4, 8 } }, // MOV+PCLMUL+MOV
4990 { ISD::CLMUL, MVT::i8, { 3, 12, 4, 8 } }, // MOV+PCLMUL+MOV
4991 };
4992 static const CostKindTblEntry X64CostTbl[] = { // 64-bit targets
4993 { ISD::ABS, MVT::i64, { 1, 2, 3, 3 } }, // SUB+CMOV
4994 { ISD::BITREVERSE, MVT::i64, { 10, 12, 20, 22 } },
4995 { ISD::BSWAP, MVT::i64, { 1, 2, 1, 2 } },
4996 { ISD::CTLZ, MVT::i64, { 1, 2, 3, 3 } }, // MOV+BSR+XOR
4997 { ISD::CTLZ, MVT::i32, { 1, 2, 3, 3 } }, // MOV+BSR+XOR
4998 { ISD::CTLZ, MVT::i16, { 2, 2, 3, 3 } }, // MOV+BSR+XOR
4999 { ISD::CTLZ, MVT::i8, { 2, 2, 4, 3 } }, // MOV+BSR+XOR
5000 { ISD::CTLZ_ZERO_POISON,MVT::i64,{ 1, 2, 2, 2 } }, // BSR+XOR
5001 { ISD::CTTZ, MVT::i64, { 1, 2, 2, 2 } }, // MOV+BSF
5002 { ISD::CTTZ, MVT::i32, { 1, 2, 2, 2 } }, // MOV+BSF
5003 { ISD::CTTZ, MVT::i16, { 2, 2, 2, 2 } }, // MOV+BSF
5004 { ISD::CTTZ, MVT::i8, { 2, 2, 2, 2 } }, // MOV+BSF
5005 { ISD::CTTZ_ZERO_POISON,MVT::i64,{ 1, 2, 1, 2 } }, // BSF
5006 { ISD::CTPOP, MVT::i64, { 10, 6, 19, 19 } },
5007 { ISD::ROTL, MVT::i64, { 2, 3, 1, 3 } },
5008 { ISD::ROTR, MVT::i64, { 2, 3, 1, 3 } },
5009 { X86ISD::VROTLI, MVT::i64, { 1, 1, 1, 1 } },
5010 { ISD::FSHL, MVT::i64, { 4, 4, 1, 4 } },
5011 { ISD::SADDSAT, MVT::i64, { 4, 4, 7, 10 } },
5012 { ISD::SSUBSAT, MVT::i64, { 4, 5, 8, 11 } },
5013 { ISD::UADDSAT, MVT::i64, { 2, 3, 4, 7 } },
5014 { ISD::USUBSAT, MVT::i64, { 2, 3, 4, 7 } },
5015 { ISD::SMAX, MVT::i64, { 1, 3, 2, 3 } },
5016 { ISD::SMIN, MVT::i64, { 1, 3, 2, 3 } },
5017 { ISD::UMAX, MVT::i64, { 1, 3, 2, 3 } },
5018 { ISD::UMIN, MVT::i64, { 1, 3, 2, 3 } },
5019 { ISD::SADDO, MVT::i64, { 2, 2, 4, 6 } },
5020 { ISD::UADDO, MVT::i64, { 2, 2, 4, 6 } },
5021 { ISD::SMULO, MVT::i64, { 4, 4, 4, 6 } },
5022 { ISD::UMULO, MVT::i64, { 8, 8, 4, 7 } },
5023 };
5024 static const CostKindTblEntry X86CostTbl[] = { // 32 or 64-bit targets
5025 { ISD::ABS, MVT::i32, { 1, 2, 3, 3 } }, // SUB+XOR+SRA or SUB+CMOV
5026 { ISD::ABS, MVT::i16, { 2, 2, 3, 3 } }, // SUB+XOR+SRA or SUB+CMOV
5027 { ISD::ABS, MVT::i8, { 2, 4, 4, 3 } }, // SUB+XOR+SRA
5028 { ISD::BITREVERSE, MVT::i32, { 9, 12, 17, 19 } },
5029 { ISD::BITREVERSE, MVT::i16, { 9, 12, 17, 19 } },
5030 { ISD::BITREVERSE, MVT::i8, { 7, 9, 13, 14 } },
5031 { ISD::BSWAP, MVT::i32, { 1, 1, 1, 1 } },
5032 { ISD::BSWAP, MVT::i16, { 1, 2, 1, 2 } }, // ROL
5033 { ISD::CTLZ, MVT::i32, { 2, 2, 4, 5 } }, // BSR+XOR or BSR+XOR+CMOV
5034 { ISD::CTLZ, MVT::i16, { 2, 2, 4, 5 } }, // BSR+XOR or BSR+XOR+CMOV
5035 { ISD::CTLZ, MVT::i8, { 2, 2, 5, 6 } }, // BSR+XOR or BSR+XOR+CMOV
5036 { ISD::CTLZ_ZERO_POISON,MVT::i32,{ 1, 2, 2, 2 } }, // BSR+XOR
5037 { ISD::CTLZ_ZERO_POISON,MVT::i16,{ 2, 2, 2, 2 } }, // BSR+XOR
5038 { ISD::CTLZ_ZERO_POISON,MVT::i8, { 2, 2, 3, 3 } }, // BSR+XOR
5039 { ISD::CTTZ, MVT::i32, { 2, 2, 3, 3 } }, // TEST+BSF+CMOV/BRANCH
5040 { ISD::CTTZ, MVT::i16, { 2, 2, 2, 3 } }, // TEST+BSF+CMOV/BRANCH
5041 { ISD::CTTZ, MVT::i8, { 2, 2, 2, 3 } }, // TEST+BSF+CMOV/BRANCH
5042 { ISD::CTTZ_ZERO_POISON,MVT::i32,{ 1, 2, 1, 2 } }, // BSF
5043 { ISD::CTTZ_ZERO_POISON,MVT::i16,{ 2, 2, 1, 2 } }, // BSF
5044 { ISD::CTTZ_ZERO_POISON,MVT::i8, { 2, 2, 1, 2 } }, // BSF
5045 { ISD::CTPOP, MVT::i32, { 8, 7, 15, 15 } },
5046 { ISD::CTPOP, MVT::i16, { 9, 8, 17, 17 } },
5047 { ISD::CTPOP, MVT::i8, { 7, 6, 6, 6 } },
5048 { ISD::ROTL, MVT::i32, { 2, 3, 1, 3 } },
5049 { ISD::ROTL, MVT::i16, { 2, 3, 1, 3 } },
5050 { ISD::ROTL, MVT::i8, { 2, 3, 1, 3 } },
5051 { ISD::ROTR, MVT::i32, { 2, 3, 1, 3 } },
5052 { ISD::ROTR, MVT::i16, { 2, 3, 1, 3 } },
5053 { ISD::ROTR, MVT::i8, { 2, 3, 1, 3 } },
5054 { X86ISD::VROTLI, MVT::i32, { 1, 1, 1, 1 } },
5055 { X86ISD::VROTLI, MVT::i16, { 1, 1, 1, 1 } },
5056 { X86ISD::VROTLI, MVT::i8, { 1, 1, 1, 1 } },
5057 { ISD::FSHL, MVT::i32, { 4, 4, 1, 4 } },
5058 { ISD::FSHL, MVT::i16, { 4, 4, 2, 5 } },
5059 { ISD::FSHL, MVT::i8, { 4, 4, 2, 5 } },
5060 { ISD::SADDSAT, MVT::i32, { 3, 4, 6, 9 } },
5061 { ISD::SADDSAT, MVT::i16, { 4, 4, 7, 10 } },
5062 { ISD::SADDSAT, MVT::i8, { 4, 5, 8, 11 } },
5063 { ISD::SSUBSAT, MVT::i32, { 4, 4, 7, 10 } },
5064 { ISD::SSUBSAT, MVT::i16, { 4, 4, 7, 10 } },
5065 { ISD::SSUBSAT, MVT::i8, { 4, 5, 8, 11 } },
5066 { ISD::UADDSAT, MVT::i32, { 2, 3, 4, 7 } },
5067 { ISD::UADDSAT, MVT::i16, { 2, 3, 4, 7 } },
5068 { ISD::UADDSAT, MVT::i8, { 3, 3, 5, 8 } },
5069 { ISD::USUBSAT, MVT::i32, { 2, 3, 4, 7 } },
5070 { ISD::USUBSAT, MVT::i16, { 2, 3, 4, 7 } },
5071 { ISD::USUBSAT, MVT::i8, { 3, 3, 5, 8 } },
5072 { ISD::SMAX, MVT::i32, { 1, 2, 2, 3 } },
5073 { ISD::SMAX, MVT::i16, { 1, 4, 2, 4 } },
5074 { ISD::SMAX, MVT::i8, { 1, 4, 2, 4 } },
5075 { ISD::SMIN, MVT::i32, { 1, 2, 2, 3 } },
5076 { ISD::SMIN, MVT::i16, { 1, 4, 2, 4 } },
5077 { ISD::SMIN, MVT::i8, { 1, 4, 2, 4 } },
5078 { ISD::UMAX, MVT::i32, { 1, 2, 2, 3 } },
5079 { ISD::UMAX, MVT::i16, { 1, 4, 2, 4 } },
5080 { ISD::UMAX, MVT::i8, { 1, 4, 2, 4 } },
5081 { ISD::UMIN, MVT::i32, { 1, 2, 2, 3 } },
5082 { ISD::UMIN, MVT::i16, { 1, 4, 2, 4 } },
5083 { ISD::UMIN, MVT::i8, { 1, 4, 2, 4 } },
5084 { ISD::SADDO, MVT::i32, { 2, 2, 4, 6 } },
5085 { ISD::SADDO, MVT::i16, { 2, 2, 4, 6 } },
5086 { ISD::SADDO, MVT::i8, { 2, 2, 4, 6 } },
5087 { ISD::UADDO, MVT::i32, { 2, 2, 4, 6 } },
5088 { ISD::UADDO, MVT::i16, { 2, 2, 4, 6 } },
5089 { ISD::UADDO, MVT::i8, { 2, 2, 4, 6 } },
5090 { ISD::SMULO, MVT::i32, { 2, 2, 4, 6 } },
5091 { ISD::SMULO, MVT::i16, { 5, 5, 4, 6 } },
5092 { ISD::SMULO, MVT::i8, { 6, 6, 4, 6 } },
5093 { ISD::UMULO, MVT::i32, { 6, 6, 4, 8 } },
5094 { ISD::UMULO, MVT::i16, { 6, 6, 4, 9 } },
5095 { ISD::UMULO, MVT::i8, { 6, 6, 4, 6 } },
5096 };
5097
5098 Type *RetTy = ICA.getReturnType();
5099 Type *OpTy = RetTy;
5100 Intrinsic::ID IID = ICA.getID();
5101 unsigned ISD = ISD::DELETED_NODE;
5102 switch (IID) {
5103 default:
5104 break;
5105 case Intrinsic::abs:
5106 ISD = ISD::ABS;
5107 break;
5108 case Intrinsic::bitreverse:
5110 break;
5111 case Intrinsic::bswap:
5112 ISD = ISD::BSWAP;
5113 break;
5114 case Intrinsic::ctlz:
5115 ISD = ISD::CTLZ;
5116 break;
5117 case Intrinsic::ctpop:
5118 ISD = ISD::CTPOP;
5119 break;
5120 case Intrinsic::cttz:
5121 ISD = ISD::CTTZ;
5122 break;
5123 case Intrinsic::fshl:
5124 ISD = ISD::FSHL;
5125 if (!ICA.isTypeBasedOnly()) {
5126 const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
5127 if (Args[0] == Args[1]) {
5128 ISD = ISD::ROTL;
5129 // Handle uniform constant rotation amounts.
5130 // TODO: Handle funnel-shift cases.
5131 const APInt *Amt;
5132 if (Args[2] &&
5134 ISD = X86ISD::VROTLI;
5135 }
5136 }
5137 break;
5138 case Intrinsic::fshr:
5139 // FSHR has same costs so don't duplicate.
5140 ISD = ISD::FSHL;
5141 if (!ICA.isTypeBasedOnly()) {
5142 const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
5143 if (Args[0] == Args[1]) {
5144 ISD = ISD::ROTR;
5145 // Handle uniform constant rotation amount.
5146 // TODO: Handle funnel-shift cases.
5147 const APInt *Amt;
5148 if (Args[2] &&
5150 ISD = X86ISD::VROTLI;
5151 }
5152 }
5153 break;
5154 case Intrinsic::lrint:
5155 case Intrinsic::llrint: {
5156 // X86 can use the CVTP2SI instructions to lower lrint/llrint calls, which
5157 // have the same costs as the CVTTP2SI (fptosi) instructions
5158 const SmallVectorImpl<Type *> &ArgTys = ICA.getArgTypes();
5159 return getCastInstrCost(Instruction::FPToSI, RetTy, ArgTys[0],
5160 TTI::CastContextHint::None, CostKind);
5161 }
5162 case Intrinsic::mask_beforefirst:
5164 break;
5165 case Intrinsic::maxnum:
5166 case Intrinsic::minnum:
5167 // FMINNUM has same costs so don't duplicate.
5168 ISD = ISD::FMAXNUM;
5169 break;
5170 case Intrinsic::sadd_sat:
5171 ISD = ISD::SADDSAT;
5172 break;
5173 case Intrinsic::smax:
5174 ISD = ISD::SMAX;
5175 break;
5176 case Intrinsic::smin:
5177 ISD = ISD::SMIN;
5178 break;
5179 case Intrinsic::smulh:
5180 ISD = ISD::MULHS;
5181 break;
5182 case Intrinsic::ssub_sat:
5183 ISD = ISD::SSUBSAT;
5184 break;
5185 case Intrinsic::uadd_sat:
5186 ISD = ISD::UADDSAT;
5187 break;
5188 case Intrinsic::umax:
5189 ISD = ISD::UMAX;
5190 break;
5191 case Intrinsic::umin:
5192 ISD = ISD::UMIN;
5193 break;
5194 case Intrinsic::usub_sat:
5195 ISD = ISD::USUBSAT;
5196 break;
5197 case Intrinsic::umulh:
5198 ISD = ISD::MULHU;
5199 break;
5200 case Intrinsic::sqrt:
5201 // The estimate sequence is costed with the division it is folded into.
5202 if (isFoldedIntoRsqrtEstimate(ICA.getInst(), *TLI,
5203 getTypeLegalizationCost(RetTy).second,
5204 CostKind))
5205 return TTI::TCC_Free;
5206 ISD = ISD::FSQRT;
5207 break;
5208 case Intrinsic::sadd_with_overflow:
5209 case Intrinsic::ssub_with_overflow:
5210 // SSUBO has same costs so don't duplicate.
5211 ISD = ISD::SADDO;
5212 OpTy = RetTy->getContainedType(0);
5213 break;
5214 case Intrinsic::uadd_with_overflow:
5215 case Intrinsic::usub_with_overflow:
5216 // USUBO has same costs so don't duplicate.
5217 ISD = ISD::UADDO;
5218 OpTy = RetTy->getContainedType(0);
5219 break;
5220 case Intrinsic::smul_with_overflow:
5221 ISD = ISD::SMULO;
5222 OpTy = RetTy->getContainedType(0);
5223 break;
5224 case Intrinsic::umul_with_overflow:
5225 ISD = ISD::UMULO;
5226 OpTy = RetTy->getContainedType(0);
5227 break;
5228 case Intrinsic::clmul:
5229 ISD = ISD::CLMUL;
5230 break;
5231 }
5232
5233 if (ISD != ISD::DELETED_NODE) {
5234 auto adjustTableCost = [&](int ISD, unsigned Cost,
5235 std::pair<InstructionCost, MVT> LT,
5237 InstructionCost LegalizationCost = LT.first;
5238 MVT MTy = LT.second;
5239
5240 // If there are no NANs to deal with, then these are reduced to a
5241 // single MIN** or MAX** instruction instead of the MIN/CMP/SELECT that we
5242 // assume is used in the non-fast case.
5243 if (ISD == ISD::FMAXNUM || ISD == ISD::FMINNUM) {
5244 if (FMF.noNaNs())
5245 return LegalizationCost * 1;
5246 }
5247
5248 // For cases where some ops can be folded into a load/store, assume free.
5249 if (MTy.isScalarInteger()) {
5250 if (ISD == ISD::BSWAP && ST->hasMOVBE() && ST->hasFastMOVBE()) {
5251 if (const Instruction *II = ICA.getInst()) {
5252 if (II->hasOneUse() && isa<StoreInst>(II->user_back()))
5253 return TTI::TCC_Free;
5254 if (auto *LI = dyn_cast<LoadInst>(II->getOperand(0))) {
5255 if (LI->hasOneUse())
5256 return TTI::TCC_Free;
5257 }
5258 }
5259 }
5260 }
5261
5262 return LegalizationCost * (int)Cost;
5263 };
5264
5265 // Legalize the type.
5266 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(OpTy);
5267 MVT MTy = LT.second;
5268
5269 // Without BMI/LZCNT see if we're only looking for a *_ZERO_POISON cost.
5270 if (((ISD == ISD::CTTZ && !ST->hasBMI()) ||
5271 (ISD == ISD::CTLZ && !ST->hasLZCNT())) &&
5272 !MTy.isVector() && !ICA.isTypeBasedOnly()) {
5273 const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
5274 if (auto *Cst = dyn_cast<ConstantInt>(Args[1]))
5275 if (Cst->isAllOnesValue())
5276 ISD =
5278 }
5279
5280 // FSQRT is a single instruction.
5281 if (ISD == ISD::FSQRT && CostKind == TTI::TCK_CodeSize)
5282 return LT.first;
5283
5284 if (ST->useGLMDivSqrtCosts())
5285 if (const auto *Entry = CostTableLookup(GLMCostTbl, ISD, MTy))
5286 if (auto KindCost = Entry->Cost[CostKind])
5287 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5288
5289 if (ST->useSLMArithCosts())
5290 if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
5291 if (auto KindCost = Entry->Cost[CostKind])
5292 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5293
5294 if (ST->hasVBMI2())
5295 if (const auto *Entry = CostTableLookup(AVX512VBMI2CostTbl, ISD, MTy))
5296 if (auto KindCost = Entry->Cost[CostKind])
5297 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5298
5299 if (ST->hasBITALG())
5300 if (const auto *Entry = CostTableLookup(AVX512BITALGCostTbl, ISD, MTy))
5301 if (auto KindCost = Entry->Cost[CostKind])
5302 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5303
5304 if (ST->hasVPOPCNTDQ())
5305 if (const auto *Entry = CostTableLookup(AVX512VPOPCNTDQCostTbl, ISD, MTy))
5306 if (auto KindCost = Entry->Cost[CostKind])
5307 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5308
5309 if (ST->hasGFNI())
5310 if (const auto *Entry = CostTableLookup(GFNICostTbl, ISD, MTy))
5311 if (auto KindCost = Entry->Cost[CostKind])
5312 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5313
5314 if (ST->hasCDI())
5315 if (const auto *Entry = CostTableLookup(AVX512CDCostTbl, ISD, MTy))
5316 if (auto KindCost = Entry->Cost[CostKind])
5317 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5318
5319 if (ST->hasBWI())
5320 if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
5321 if (auto KindCost = Entry->Cost[CostKind])
5322 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5323
5324 if (ST->hasAVX512())
5325 if (const auto *Entry = CostTableLookup(AVX512CostTbl, ISD, MTy))
5326 if (auto KindCost = Entry->Cost[CostKind])
5327 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5328
5329 if (ST->hasXOP())
5330 if (const auto *Entry = CostTableLookup(XOPCostTbl, ISD, MTy))
5331 if (auto KindCost = Entry->Cost[CostKind])
5332 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5333
5334 if (ST->hasAVX2())
5335 if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
5336 if (auto KindCost = Entry->Cost[CostKind])
5337 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5338
5339 if (ST->hasAVX())
5340 if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
5341 if (auto KindCost = Entry->Cost[CostKind])
5342 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5343
5344 if (ST->hasSSE42())
5345 if (const auto *Entry = CostTableLookup(SSE42CostTbl, ISD, MTy))
5346 if (auto KindCost = Entry->Cost[CostKind])
5347 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5348
5349 if (ST->hasSSE41())
5350 if (const auto *Entry = CostTableLookup(SSE41CostTbl, ISD, MTy))
5351 if (auto KindCost = Entry->Cost[CostKind])
5352 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5353
5354 if (ST->hasSSSE3())
5355 if (const auto *Entry = CostTableLookup(SSSE3CostTbl, ISD, MTy))
5356 if (auto KindCost = Entry->Cost[CostKind])
5357 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5358
5359 if (ST->hasSSE2())
5360 if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
5361 if (auto KindCost = Entry->Cost[CostKind])
5362 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5363
5364 if (ST->hasSSE1())
5365 if (const auto *Entry = CostTableLookup(SSE1CostTbl, ISD, MTy))
5366 if (auto KindCost = Entry->Cost[CostKind])
5367 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5368
5369 if (ST->hasBMI()) {
5370 if (ST->is64Bit())
5371 if (const auto *Entry = CostTableLookup(BMI64CostTbl, ISD, MTy))
5372 if (auto KindCost = Entry->Cost[CostKind])
5373 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5374
5375 if (const auto *Entry = CostTableLookup(BMI32CostTbl, ISD, MTy))
5376 if (auto KindCost = Entry->Cost[CostKind])
5377 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5378 }
5379
5380 if (ST->hasLZCNT()) {
5381 if (ST->is64Bit())
5382 if (const auto *Entry = CostTableLookup(LZCNT64CostTbl, ISD, MTy))
5383 if (auto KindCost = Entry->Cost[CostKind])
5384 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5385
5386 if (const auto *Entry = CostTableLookup(LZCNT32CostTbl, ISD, MTy))
5387 if (auto KindCost = Entry->Cost[CostKind])
5388 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5389 }
5390
5391 if (ST->hasPOPCNT()) {
5392 if (ST->is64Bit())
5393 if (const auto *Entry = CostTableLookup(POPCNT64CostTbl, ISD, MTy))
5394 if (auto KindCost = Entry->Cost[CostKind])
5395 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5396
5397 if (const auto *Entry = CostTableLookup(POPCNT32CostTbl, ISD, MTy))
5398 if (auto KindCost = Entry->Cost[CostKind])
5399 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5400 }
5401
5402 // FIXME: PCLMUL w/ AVX/AVX512 and VPCLMULQDQ are not handled properly.
5403 if (ST->hasPCLMUL())
5404 if (const auto *Entry = CostTableLookup(PCLMULCostTbl, ISD, MTy))
5405 if (auto KindCost = Entry->Cost[CostKind])
5406 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5407
5408 if (ST->is64Bit())
5409 if (const auto *Entry = CostTableLookup(X64CostTbl, ISD, MTy))
5410 if (auto KindCost = Entry->Cost[CostKind])
5411 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5412
5413 if (const auto *Entry = CostTableLookup(X86CostTbl, ISD, MTy))
5414 if (auto KindCost = Entry->Cost[CostKind])
5415 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5416
5417 // Without arg data, we need to compute the expanded costs of custom lowered
5418 // intrinsics to prevent use of the (very low) default costs.
5419 if (ICA.isTypeBasedOnly() &&
5420 (IID == Intrinsic::fshl || IID == Intrinsic::fshr)) {
5421 Type *CondTy = RetTy->getWithNewBitWidth(1);
5423 Cost += getArithmeticInstrCost(BinaryOperator::Or, RetTy, CostKind);
5424 Cost += getArithmeticInstrCost(BinaryOperator::Sub, RetTy, CostKind);
5425 Cost += getArithmeticInstrCost(BinaryOperator::Shl, RetTy, CostKind);
5426 Cost += getArithmeticInstrCost(BinaryOperator::LShr, RetTy, CostKind);
5427 Cost += getArithmeticInstrCost(BinaryOperator::And, RetTy, CostKind);
5428 Cost += getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, CondTy,
5429 CmpInst::ICMP_EQ, CostKind);
5430 Cost += getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
5431 CmpInst::ICMP_EQ, CostKind);
5432 return Cost;
5433 }
5434 }
5435
5436 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
5437}
5438
5440 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
5441 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
5442 static const CostTblEntry SLMCostTbl[] = {
5443 { ISD::EXTRACT_VECTOR_ELT, MVT::i8, 4 },
5444 { ISD::EXTRACT_VECTOR_ELT, MVT::i16, 4 },
5445 { ISD::EXTRACT_VECTOR_ELT, MVT::i32, 4 },
5446 { ISD::EXTRACT_VECTOR_ELT, MVT::i64, 7 }
5447 };
5448
5449 assert(Val->isVectorTy() && "This must be a vector type");
5450 auto *VT = cast<VectorType>(Val);
5451 if (VT->isScalableTy())
5453
5454 Type *ScalarType = Val->getScalarType();
5455 InstructionCost RegisterFileMoveCost = 0;
5456
5457 // Non-immediate extraction/insertion can be handled as a sequence of
5458 // aliased loads+stores via the stack.
5459 if (Index == -1U && (Opcode == Instruction::ExtractElement ||
5460 Opcode == Instruction::InsertElement)) {
5461 // TODO: On some SSE41+ targets, we expand to cmp+splat+select patterns:
5462 // inselt N0, N1, N2 --> select (SplatN2 == {0,1,2...}) ? SplatN1 : N0.
5463
5464 // TODO: Move this to BasicTTIImpl.h? We'd need better gep + index handling.
5465 assert(isa<FixedVectorType>(Val) && "Fixed vector type expected");
5466 Align VecAlign = DL.getPrefTypeAlign(Val);
5467 Align SclAlign = DL.getPrefTypeAlign(ScalarType);
5468
5469 // Extract - store vector to stack, load scalar.
5470 if (Opcode == Instruction::ExtractElement) {
5471 return getMemoryOpCost(Instruction::Store, Val, VecAlign, 0, CostKind) +
5472 getMemoryOpCost(Instruction::Load, ScalarType, SclAlign, 0,
5473 CostKind);
5474 }
5475 // Insert - store vector to stack, store scalar, load vector.
5476 if (Opcode == Instruction::InsertElement) {
5477 return getMemoryOpCost(Instruction::Store, Val, VecAlign, 0, CostKind) +
5478 getMemoryOpCost(Instruction::Store, ScalarType, SclAlign, 0,
5479 CostKind) +
5480 getMemoryOpCost(Instruction::Load, Val, VecAlign, 0, CostKind);
5481 }
5482 }
5483
5484 if (Index != -1U && (Opcode == Instruction::ExtractElement ||
5485 Opcode == Instruction::InsertElement)) {
5486 // Extraction of vXi1 elements are now efficiently handled by MOVMSK.
5487 if (Opcode == Instruction::ExtractElement &&
5488 ScalarType->getScalarSizeInBits() == 1 &&
5489 cast<FixedVectorType>(Val)->getNumElements() > 1)
5490 return 1;
5491
5492 // Legalize the type.
5493 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Val);
5494
5495 // This type is legalized to a scalar type.
5496 if (!LT.second.isVector())
5497 return TTI::TCC_Free;
5498
5499 // The type may be split. Normalize the index to the new type.
5500 unsigned SizeInBits = LT.second.getSizeInBits();
5501 unsigned NumElts = LT.second.getVectorNumElements();
5502 unsigned SubNumElts = NumElts;
5503 Index = Index % NumElts;
5504
5505 // For >128-bit vectors, we need to extract higher 128-bit subvectors.
5506 // For inserts, we also need to insert the subvector back.
5507 if (SizeInBits > 128) {
5508 assert((SizeInBits % 128) == 0 && "Illegal vector");
5509 unsigned NumSubVecs = SizeInBits / 128;
5510 SubNumElts = NumElts / NumSubVecs;
5511 if (SubNumElts <= Index) {
5512 RegisterFileMoveCost += (Opcode == Instruction::InsertElement ? 2 : 1);
5513 Index %= SubNumElts;
5514 }
5515 }
5516
5517 MVT MScalarTy = LT.second.getScalarType();
5518 auto IsCheapPInsrPExtrInsertPS = [&]() {
5519 // Assume pinsr/pextr XMM <-> GPR is relatively cheap on all targets.
5520 // Inserting f32 into index0 is just movss.
5521 // Also, assume insertps is relatively cheap on all >= SSE41 targets.
5522 return (MScalarTy == MVT::i16 && ST->hasSSE2()) ||
5523 (MScalarTy.isInteger() && ST->hasSSE41()) ||
5524 (MScalarTy == MVT::f32 && ST->hasSSE1() && Index == 0 &&
5525 Opcode == Instruction::InsertElement) ||
5526 (MScalarTy == MVT::f32 && ST->hasSSE41() &&
5527 Opcode == Instruction::InsertElement);
5528 };
5529
5530 if (Index == 0) {
5531 // Floating point scalars are already located in index #0.
5532 // Many insertions to #0 can fold away for scalar fp-ops, so let's assume
5533 // true for all.
5534 if (ScalarType->isFloatingPointTy() &&
5535 (Opcode != Instruction::InsertElement || !Op0 ||
5536 isa<UndefValue>(Op0)))
5537 return RegisterFileMoveCost;
5538
5539 if (Opcode == Instruction::InsertElement &&
5541 // Consider the gather cost to be cheap.
5543 return RegisterFileMoveCost;
5544 if (!IsCheapPInsrPExtrInsertPS()) {
5545 // mov constant-to-GPR + movd/movq GPR -> XMM.
5546 if (isa_and_nonnull<Constant>(Op1) && Op1->getType()->isIntegerTy())
5547 return 2 + RegisterFileMoveCost;
5548 // Assume movd/movq GPR -> XMM is relatively cheap on all targets.
5549 return 1 + RegisterFileMoveCost;
5550 }
5551 }
5552
5553 // Assume movd/movq XMM -> GPR is relatively cheap on all targets.
5554 if (ScalarType->isIntegerTy() && Opcode == Instruction::ExtractElement)
5555 return 1 + RegisterFileMoveCost;
5556 }
5557
5558 int ISD = TLI->InstructionOpcodeToISD(Opcode);
5559 assert(ISD && "Unexpected vector opcode");
5560 if (ST->useSLMArithCosts())
5561 if (auto *Entry = CostTableLookup(SLMCostTbl, ISD, MScalarTy))
5562 return Entry->Cost + RegisterFileMoveCost;
5563
5564 // Consider cheap cases.
5565 if (IsCheapPInsrPExtrInsertPS())
5566 return 1 + RegisterFileMoveCost;
5567
5568 // For extractions we just need to shuffle the element to index 0, which
5569 // should be very cheap (assume cost = 1). For insertions we need to shuffle
5570 // the elements to its destination. In both cases we must handle the
5571 // subvector move(s).
5572 // If the vector type is already less than 128-bits then don't reduce it.
5573 // TODO: Under what circumstances should we shuffle using the full width?
5574 InstructionCost ShuffleCost = 1;
5575 if (Opcode == Instruction::InsertElement) {
5576 auto *SubTy = cast<VectorType>(Val);
5577 EVT VT = TLI->getValueType(DL, Val);
5578 if (VT.getScalarType() != MScalarTy || VT.getSizeInBits() >= 128)
5579 SubTy = FixedVectorType::get(ScalarType, SubNumElts);
5580 ShuffleCost = getShuffleCost(TTI::SK_PermuteTwoSrc, SubTy, SubTy,
5581 CostKind, {}, 0, SubTy);
5582 }
5583 int IntOrFpCost = ScalarType->isFloatingPointTy() ? 0 : 1;
5584 return ShuffleCost + IntOrFpCost + RegisterFileMoveCost;
5585 }
5586
5587 return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
5588 VIC) +
5589 RegisterFileMoveCost;
5590}
5591
5593 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
5594 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
5595 TTI::VectorInstrContext VIC) const {
5596 assert(DemandedElts.getBitWidth() ==
5597 cast<FixedVectorType>(Ty)->getNumElements() &&
5598 "Vector size mismatch");
5599
5600 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
5601 MVT MScalarTy = LT.second.getScalarType();
5602 unsigned LegalVectorBitWidth = LT.second.getSizeInBits();
5604 // The square roots, folded with the divisions into the estimate, are
5605 // computed separately to be inserted.
5606 if (Insert && !VL.empty())
5607 for (unsigned I : seq(DemandedElts.getBitWidth()))
5608 if (DemandedElts[I] &&
5609 isFoldedIntoRsqrtEstimate(VL[I], *TLI, MScalarTy, CostKind))
5611 IntrinsicCostAttributes(Intrinsic::sqrt, VL[I]->getType(),
5612 {VL[I]->getType()}),
5613 CostKind);
5614
5615 constexpr unsigned LaneBitWidth = 128;
5616 assert((LegalVectorBitWidth < LaneBitWidth ||
5617 (LegalVectorBitWidth % LaneBitWidth) == 0) &&
5618 "Illegal vector");
5619
5620 const int NumLegalVectors = LT.first.getValue();
5621 assert(NumLegalVectors >= 0 && "Negative cost!");
5622
5623 // For insertions, a ISD::BUILD_VECTOR style vector initialization can be much
5624 // cheaper than an accumulation of ISD::INSERT_VECTOR_ELT. SLPVectorizer has
5625 // a special heuristic regarding poison input which is passed here in
5626 // ForPoisonSrc.
5627 if (Insert && !ForPoisonSrc) {
5628 // This is nearly identical to BaseT::getScalarizationOverhead(), except
5629 // it is passing nullptr to getVectorInstrCost() for Op0 (instead of
5630 // Constant::getNullValue()), which makes the X86TTIImpl
5631 // getVectorInstrCost() return 0 instead of 1.
5632 for (unsigned I : seq(DemandedElts.getBitWidth())) {
5633 if (!DemandedElts[I])
5634 continue;
5635 Cost += getVectorInstrCost(Instruction::InsertElement, Ty, CostKind, I,
5637 VL.empty() ? nullptr : VL[I],
5639 }
5640 return Cost;
5641 }
5642
5643 if (Insert) {
5644 if ((MScalarTy == MVT::i16 && ST->hasSSE2()) ||
5645 (MScalarTy.isInteger() && ST->hasSSE41()) ||
5646 (MScalarTy == MVT::f32 && ST->hasSSE41())) {
5647 // For types we can insert directly, insertion into 128-bit sub vectors is
5648 // cheap, followed by a cheap chain of concatenations.
5649 if (LegalVectorBitWidth <= LaneBitWidth) {
5650 Cost += BaseT::getScalarizationOverhead(Ty, DemandedElts, Insert,
5651 /*Extract*/ false, CostKind);
5652 } else {
5653 // In each 128-lane, if at least one index is demanded but not all
5654 // indices are demanded and this 128-lane is not the first 128-lane of
5655 // the legalized-vector, then this 128-lane needs a extracti128; If in
5656 // each 128-lane, there is at least one demanded index, this 128-lane
5657 // needs a inserti128.
5658
5659 // The following cases will help you build a better understanding:
5660 // Assume we insert several elements into a v8i32 vector in avx2,
5661 // Case#1: inserting into 1th index needs vpinsrd + inserti128.
5662 // Case#2: inserting into 5th index needs extracti128 + vpinsrd +
5663 // inserti128.
5664 // Case#3: inserting into 4,5,6,7 index needs 4*vpinsrd + inserti128.
5665 assert((LegalVectorBitWidth % LaneBitWidth) == 0 && "Illegal vector");
5666 unsigned NumLegalLanes = LegalVectorBitWidth / LaneBitWidth;
5667 unsigned NumLanesTotal = NumLegalLanes * NumLegalVectors;
5668 unsigned NumLegalElts =
5669 LT.second.getVectorNumElements() * NumLegalVectors;
5670 assert(NumLegalElts >= DemandedElts.getBitWidth() &&
5671 "Vector has been legalized to smaller element count");
5672 assert((NumLegalElts % NumLanesTotal) == 0 &&
5673 "Unexpected elts per lane");
5674 unsigned NumEltsPerLane = NumLegalElts / NumLanesTotal;
5675
5676 APInt WidenedDemandedElts = DemandedElts.zext(NumLegalElts);
5677 auto *LaneTy =
5678 FixedVectorType::get(Ty->getElementType(), NumEltsPerLane);
5679
5680 for (unsigned I = 0; I != NumLanesTotal; ++I) {
5681 APInt LaneEltMask = WidenedDemandedElts.extractBits(
5682 NumEltsPerLane, NumEltsPerLane * I);
5683 if (LaneEltMask.isZero())
5684 continue;
5685 // FIXME: we don't need to extract if all non-demanded elements
5686 // are legalization-inserted padding.
5687 if (!LaneEltMask.isAllOnes())
5688 Cost += getShuffleCost(TTI::SK_ExtractSubvector, Ty, Ty, CostKind,
5689 {}, I * NumEltsPerLane, LaneTy);
5690 Cost += BaseT::getScalarizationOverhead(LaneTy, LaneEltMask, Insert,
5691 /*Extract*/ false, CostKind);
5692 }
5693
5694 APInt AffectedLanes =
5695 APIntOps::ScaleBitMask(WidenedDemandedElts, NumLanesTotal);
5696 APInt FullyAffectedLegalVectors = APIntOps::ScaleBitMask(
5697 AffectedLanes, NumLegalVectors, /*MatchAllBits=*/true);
5698 for (int LegalVec = 0; LegalVec != NumLegalVectors; ++LegalVec) {
5699 for (unsigned Lane = 0; Lane != NumLegalLanes; ++Lane) {
5700 unsigned I = NumLegalLanes * LegalVec + Lane;
5701 // No need to insert unaffected lane; or lane 0 of each legal vector
5702 // iff ALL lanes of that vector were affected and will be inserted.
5703 if (!AffectedLanes[I] ||
5704 (Lane == 0 && FullyAffectedLegalVectors[LegalVec]))
5705 continue;
5706 Cost += getShuffleCost(TTI::SK_InsertSubvector, Ty, Ty, CostKind,
5707 {}, I * NumEltsPerLane, LaneTy);
5708 }
5709 }
5710 }
5711 } else if (LT.second.isVector()) {
5712 // Without fast insertion, we need to use MOVD/MOVQ to pass each demanded
5713 // integer element as a SCALAR_TO_VECTOR, then we build the vector as a
5714 // series of UNPCK followed by CONCAT_VECTORS - all of these can be
5715 // considered cheap.
5716 if (Ty->isIntOrIntVectorTy())
5717 Cost += DemandedElts.popcount();
5718
5719 // Get the smaller of the legalized or original pow2-extended number of
5720 // vector elements, which represents the number of unpacks we'll end up
5721 // performing.
5722 unsigned NumElts = LT.second.getVectorNumElements();
5723 unsigned Pow2Elts =
5724 PowerOf2Ceil(cast<FixedVectorType>(Ty)->getNumElements());
5725 Cost += (std::min<unsigned>(NumElts, Pow2Elts) - 1) * LT.first;
5726 }
5727 }
5728
5729 if (Extract) {
5730 // vXi1 can be efficiently extracted with MOVMSK.
5731 // TODO: AVX512 predicate mask handling.
5732 // NOTE: This doesn't work well for roundtrip scalarization.
5733 if (!Insert && Ty->getScalarSizeInBits() == 1 && !ST->hasAVX512()) {
5734 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
5735 unsigned MaxElts = ST->hasAVX2() ? 32 : 16;
5736 unsigned MOVMSKCost = (NumElts + MaxElts - 1) / MaxElts;
5737 return MOVMSKCost;
5738 }
5739
5740 if (LT.second.isVector()) {
5741 unsigned NumLegalElts =
5742 LT.second.getVectorNumElements() * NumLegalVectors;
5743 assert(NumLegalElts >= DemandedElts.getBitWidth() &&
5744 "Vector has been legalized to smaller element count");
5745
5746 // If we're extracting elements from a 128-bit subvector lane,
5747 // we only need to extract each lane once, not for every element.
5748 if (LegalVectorBitWidth > LaneBitWidth) {
5749 unsigned NumLegalLanes = LegalVectorBitWidth / LaneBitWidth;
5750 unsigned NumLanesTotal = NumLegalLanes * NumLegalVectors;
5751 assert((NumLegalElts % NumLanesTotal) == 0 &&
5752 "Unexpected elts per lane");
5753 unsigned NumEltsPerLane = NumLegalElts / NumLanesTotal;
5754
5755 // Add cost for each demanded 128-bit subvector extraction.
5756 // Luckily this is a lot easier than for insertion.
5757 APInt WidenedDemandedElts = DemandedElts.zext(NumLegalElts);
5758 auto *LaneTy =
5759 FixedVectorType::get(Ty->getElementType(), NumEltsPerLane);
5760
5761 for (unsigned I = 0; I != NumLanesTotal; ++I) {
5762 APInt LaneEltMask = WidenedDemandedElts.extractBits(
5763 NumEltsPerLane, I * NumEltsPerLane);
5764 if (LaneEltMask.isZero())
5765 continue;
5766 Cost += getShuffleCost(TTI::SK_ExtractSubvector, Ty, Ty, CostKind, {},
5767 I * NumEltsPerLane, LaneTy);
5769 LaneTy, LaneEltMask, /*Insert*/ false, Extract, CostKind);
5770 }
5771
5772 return Cost;
5773 }
5774 }
5775
5776 // Fallback to default extraction.
5777 Cost += BaseT::getScalarizationOverhead(Ty, DemandedElts, /*Insert*/ false,
5778 Extract, CostKind);
5779 }
5780
5781 return Cost;
5782}
5783
5785X86TTIImpl::getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
5786 int VF, const APInt &DemandedDstElts,
5787 TTI::TargetCostKind CostKind) const {
5788 const unsigned EltTyBits = DL.getTypeSizeInBits(EltTy);
5789 // We don't differentiate element types here, only element bit width.
5790 EltTy = IntegerType::getIntNTy(EltTy->getContext(), EltTyBits);
5791
5792 auto bailout = [&]() {
5793 return BaseT::getReplicationShuffleCost(EltTy, ReplicationFactor, VF,
5794 DemandedDstElts, CostKind);
5795 };
5796
5797 // For now, only deal with AVX512 cases.
5798 if (!ST->hasAVX512())
5799 return bailout();
5800
5801 // Do we have a native shuffle for this element type, or should we promote?
5802 unsigned PromEltTyBits = EltTyBits;
5803 switch (EltTyBits) {
5804 case 32:
5805 case 64:
5806 break; // AVX512F.
5807 case 16:
5808 if (!ST->hasBWI())
5809 PromEltTyBits = 32; // promote to i32, AVX512F.
5810 break; // AVX512BW
5811 case 8:
5812 if (!ST->hasVBMI())
5813 PromEltTyBits = 32; // promote to i32, AVX512F.
5814 break; // AVX512VBMI
5815 case 1:
5816 // There is no support for shuffling i1 elements. We *must* promote.
5817 if (ST->hasBWI()) {
5818 if (ST->hasVBMI())
5819 PromEltTyBits = 8; // promote to i8, AVX512VBMI.
5820 else
5821 PromEltTyBits = 16; // promote to i16, AVX512BW.
5822 break;
5823 }
5824 PromEltTyBits = 32; // promote to i32, AVX512F.
5825 break;
5826 default:
5827 return bailout();
5828 }
5829 auto *PromEltTy = IntegerType::getIntNTy(EltTy->getContext(), PromEltTyBits);
5830
5831 auto *SrcVecTy = FixedVectorType::get(EltTy, VF);
5832 auto *PromSrcVecTy = FixedVectorType::get(PromEltTy, VF);
5833
5834 int NumDstElements = VF * ReplicationFactor;
5835 auto *PromDstVecTy = FixedVectorType::get(PromEltTy, NumDstElements);
5836 auto *DstVecTy = FixedVectorType::get(EltTy, NumDstElements);
5837
5838 // Legalize the types.
5839 MVT LegalSrcVecTy = getTypeLegalizationCost(SrcVecTy).second;
5840 MVT LegalPromSrcVecTy = getTypeLegalizationCost(PromSrcVecTy).second;
5841 MVT LegalPromDstVecTy = getTypeLegalizationCost(PromDstVecTy).second;
5842 MVT LegalDstVecTy = getTypeLegalizationCost(DstVecTy).second;
5843 // They should have legalized into vector types.
5844 if (!LegalSrcVecTy.isVector() || !LegalPromSrcVecTy.isVector() ||
5845 !LegalPromDstVecTy.isVector() || !LegalDstVecTy.isVector())
5846 return bailout();
5847
5848 if (PromEltTyBits != EltTyBits) {
5849 // If we have to perform the shuffle with wider elt type than our data type,
5850 // then we will first need to anyext (we don't care about the new bits)
5851 // the source elements, and then truncate Dst elements.
5852 InstructionCost PromotionCost;
5853 PromotionCost += getCastInstrCost(
5854 Instruction::SExt, /*Dst=*/PromSrcVecTy, /*Src=*/SrcVecTy,
5856 PromotionCost +=
5857 getCastInstrCost(Instruction::Trunc, /*Dst=*/DstVecTy,
5858 /*Src=*/PromDstVecTy,
5860 return PromotionCost + getReplicationShuffleCost(PromEltTy,
5861 ReplicationFactor, VF,
5862 DemandedDstElts, CostKind);
5863 }
5864
5865 assert(LegalSrcVecTy.getScalarSizeInBits() == EltTyBits &&
5866 LegalSrcVecTy.getScalarType() == LegalDstVecTy.getScalarType() &&
5867 "We expect that the legalization doesn't affect the element width, "
5868 "doesn't coalesce/split elements.");
5869
5870 unsigned NumEltsPerDstVec = LegalDstVecTy.getVectorNumElements();
5871 unsigned NumDstVectors =
5872 divideCeil(DstVecTy->getNumElements(), NumEltsPerDstVec);
5873
5874 auto *SingleDstVecTy = FixedVectorType::get(EltTy, NumEltsPerDstVec);
5875
5876 // Not all the produced Dst elements may be demanded. In our case,
5877 // given that a single Dst vector is formed by a single shuffle,
5878 // if all elements that will form a single Dst vector aren't demanded,
5879 // then we won't need to do that shuffle, so adjust the cost accordingly.
5880 APInt DemandedDstVectors = APIntOps::ScaleBitMask(
5881 DemandedDstElts.zext(NumDstVectors * NumEltsPerDstVec), NumDstVectors);
5882 unsigned NumDstVectorsDemanded = DemandedDstVectors.popcount();
5883
5884 InstructionCost SingleShuffleCost =
5885 getShuffleCost(TTI::SK_PermuteSingleSrc, SingleDstVecTy, SingleDstVecTy,
5886 CostKind, /*Mask=*/{},
5887 /*Index=*/0, /*SubTp=*/nullptr);
5888 return NumDstVectorsDemanded * SingleShuffleCost;
5889}
5890
5892 Align Alignment,
5893 unsigned AddressSpace,
5894 TTI::TargetCostKind CostKind,
5895 TTI::OperandValueInfo OpInfo,
5896 const Instruction *I) const {
5897 // FIXME: Load latency isn't handled here
5898 if (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency)
5899 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5900 CostKind, OpInfo, I);
5901
5902 // TODO: Handle other cost kinds.
5903 if (CostKind != TTI::TCK_RecipThroughput) {
5904 if (auto *SI = dyn_cast_or_null<StoreInst>(I)) {
5905 // Store instruction with index and scale costs 2 Uops.
5906 // Check the preceding GEP to identify non-const indices.
5907 if (auto *GEP = dyn_cast<GetElementPtrInst>(SI->getPointerOperand())) {
5908 if (!all_of(GEP->indices(), [](Value *V) { return isa<Constant>(V); }))
5909 return TTI::TCC_Basic * 2;
5910 }
5911 }
5912 return TTI::TCC_Basic;
5913 }
5914
5915 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
5916 "Invalid Opcode");
5917 // Type legalization can't handle structs
5918 if (TLI->getValueType(DL, Src, true) == MVT::Other)
5919 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5920 CostKind, OpInfo, I);
5921
5922 // Legalize the type.
5923 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Src);
5924
5925 auto *VTy = dyn_cast<FixedVectorType>(Src);
5926
5928
5929 // Add a cost for constant load to vector.
5930 if (Opcode == Instruction::Store && OpInfo.isConstant())
5931 Cost += getMemoryOpCost(Instruction::Load, Src, DL.getABITypeAlign(Src),
5932 /*AddressSpace=*/0, CostKind, OpInfo);
5933
5934 // Handle the simple case of non-vectors.
5935 // NOTE: this assumes that legalization never creates vector from scalars!
5936 if (!VTy || !LT.second.isVector()) {
5937 // Each load/store unit costs 1.
5938 return (LT.second.isFloatingPoint() ? Cost : 0) + LT.first * 1;
5939 }
5940
5941 bool IsLoad = Opcode == Instruction::Load;
5942
5943 Type *EltTy = VTy->getElementType();
5944
5945 const int EltTyBits = DL.getTypeSizeInBits(EltTy);
5946
5947 // Source of truth: how many elements were there in the original IR vector?
5948 const unsigned SrcNumElt = VTy->getNumElements();
5949
5950 // How far have we gotten?
5951 int NumEltRemaining = SrcNumElt;
5952 // Note that we intentionally capture by-reference, NumEltRemaining changes.
5953 auto NumEltDone = [&]() { return SrcNumElt - NumEltRemaining; };
5954
5955 const int MaxLegalOpSizeBytes = divideCeil(LT.second.getSizeInBits(), 8);
5956
5957 // Note that even if we can store 64 bits of an XMM, we still operate on XMM.
5958 const unsigned XMMBits = 128;
5959 if (XMMBits % EltTyBits != 0)
5960 // Vector size must be a multiple of the element size. I.e. no padding.
5961 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5962 CostKind, OpInfo, I);
5963 const int NumEltPerXMM = XMMBits / EltTyBits;
5964
5965 auto *XMMVecTy = FixedVectorType::get(EltTy, NumEltPerXMM);
5966
5967 for (int CurrOpSizeBytes = MaxLegalOpSizeBytes, SubVecEltsLeft = 0;
5968 NumEltRemaining > 0; CurrOpSizeBytes /= 2) {
5969 // How many elements would a single op deal with at once?
5970 if ((8 * CurrOpSizeBytes) % EltTyBits != 0)
5971 // Vector size must be a multiple of the element size. I.e. no padding.
5972 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5973 CostKind, OpInfo, I);
5974 int CurrNumEltPerOp = (8 * CurrOpSizeBytes) / EltTyBits;
5975
5976 assert(CurrOpSizeBytes > 0 && CurrNumEltPerOp > 0 && "How'd we get here?");
5977 assert((((NumEltRemaining * EltTyBits) < (2 * 8 * CurrOpSizeBytes)) ||
5978 (CurrOpSizeBytes == MaxLegalOpSizeBytes)) &&
5979 "Unless we haven't halved the op size yet, "
5980 "we have less than two op's sized units of work left.");
5981
5982 auto *CurrVecTy = CurrNumEltPerOp > NumEltPerXMM
5983 ? FixedVectorType::get(EltTy, CurrNumEltPerOp)
5984 : XMMVecTy;
5985
5986 assert(CurrVecTy->getNumElements() % CurrNumEltPerOp == 0 &&
5987 "After halving sizes, the vector elt count is no longer a multiple "
5988 "of number of elements per operation?");
5989 auto *CoalescedVecTy =
5990 CurrNumEltPerOp == 1
5991 ? CurrVecTy
5993 IntegerType::get(Src->getContext(),
5994 EltTyBits * CurrNumEltPerOp),
5995 CurrVecTy->getNumElements() / CurrNumEltPerOp);
5996 assert(DL.getTypeSizeInBits(CoalescedVecTy) ==
5997 DL.getTypeSizeInBits(CurrVecTy) &&
5998 "coalesciing elements doesn't change vector width.");
5999
6000 while (NumEltRemaining > 0) {
6001 assert(SubVecEltsLeft >= 0 && "Subreg element count overconsumtion?");
6002
6003 // Can we use this vector size, as per the remaining element count?
6004 // Iff the vector is naturally aligned, we can do a wide load regardless.
6005 if (NumEltRemaining < CurrNumEltPerOp &&
6006 (!IsLoad || Alignment < CurrOpSizeBytes) && CurrOpSizeBytes != 1)
6007 break; // Try smalled vector size.
6008
6009 // This isn't exactly right. We're using slow unaligned 32-byte accesses
6010 // as a proxy for a double-pumped AVX memory interface such as on
6011 // Sandybridge.
6012 // Sub-32-bit loads/stores will be slower either with PINSR*/PEXTR* or
6013 // will be scalarized.
6014 //
6015 // For a vector load, each non-0th 1/2/4-byte in-lane remainder chunk is
6016 // materialized by a *single* folded PINSR*(mem) that both loads and
6017 // inserts the lane (1B->PINSRB, 2B->PINSRW, 4B->PINSRD; the byte and
6018 // dword folds need SSE4.1, the word fold only SSE2).
6019 // When that fold is available the chunk is one instruction, so it must be
6020 // priced once here (as a plain load) and the separate lane-insert charge
6021 // below must be skipped - otherwise the folded insert is double-counted.
6022 // Stores (the symmetric PEXTR*(mem) fold) are left unchanged here.
6023 bool Is0thSubVec = (NumEltDone() % LT.second.getVectorNumElements()) == 0;
6024 bool FoldedInLaneInsert = IsLoad && !Is0thSubVec &&
6025 ((CurrOpSizeBytes == 1 && ST->hasSSE41()) ||
6026 (CurrOpSizeBytes == 2 && ST->hasSSE2()) ||
6027 (CurrOpSizeBytes == 4 && ST->hasSSE41()));
6028 if (CurrOpSizeBytes == 32 && ST->isUnalignedMem32Slow())
6029 Cost += 2;
6030 else if (CurrOpSizeBytes < 4 && !FoldedInLaneInsert)
6031 Cost += 2;
6032 else
6033 Cost += 1;
6034
6035 // If we're loading a uniform value, then we don't need to split the load,
6036 // loading just a single (widest) vector can be reused by all splits.
6037 if (IsLoad && OpInfo.isUniform())
6038 return Cost;
6039
6040 // If we have fully processed the previous reg, we need to replenish it.
6041 if (SubVecEltsLeft == 0) {
6042 SubVecEltsLeft += CurrVecTy->getNumElements();
6043 // And that's free only for the 0'th subvector of a legalized vector.
6044 if (!Is0thSubVec)
6045 Cost +=
6048 VTy, VTy, CostKind, {}, NumEltDone(), CurrVecTy);
6049 }
6050
6051 // While we can directly load/store ZMM, YMM, and 64-bit halves of XMM,
6052 // for smaller widths (32/16/8) we have to insert/extract them separately.
6053 // Again, it's free for the 0'th subreg (if op is 32/64 bit wide,
6054 // but let's pretend that it is also true for 16/8 bit wide ops...)
6055 if (CurrOpSizeBytes <= 32 / 8 && !Is0thSubVec && !FoldedInLaneInsert) {
6056 int NumEltDoneInCurrXMM = NumEltDone() % NumEltPerXMM;
6057 assert(NumEltDoneInCurrXMM % CurrNumEltPerOp == 0 && "");
6058 int CoalescedVecEltIdx = NumEltDoneInCurrXMM / CurrNumEltPerOp;
6059 APInt DemandedElts =
6060 APInt::getBitsSet(CoalescedVecTy->getNumElements(),
6061 CoalescedVecEltIdx, CoalescedVecEltIdx + 1);
6062 assert(DemandedElts.popcount() == 1 && "Inserting single value");
6063 Cost += getScalarizationOverhead(CoalescedVecTy, DemandedElts, IsLoad,
6064 !IsLoad, CostKind);
6065 }
6066
6067 SubVecEltsLeft -= CurrNumEltPerOp;
6068 NumEltRemaining -= CurrNumEltPerOp;
6069 Alignment = commonAlignment(Alignment, CurrOpSizeBytes);
6070 }
6071 }
6072
6073 assert(NumEltRemaining <= 0 && "Should have processed all the elements.");
6074
6075 return Cost;
6076}
6077
6080 TTI::TargetCostKind CostKind) const {
6081 switch (MICA.getID()) {
6082 case Intrinsic::masked_scatter:
6083 case Intrinsic::masked_gather:
6084 return getGatherScatterOpCost(MICA, CostKind);
6085 case Intrinsic::masked_load:
6086 case Intrinsic::masked_store:
6087 return getMaskedMemoryOpCost(MICA, CostKind);
6088 case Intrinsic::masked_compressstore:
6089 // Fallback to scalarization for slow compressstore targets (e.g. znver4).
6090 if (ST->isVecCompressStoreSlow())
6091 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
6092 break;
6093 }
6094
6095 static const CostKindTblEntry AVX512VBMI2CostTable[] = {
6096 { Intrinsic::masked_expandload, MVT::v16i8, { 2, 7, 1, 3 } },
6097 { Intrinsic::masked_expandload, MVT::v32i8, { 2, 8, 1, 3 } },
6098 { Intrinsic::masked_expandload, MVT::v64i8, { 2, 9, 1, 3 } },
6099
6100 { Intrinsic::masked_expandload, MVT::v8i16, { 2, 7, 1, 3 } },
6101 { Intrinsic::masked_expandload, MVT::v16i16, { 2, 8, 1, 3 } },
6102 { Intrinsic::masked_expandload, MVT::v32i16, { 2, 9, 1, 3 } },
6103
6104 { Intrinsic::masked_expandload, MVT::v8f16, { 2, 7, 1, 3 } },
6105 { Intrinsic::masked_expandload, MVT::v16f16, { 2, 8, 1, 3 } },
6106 { Intrinsic::masked_expandload, MVT::v32f16, { 2, 9, 1, 3 } },
6107
6108 { Intrinsic::masked_expandload, MVT::v8bf16, { 2, 7, 1, 3 } },
6109 { Intrinsic::masked_expandload, MVT::v16bf16,{ 2, 8, 1, 3 } },
6110 { Intrinsic::masked_expandload, MVT::v32bf16,{ 2, 9, 1, 3 } },
6111
6112 { Intrinsic::masked_compressstore, MVT::v16i8, { 3,10, 1, 7 } },
6113 { Intrinsic::masked_compressstore, MVT::v32i8, { 3,11, 1, 7 } },
6114 { Intrinsic::masked_compressstore, MVT::v64i8, { 3,12, 1, 8 } },
6115
6116 { Intrinsic::masked_compressstore, MVT::v8i16, { 3,10, 1, 7 } },
6117 { Intrinsic::masked_compressstore, MVT::v16i16, { 3,11, 1, 7 } },
6118 { Intrinsic::masked_compressstore, MVT::v32i16, { 3,12, 1, 8 } },
6119
6120 { Intrinsic::masked_compressstore, MVT::v8f16, { 3,10, 1, 7 } },
6121 { Intrinsic::masked_compressstore, MVT::v16f16, { 3,11, 1, 7 } },
6122 { Intrinsic::masked_compressstore, MVT::v32f16, { 3,12, 1, 8 } },
6123
6124 { Intrinsic::masked_compressstore, MVT::v8bf16, { 3,10, 1, 7 } },
6125 { Intrinsic::masked_compressstore, MVT::v16bf16,{ 3,11, 1, 7 } },
6126 { Intrinsic::masked_compressstore, MVT::v32bf16,{ 3,12, 1, 8 } },
6127 };
6128
6129 static const CostKindTblEntry AVX512CostTable[] = {
6130 { Intrinsic::masked_expandload, MVT::v4i32, { 2, 7, 1, 3 } },
6131 { Intrinsic::masked_expandload, MVT::v4f32, { 2, 7, 1, 3 } },
6132 { Intrinsic::masked_expandload, MVT::v8i32, { 2, 8, 1, 3 } },
6133 { Intrinsic::masked_expandload, MVT::v8f32, { 2, 8, 1, 3 } },
6134 { Intrinsic::masked_expandload, MVT::v16i32, { 2, 9, 1, 3 } },
6135 { Intrinsic::masked_expandload, MVT::v16f32, { 2, 9, 1, 3 } },
6136
6137 { Intrinsic::masked_expandload, MVT::v2i64, { 2, 7, 1, 3 } },
6138 { Intrinsic::masked_expandload, MVT::v2f64, { 2, 7, 1, 3 } },
6139 { Intrinsic::masked_expandload, MVT::v4i64, { 2, 8, 1, 3 } },
6140 { Intrinsic::masked_expandload, MVT::v4f64, { 2, 8, 1, 3 } },
6141 { Intrinsic::masked_expandload, MVT::v8i64, { 2, 9, 1, 3 } },
6142 { Intrinsic::masked_expandload, MVT::v8f64, { 2, 9, 1, 3 } },
6143
6144 { Intrinsic::masked_compressstore, MVT::v4i32, { 3,10, 1, 7 } },
6145 { Intrinsic::masked_compressstore, MVT::v4f32, { 3,10, 1, 7 } },
6146 { Intrinsic::masked_compressstore, MVT::v8i32, { 3,11, 1, 7 } },
6147 { Intrinsic::masked_compressstore, MVT::v8f32, { 3,11, 1, 7 } },
6148 { Intrinsic::masked_compressstore, MVT::v16i32, { 3,12, 1, 8 } },
6149 { Intrinsic::masked_compressstore, MVT::v16f32, { 3,12, 1, 8 } },
6150
6151 { Intrinsic::masked_compressstore, MVT::v2i64, { 3,10, 1, 7 } },
6152 { Intrinsic::masked_compressstore, MVT::v2f64, { 3,10, 1, 7 } },
6153 { Intrinsic::masked_compressstore, MVT::v4i64, { 3,11, 1, 7 } },
6154 { Intrinsic::masked_compressstore, MVT::v4f64, { 3,11, 1, 7 } },
6155 { Intrinsic::masked_compressstore, MVT::v8i64, { 3,12, 1, 8 } },
6156 { Intrinsic::masked_compressstore, MVT::v8f64, { 3,12, 1, 8 } },
6157 };
6158
6159 std::pair<InstructionCost, MVT> LT =
6161
6162 if (ST->hasVBMI2())
6163 if (const auto *Entry =
6164 CostTableLookup(AVX512VBMI2CostTable, MICA.getID(), LT.second))
6165 if (auto KindCost = Entry->Cost[CostKind])
6166 return LT.first * *KindCost;
6167
6168 if (ST->hasAVX512())
6169 if (const auto *Entry =
6170 CostTableLookup(AVX512CostTable, MICA.getID(), LT.second))
6171 if (auto KindCost = Entry->Cost[CostKind])
6172 return LT.first * *KindCost;
6173
6174 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
6175}
6176
6179 TTI::TargetCostKind CostKind) const {
6180 unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
6181 : Instruction::Store;
6182 Type *SrcTy = MICA.getDataType();
6183 Align Alignment = MICA.getAlignment();
6184 unsigned AddressSpace = MICA.getAddressSpace();
6185
6186 bool IsLoad = (Instruction::Load == Opcode);
6187 bool IsStore = (Instruction::Store == Opcode);
6188
6189 auto *SrcVTy = dyn_cast<FixedVectorType>(SrcTy);
6190 if (!SrcVTy)
6191 // To calculate scalar take the regular cost, without mask
6192 return getMemoryOpCost(Opcode, SrcTy, Alignment, AddressSpace, CostKind);
6193
6194 unsigned NumElem = SrcVTy->getNumElements();
6195 auto *MaskTy =
6196 FixedVectorType::get(Type::getInt8Ty(SrcVTy->getContext()), NumElem);
6197 if ((IsLoad && !isLegalMaskedLoad(SrcVTy, Alignment, AddressSpace)) ||
6198 (IsStore && !isLegalMaskedStore(SrcVTy, Alignment, AddressSpace))) {
6199 // Scalarization
6200 APInt DemandedElts = APInt::getAllOnes(NumElem);
6202 MaskTy, DemandedElts, /*Insert*/ false, /*Extract*/ true, CostKind);
6203 InstructionCost ScalarCompareCost = getCmpSelInstrCost(
6204 Instruction::ICmp, Type::getInt8Ty(SrcVTy->getContext()), nullptr,
6205 CmpInst::BAD_ICMP_PREDICATE, CostKind);
6206 InstructionCost BranchCost = getCFInstrCost(Instruction::CondBr, CostKind);
6207 InstructionCost MaskCmpCost = NumElem * (BranchCost + ScalarCompareCost);
6209 SrcVTy, DemandedElts, IsLoad, IsStore, CostKind);
6210 InstructionCost MemopCost =
6211 NumElem * BaseT::getMemoryOpCost(Opcode, SrcVTy->getScalarType(),
6212 Alignment, AddressSpace, CostKind);
6213 return MemopCost + ValueSplitCost + MaskSplitCost + MaskCmpCost;
6214 }
6215
6216 // Legalize the type.
6217 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcVTy);
6218 auto VT = TLI->getValueType(DL, SrcVTy);
6220 MVT Ty = LT.second;
6221 if (Ty == MVT::i16 || Ty == MVT::i32 || Ty == MVT::i64)
6222 // APX masked load/store for scalar is cheap.
6223 return Cost + LT.first;
6224
6225 if (VT.isSimple() && Ty != VT.getSimpleVT() &&
6226 LT.second.getVectorNumElements() == NumElem)
6227 // Promotion requires extend/truncate for data and a shuffle for mask.
6228 Cost += getShuffleCost(TTI::SK_PermuteTwoSrc, SrcVTy, SrcVTy, CostKind, {},
6229 0, nullptr) +
6230 getShuffleCost(TTI::SK_PermuteTwoSrc, MaskTy, MaskTy, CostKind, {},
6231 0, nullptr);
6232
6233 else if (LT.first * Ty.getVectorNumElements() > NumElem) {
6234 auto *NewMaskTy = FixedVectorType::get(MaskTy->getElementType(),
6235 (unsigned)LT.first.getValue() *
6236 Ty.getVectorNumElements());
6237 // Expanding requires fill mask with zeroes
6238 Cost += getShuffleCost(TTI::SK_InsertSubvector, NewMaskTy, NewMaskTy,
6239 CostKind, {}, 0, MaskTy);
6240 }
6241
6242 // Predicate fanout (see getPredicateFanoutCost): only a variable mask needs
6243 // a per-part sub-mask; a constant mask is folded by codegen and pays nothing.
6244 InstructionCost MaskExpCost = 0;
6245 if (MICA.getVariableMask()) {
6246 auto *MaskVecTy =
6247 FixedVectorType::get(Type::getInt1Ty(SrcVTy->getContext()), NumElem);
6248 MaskExpCost = getPredicateFanoutCost(*this, SrcVTy, MaskVecTy);
6249 }
6250
6251 // Pre-AVX512 - each maskmov load costs 2 + store costs ~8.
6252 if (!ST->hasAVX512())
6253 return Cost + LT.first * (IsLoad ? 2 : 8) + MaskExpCost;
6254
6255 // AVX-512 masked load/store is cheaper.
6256 return Cost + LT.first + MaskExpCost;
6257}
6258
6260 ArrayRef<const Value *> Ptrs, const Value *Base,
6261 const TTI::PointersChainInfo &Info, Type *AccessTy,
6262 const TTI::TargetCostKind CostKind) const {
6263 if (Info.isSameBase() && Info.isKnownStride()) {
6264 // If all the pointers have known stride all the differences are translated
6265 // into constants. X86 memory addressing allows encoding it into
6266 // displacement. So we just need to take the base GEP cost.
6267 if (const auto *BaseGEP = dyn_cast<GetElementPtrInst>(Base)) {
6268 SmallVector<const Value *> Indices(BaseGEP->indices());
6269 return getGEPCost(BaseGEP->getSourceElementType(),
6270 BaseGEP->getPointerOperand(), Indices, CostKind,
6271 nullptr);
6272 }
6273 return TTI::TCC_Free;
6274 }
6275 return BaseT::getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);
6276}
6277
6280 const SCEV *Ptr,
6281 TTI::TargetCostKind CostKind) const {
6282 // Address computations in vectorized code with non-consecutive addresses will
6283 // likely result in more instructions compared to scalar code where the
6284 // computation can more often be merged into the index mode. The resulting
6285 // extra micro-ops can significantly decrease throughput.
6286 const unsigned NumVectorInstToHideOverhead = 10;
6287
6288 // Cost modeling of Strided Access Computation is hidden by the indexing
6289 // modes of X86 regardless of the stride value. We dont believe that there
6290 // is a difference between constant strided access in gerenal and constant
6291 // strided value which is less than or equal to 64.
6292 // Even in the case of (loop invariant) stride whose value is not known at
6293 // compile time, the address computation will not incur more than one extra
6294 // ADD instruction.
6295 if (PtrTy->isVectorTy() && SE) {
6297 return 1;
6298 if (!ST->hasAVX2()) {
6299 // TODO: AVX2 is the current cut-off because we don't have correct
6300 // interleaving costs for prior ISA's.
6301 if (!BaseT::isStridedAccess(Ptr))
6302 return NumVectorInstToHideOverhead;
6303 }
6304 }
6305
6306 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
6307}
6308
6310 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
6312 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
6313 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
6314 auto ExpandCost = [&]() {
6315 return BaseT::getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
6316 AccumType, VF, OpAExtend, OpBExtend,
6317 BinOp, CostKind, FMF);
6318 };
6319
6320 // The dot product instructions multiply-accumulate i8 x i8 -> i32,
6321 // i16 x i16 -> i32, bf16 x bf16 -> f32 or f16 x f16 -> f32. Partial
6322 // reductions may also multiply inputs extended from different types, which
6323 // they can't handle.
6324 if (VF.isScalable() || !BinOp || OpAExtend == TTI::PR_None ||
6325 OpBExtend == TTI::PR_None || InputTypeA != InputTypeB)
6326 return ExpandCost();
6327
6328 unsigned Opc;
6329 if (Opcode == Instruction::Add && *BinOp == Instruction::Mul &&
6330 AccumType->isIntegerTy(32) &&
6331 (InputTypeA->isIntegerTy(8) || InputTypeA->isIntegerTy(16))) {
6332 if (OpAExtend != OpBExtend)
6334 else if (OpAExtend == TTI::PR_SignExtend)
6336 else
6338 } else if (Opcode == Instruction::FAdd && *BinOp == Instruction::FMul &&
6339 AccumType->isFloatTy() &&
6340 (InputTypeA->isBFloatTy() || InputTypeA->isHalfTy())) {
6341 // VDPBF16PS and VDPPHPS, like the expansion, reassociate the additions and
6342 // fuse the multiplications.
6343 if (!FMF || !FMF->allowReassoc() || !FMF->allowContract())
6346 } else {
6347 return ExpandCost();
6348 }
6349
6350 unsigned Ratio =
6351 AccumType->getScalarSizeInBits() / InputTypeA->getScalarSizeInBits();
6352 if (!VF.isKnownMultipleOf(Ratio))
6353 return ExpandCost();
6354
6355 // One dot product per legal accumulator vector. Accumulators narrower than
6356 // a legal vector are widened by expanding the partial reduction instead.
6357 auto *AccVecTy = VectorType::get(AccumType, VF.divideCoefficientBy(Ratio));
6358 auto *InputVecTy = VectorType::get(InputTypeA, VF);
6359 std::pair<InstructionCost, MVT> AccLT = getTypeLegalizationCost(AccVecTy);
6360 std::pair<InstructionCost, MVT> InputLT = getTypeLegalizationCost(InputVecTy);
6361 if (AccLT.second.getFixedSizeInBits() >
6362 AccVecTy->getPrimitiveSizeInBits().getFixedValue() ||
6363 !TLI->isPartialReduceMLALegalOrCustom(Opc, AccLT.second, InputLT.second))
6364 return ExpandCost();
6365
6366 return AccLT.first;
6367}
6368
6371 std::optional<FastMathFlags> FMF,
6372 TTI::TargetCostKind CostKind) const {
6374 return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);
6375
6376 // We use llvm-mca across all supported CPUs to measure the logic cost stats.
6377 // We use the Intel Architecture Code Analyzer(IACA) to measure the throughput
6378 // and make it as the cost. TODO: Update old IACA numbers to llvm-mca.
6379
6380 static const CostKindTblEntry SLMCostTbl[] = {
6381 { ISD::FADD, MVT::v2f64, {3, 3, 3, 3} },
6382 { ISD::ADD, MVT::v2i64, {5, 5, 5, 5} },
6383 };
6384
6385 static const CostKindTblEntry SSE2CostTbl[] = {
6386 { ISD::FADD, MVT::v2f64, {2, 2, 2, 2} },
6387 { ISD::FADD, MVT::v2f32, {2, 2, 2, 2} },
6388 { ISD::FADD, MVT::v4f32, {4, 4, 4, 4} },
6389 { ISD::ADD, MVT::v2i64, {2, 2, 2, 2} }, // The data reported by the IACA tool is "1.6".
6390 { ISD::ADD, MVT::v2i32, {2, 2, 2, 2} }, // FIXME: chosen to be less than v4i32
6391 { ISD::ADD, MVT::v4i32, {3, 3, 3, 3} }, // The data reported by the IACA tool is "3.3".
6392 { ISD::ADD, MVT::v2i16, {2, 2, 2, 2} }, // The data reported by the IACA tool is "4.3".
6393 { ISD::ADD, MVT::v4i16, {3, 3, 3, 3} }, // The data reported by the IACA tool is "4.3".
6394 { ISD::ADD, MVT::v8i16, {4, 4, 4, 4} }, // The data reported by the IACA tool is "4.3".
6395 { ISD::ADD, MVT::v2i8, {2, 2, 2, 2} },
6396 { ISD::ADD, MVT::v4i8, {2, 2, 2, 2} },
6397 { ISD::ADD, MVT::v8i8, {2, 2, 2, 2} },
6398 { ISD::ADD, MVT::v16i8, {3, 3, 3, 3} },
6399
6400 { ISD::AND, MVT::v2i64, {2, 2, 3, 3} },
6401 { ISD::AND, MVT::v4i32, {3, 4, 5, 5} },
6402 { ISD::AND, MVT::v8i16, {4, 7, 8, 8} },
6403 { ISD::AND, MVT::v16i8, {6,10,11,11} },
6404 { ISD::OR, MVT::v2i64, {2, 2, 3, 3} },
6405 { ISD::OR, MVT::v4i32, {3, 4, 5, 5} },
6406 { ISD::OR, MVT::v8i16, {4, 7, 8, 8} },
6407 { ISD::OR, MVT::v16i8, {6,10,11,11} },
6408 { ISD::XOR, MVT::v2i64, {2, 2, 3, 3} },
6409 { ISD::XOR, MVT::v4i32, {3, 4, 5, 5} },
6410 { ISD::XOR, MVT::v8i16, {4, 7, 8, 8} },
6411 { ISD::XOR, MVT::v16i8, {6,10,11,11} },
6412 };
6413
6414 static const CostKindTblEntry AVX1CostTbl[] = {
6415 { ISD::FADD, MVT::v4f64, {3, 3, 3, 3} },
6416 { ISD::FADD, MVT::v4f32, {3, 3, 3, 3} },
6417 { ISD::FADD, MVT::v8f32, {4, 4, 4, 4} },
6418 { ISD::ADD, MVT::v2i64, {1, 1, 1, 1} }, // The data reported by the IACA tool is "1.5".
6419 { ISD::ADD, MVT::v4i64, {3, 3, 3, 3} },
6420 { ISD::ADD, MVT::v8i32, {5, 5, 5, 5} },
6421 { ISD::ADD, MVT::v16i16, {5, 5, 5, 5} },
6422 { ISD::ADD, MVT::v32i8, {4, 4, 4, 4} },
6423
6424 { ISD::AND, MVT::v4i64, {3, 7, 5, 5} },
6425 { ISD::AND, MVT::v8i32, {4, 9, 7, 7} },
6426 { ISD::AND, MVT::v16i16, {5,11, 9, 9} },
6427 { ISD::AND, MVT::v8i16, {4, 7, 7, 7} },
6428 { ISD::AND, MVT::v32i8, {6,13,11,11} },
6429 { ISD::AND, MVT::v16i8, {5,10, 9, 9} },
6430 { ISD::OR, MVT::v4i64, {3, 7, 5, 5} },
6431 { ISD::OR, MVT::v8i32, {4, 9, 7, 7} },
6432 { ISD::OR, MVT::v16i16, {5,11, 9, 9} },
6433 { ISD::OR, MVT::v8i16, {4, 7, 7, 7} },
6434 { ISD::OR, MVT::v32i8, {6,13,11,11} },
6435 { ISD::OR, MVT::v16i8, {5,10, 9, 9} },
6436 { ISD::XOR, MVT::v4i64, {3, 7, 5, 5} },
6437 { ISD::XOR, MVT::v8i32, {4, 9, 7, 7} },
6438 { ISD::XOR, MVT::v16i16, {5,11, 9, 9} },
6439 { ISD::XOR, MVT::v8i16, {4, 7, 7, 7} },
6440 { ISD::XOR, MVT::v32i8, {6,13,11,11} },
6441 { ISD::XOR, MVT::v16i8, {5,10, 9, 9} },
6442 };
6443
6444 static const CostKindTblEntry AVX2CostTbl[] = {
6445 { ISD::AND, MVT::v4i64, {2, 7, 5, 5} },
6446 { ISD::AND, MVT::v2i64, {1, 2, 3, 3} },
6447 { ISD::AND, MVT::v8i32, {3, 9, 7, 7} },
6448 { ISD::AND, MVT::v4i32, {2, 4, 5, 5} },
6449 { ISD::AND, MVT::v16i16, {3,11, 9, 9} },
6450 { ISD::AND, MVT::v8i16, {2, 6, 7, 7} },
6451 { ISD::AND, MVT::v32i8, {3,13,11,11} },
6452 { ISD::AND, MVT::v16i8, {3, 8, 9, 9} },
6453 { ISD::OR, MVT::v4i64, {2, 7, 5, 5} },
6454 { ISD::OR, MVT::v2i64, {1, 2, 3, 3} },
6455 { ISD::OR, MVT::v8i32, {3, 9, 7, 7} },
6456 { ISD::OR, MVT::v4i32, {2, 4, 5, 5} },
6457 { ISD::OR, MVT::v16i16, {3,11, 9, 9} },
6458 { ISD::OR, MVT::v8i16, {2, 6, 7, 7} },
6459 { ISD::OR, MVT::v32i8, {3,13,11,11} },
6460 { ISD::OR, MVT::v16i8, {3, 8, 9, 9} },
6461 { ISD::XOR, MVT::v4i64, {2, 7, 5, 5} },
6462 { ISD::XOR, MVT::v2i64, {1, 2, 3, 3} },
6463 { ISD::XOR, MVT::v8i32, {3, 9, 7, 7} },
6464 { ISD::XOR, MVT::v4i32, {2, 4, 5, 5} },
6465 { ISD::XOR, MVT::v16i16, {3,11, 9, 9} },
6466 { ISD::XOR, MVT::v8i16, {2, 6, 7, 7} },
6467 { ISD::XOR, MVT::v32i8, {3,13,11,11} },
6468 { ISD::XOR, MVT::v16i8, {3, 8, 9, 9} },
6469 };
6470
6471 static const CostKindTblEntry AVX512FCostTbl[] = {
6472 { ISD::FADD, MVT::v8f64, {4, 4, 4, 4} },
6473 { ISD::FADD, MVT::v16f32, {5, 5, 5, 5} },
6474 { ISD::ADD, MVT::v8i64, {4, 4, 4, 4} },
6475 { ISD::ADD, MVT::v16i32, {6, 6, 6, 6} },
6476
6477 { ISD::AND, MVT::v8i64, {3,10, 7, 7} },
6478 { ISD::AND, MVT::v16i32, {4,12, 9, 9} },
6479 { ISD::AND, MVT::v32i16, {4,14,11,11} },
6480 { ISD::AND, MVT::v64i8, {4,16,13,13} },
6481 { ISD::AND, MVT::v16i8, {2, 8, 9, 9} },
6482 { ISD::OR, MVT::v8i64, {3,10, 7, 7} },
6483 { ISD::OR, MVT::v16i32, {4,12, 9, 9} },
6484 { ISD::OR, MVT::v32i16, {4,14,11,11} },
6485 { ISD::OR, MVT::v64i8, {4,16,13,13} },
6486 { ISD::OR, MVT::v16i8, {2, 8, 9, 9} },
6487 { ISD::XOR, MVT::v8i64, {3,10, 7, 7} },
6488 { ISD::XOR, MVT::v16i32, {4,12, 9, 9} },
6489 { ISD::XOR, MVT::v32i16, {4,14,11,11} },
6490 { ISD::XOR, MVT::v64i8, {4,16,13,13} },
6491 { ISD::XOR, MVT::v16i8, {2, 8, 9, 9} },
6492 };
6493
6494 static const CostKindTblEntry AVX512BWCostTbl[] = {
6495 { ISD::ADD, MVT::v32i16, {7, 7, 7, 7} },
6496 { ISD::ADD, MVT::v64i8, {4, 4, 4, 4} },
6497 };
6498
6499 int ISD = TLI->InstructionOpcodeToISD(Opcode);
6500 assert(ISD && "Invalid opcode");
6501
6502 // Before legalizing the type, give a chance to look up illegal narrow types
6503 // in the table.
6504 // FIXME: Is there a better way to do this?
6505 EVT VT = TLI->getValueType(DL, ValTy);
6506 if (VT.isSimple()) {
6507 MVT MTy = VT.getSimpleVT();
6508 if (ST->useSLMArithCosts())
6509 if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
6510 if (auto KindCost = Entry->Cost[CostKind])
6511 return *KindCost;
6512
6513 if (ST->hasBWI())
6514 if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
6515 if (auto KindCost = Entry->Cost[CostKind])
6516 return *KindCost;
6517
6518 if (ST->hasAVX512())
6519 if (const auto *Entry = CostTableLookup(AVX512FCostTbl, ISD, MTy))
6520 if (auto KindCost = Entry->Cost[CostKind])
6521 return *KindCost;
6522
6523 if (ST->hasAVX2())
6524 if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
6525 if (auto KindCost = Entry->Cost[CostKind])
6526 return *KindCost;
6527
6528 if (ST->hasAVX())
6529 if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
6530 if (auto KindCost = Entry->Cost[CostKind])
6531 return *KindCost;
6532
6533 if (ST->hasSSE2())
6534 if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
6535 if (auto KindCost = Entry->Cost[CostKind])
6536 return *KindCost;
6537 }
6538
6539 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
6540
6541 MVT MTy = LT.second;
6542
6543 auto *ValVTy = cast<FixedVectorType>(ValTy);
6544
6545 InstructionCost ArithmeticCost = 0;
6546 if (LT.first != 1 && MTy.isVector() &&
6547 MTy.getVectorNumElements() < ValVTy->getNumElements()) {
6548 // Type needs to be split. We need LT.first - 1 arithmetic ops.
6549 auto *SingleOpTy = FixedVectorType::get(ValVTy->getElementType(),
6550 MTy.getVectorNumElements());
6551 ArithmeticCost = getArithmeticInstrCost(Opcode, SingleOpTy, CostKind);
6552 ArithmeticCost *= LT.first - 1;
6553 }
6554
6555 // FIXME: These assume a naive kshift+binop lowering, which is probably
6556 // conservative in most cases.
6557 static const CostKindTblEntry AVX512BoolReduction[] = {
6558 { ISD::AND, MVT::v2i1, { 3, 3, 3, 3} },
6559 { ISD::AND, MVT::v4i1, { 5, 5, 5, 5} },
6560 { ISD::AND, MVT::v8i1, { 7, 7, 7, 7} },
6561 { ISD::AND, MVT::v16i1, { 9, 9, 9, 9} },
6562 { ISD::AND, MVT::v32i1, {11,11,11,11} },
6563 { ISD::AND, MVT::v64i1, {13,13,13,13} },
6564 { ISD::OR, MVT::v2i1, { 3, 3, 3, 3} },
6565 { ISD::OR, MVT::v4i1, { 5, 5, 5, 5} },
6566 { ISD::OR, MVT::v8i1, { 7, 7, 7, 7} },
6567 { ISD::OR, MVT::v16i1, { 9, 9, 9, 9} },
6568 { ISD::OR, MVT::v32i1, {11,11,11,11} },
6569 { ISD::OR, MVT::v64i1, {13,13,13,13} },
6570 };
6571
6572 static const CostKindTblEntry AVX2BoolReduction[] = {
6573 { ISD::AND, MVT::v16i16, { 2, 2, 2, 2} }, // vpmovmskb + cmp
6574 { ISD::AND, MVT::v32i8, { 2, 2, 2, 2} }, // vpmovmskb + cmp
6575 { ISD::OR, MVT::v16i16, { 2, 2, 2, 2} }, // vpmovmskb + cmp
6576 { ISD::OR, MVT::v32i8, { 2, 2, 2, 2} }, // vpmovmskb + cmp
6577 };
6578
6579 static const CostKindTblEntry AVX1BoolReduction[] = {
6580 { ISD::AND, MVT::v4i64, {2, 2, 2, 2} }, // vmovmskpd + cmp
6581 { ISD::AND, MVT::v8i32, {2, 2, 2, 2} }, // vmovmskps + cmp
6582 { ISD::AND, MVT::v16i16, {4, 4, 4, 4} }, // vextractf128 + vpand + vpmovmskb + cmp
6583 { ISD::AND, MVT::v32i8, {4, 4, 4, 4} }, // vextractf128 + vpand + vpmovmskb + cmp
6584 { ISD::OR, MVT::v4i64, {2, 2, 2, 2} }, // vmovmskpd + cmp
6585 { ISD::OR, MVT::v8i32, {2, 2, 2, 2} }, // vmovmskps + cmp
6586 { ISD::OR, MVT::v16i16, {4, 4, 4, 4} }, // vextractf128 + vpor + vpmovmskb + cmp
6587 { ISD::OR, MVT::v32i8, {4, 4, 4, 4} }, // vextractf128 + vpor + vpmovmskb + cmp
6588 };
6589
6590 static const CostKindTblEntry SSE2BoolReduction[] = {
6591 { ISD::AND, MVT::v2i64, {2, 2, 2, 2} }, // movmskpd + cmp
6592 { ISD::AND, MVT::v4i32, {2, 2, 2, 2} }, // movmskps + cmp
6593 { ISD::AND, MVT::v8i16, {2, 2, 2, 2} }, // pmovmskb + cmp
6594 { ISD::AND, MVT::v16i8, {2, 2, 2, 2} }, // pmovmskb + cmp
6595 { ISD::OR, MVT::v2i64, {2, 2, 2, 2} }, // movmskpd + cmp
6596 { ISD::OR, MVT::v4i32, {2, 2, 2, 2} }, // movmskps + cmp
6597 { ISD::OR, MVT::v8i16, {2, 2, 2, 2} }, // pmovmskb + cmp
6598 { ISD::OR, MVT::v16i8, {2, 2, 2, 2} }, // pmovmskb + cmp
6599 };
6600
6601 // Handle bool allof/anyof vXi1 patterns before we check legal types.
6602 if (ValVTy->getElementType()->isIntegerTy(1)) {
6603 if (ISD == ISD::ADD) {
6604 // vXi1 addition reduction will bitcast to scalar and perform a popcount.
6605 auto *IntTy = IntegerType::getIntNTy(ValVTy->getContext(),
6606 ValVTy->getNumElements());
6607 IntrinsicCostAttributes ICA(Intrinsic::ctpop, IntTy, {IntTy});
6608 return getCastInstrCost(Instruction::BitCast, IntTy, ValVTy,
6610 CostKind) +
6611 getIntrinsicInstrCost(ICA, CostKind);
6612 }
6613
6614 if (ST->hasAVX512())
6615 if (const auto *Entry = CostTableLookup(AVX512BoolReduction, ISD, MTy))
6616 if (auto KindCost = Entry->Cost[CostKind])
6617 return ArithmeticCost + *KindCost;
6618 if (ST->hasAVX2())
6619 if (const auto *Entry = CostTableLookup(AVX2BoolReduction, ISD, MTy))
6620 if (auto KindCost = Entry->Cost[CostKind])
6621 return ArithmeticCost + *KindCost;
6622 if (ST->hasAVX())
6623 if (const auto *Entry = CostTableLookup(AVX1BoolReduction, ISD, MTy))
6624 if (auto KindCost = Entry->Cost[CostKind])
6625 return ArithmeticCost + *KindCost;
6626 if (ST->hasSSE2())
6627 if (const auto *Entry = CostTableLookup(SSE2BoolReduction, ISD, MTy))
6628 if (auto KindCost = Entry->Cost[CostKind])
6629 return ArithmeticCost + *KindCost;
6630
6631 return BaseT::getArithmeticReductionCost(Opcode, ValVTy, FMF, CostKind);
6632 }
6633
6634 // Special case: vXi8 mul reductions are performed as vXi16.
6635 if (ISD == ISD::MUL && MTy.getScalarType() == MVT::i8) {
6636 auto *WideSclTy = IntegerType::get(ValVTy->getContext(), 16);
6637 auto *WideVecTy = FixedVectorType::get(WideSclTy, ValVTy->getNumElements());
6638 return getCastInstrCost(Instruction::ZExt, WideVecTy, ValTy,
6640 CostKind) +
6641 getArithmeticReductionCost(Opcode, WideVecTy, FMF, CostKind);
6642 }
6643
6644 if (ST->useSLMArithCosts())
6645 if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
6646 if (auto KindCost = Entry->Cost[CostKind])
6647 return ArithmeticCost + *KindCost;
6648
6649 if (ST->hasBWI())
6650 if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
6651 if (auto KindCost = Entry->Cost[CostKind])
6652 return ArithmeticCost + *KindCost;
6653
6654 if (ST->hasAVX512())
6655 if (const auto *Entry = CostTableLookup(AVX512FCostTbl, ISD, MTy))
6656 if (auto KindCost = Entry->Cost[CostKind])
6657 return ArithmeticCost + *KindCost;
6658
6659 if (ST->hasAVX2())
6660 if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
6661 if (auto KindCost = Entry->Cost[CostKind])
6662 return ArithmeticCost + *KindCost;
6663
6664 if (ST->hasAVX())
6665 if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
6666 if (auto KindCost = Entry->Cost[CostKind])
6667 return ArithmeticCost + *KindCost;
6668
6669 if (ST->hasSSE2())
6670 if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
6671 if (auto KindCost = Entry->Cost[CostKind])
6672 return ArithmeticCost + *KindCost;
6673
6674 unsigned NumVecElts = ValVTy->getNumElements();
6675 unsigned ScalarSize = ValVTy->getScalarSizeInBits();
6676
6677 // Special case power of 2 reductions where the scalar type isn't changed
6678 // by type legalization.
6679 if (!isPowerOf2_32(NumVecElts) || ScalarSize != MTy.getScalarSizeInBits())
6680 return BaseT::getArithmeticReductionCost(Opcode, ValVTy, FMF, CostKind);
6681
6682 InstructionCost ReductionCost = 0;
6683
6684 auto *Ty = ValVTy;
6685 if (LT.first != 1 && MTy.isVector() &&
6686 MTy.getVectorNumElements() < ValVTy->getNumElements()) {
6687 // Type needs to be split. We need LT.first - 1 arithmetic ops.
6688 Ty = FixedVectorType::get(ValVTy->getElementType(),
6689 MTy.getVectorNumElements());
6690 ReductionCost = getArithmeticInstrCost(Opcode, Ty, CostKind);
6691 ReductionCost *= LT.first - 1;
6692 NumVecElts = MTy.getVectorNumElements();
6693 }
6694
6695 // Now handle reduction with the legal type, taking into account size changes
6696 // at each level.
6697 while (NumVecElts > 1) {
6698 // Determine the size of the remaining vector we need to reduce.
6699 unsigned Size = NumVecElts * ScalarSize;
6700 NumVecElts /= 2;
6701 // If we're reducing from 256/512 bits, use an extract_subvector.
6702 if (Size > 128) {
6703 auto *SubTy = FixedVectorType::get(ValVTy->getElementType(), NumVecElts);
6704 ReductionCost += getShuffleCost(TTI::SK_ExtractSubvector, Ty, Ty,
6705 CostKind, {}, NumVecElts, SubTy);
6706 Ty = SubTy;
6707 } else if (Size == 128) {
6708 // Reducing from 128 bits is a permute of v2f64/v2i64.
6709 FixedVectorType *ShufTy;
6710 if (ValVTy->isFloatingPointTy())
6711 ShufTy =
6712 FixedVectorType::get(Type::getDoubleTy(ValVTy->getContext()), 2);
6713 else
6714 ShufTy =
6715 FixedVectorType::get(Type::getInt64Ty(ValVTy->getContext()), 2);
6716 ReductionCost += getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, ShufTy,
6717 CostKind, {}, 0, nullptr);
6718 } else if (Size == 64) {
6719 // Reducing from 64 bits is a shuffle of v4f32/v4i32.
6720 FixedVectorType *ShufTy;
6721 if (ValVTy->isFloatingPointTy())
6722 ShufTy =
6723 FixedVectorType::get(Type::getFloatTy(ValVTy->getContext()), 4);
6724 else
6725 ShufTy =
6726 FixedVectorType::get(Type::getInt32Ty(ValVTy->getContext()), 4);
6727 ReductionCost += getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, ShufTy,
6728 CostKind, {}, 0, nullptr);
6729 } else {
6730 // Reducing from smaller size is a shift by immediate.
6731 auto *ShiftTy = FixedVectorType::get(
6732 Type::getIntNTy(ValVTy->getContext(), Size), 128 / Size);
6733 ReductionCost += getArithmeticInstrCost(
6734 Instruction::LShr, ShiftTy, CostKind,
6737 }
6738
6739 // Add the arithmetic op for this level.
6740 ReductionCost += getArithmeticInstrCost(Opcode, Ty, CostKind);
6741 }
6742
6743 // Add the final extract element to the cost.
6744 return ReductionCost + getVectorInstrCost(Instruction::ExtractElement, Ty,
6745 CostKind, 0, nullptr, nullptr,
6747}
6748
6750 TTI::TargetCostKind CostKind,
6751 FastMathFlags FMF) const {
6752 IntrinsicCostAttributes ICA(IID, Ty, {Ty, Ty}, FMF);
6753 return getIntrinsicInstrCost(ICA, CostKind);
6754}
6755
6758 FastMathFlags FMF,
6759 TTI::TargetCostKind CostKind) const {
6760 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
6761
6762 MVT MTy = LT.second;
6763
6765 if (ValTy->isIntOrIntVectorTy()) {
6766 ISD = (IID == Intrinsic::umin || IID == Intrinsic::umax) ? ISD::UMIN
6767 : ISD::SMIN;
6768 } else {
6769 assert(ValTy->isFPOrFPVectorTy() &&
6770 "Expected float point or integer vector type.");
6771 ISD = (IID == Intrinsic::minnum || IID == Intrinsic::maxnum)
6772 ? ISD::FMINNUM
6773 : ISD::FMINIMUM;
6774 }
6775
6776 // We use llvm-mca across all supported CPUs to measure the cost stats.
6777 static const CostKindTblEntry SSE2CostTbl[] = {
6778 {ISD::SMIN, MVT::v2i64, {3, 4, 5, 6}},
6779 {ISD::UMIN, MVT::v2i64, {3, 4, 5, 6}},
6780 {ISD::SMIN, MVT::v2i32, {2, 2, 5, 6}},
6781 {ISD::UMIN, MVT::v2i32, {2, 2, 5, 6}},
6782 {ISD::SMIN, MVT::v4i32, {3, 7,11,12}},
6783 {ISD::UMIN, MVT::v4i32, {4, 7,14,15}},
6784 {ISD::SMIN, MVT::v2i16, {2, 3, 4, 4}},
6785 {ISD::UMIN, MVT::v2i16, {2, 3, 4, 6}},
6786 {ISD::SMIN, MVT::v4i16, {3, 5, 6, 6}},
6787 {ISD::UMIN, MVT::v4i16, {3, 5, 8, 10}},
6788 {ISD::SMIN, MVT::v8i16, {3, 8, 8, 8}},
6789 {ISD::UMIN, MVT::v8i16, {4, 8,12,14}},
6790 {ISD::SMIN, MVT::v2i8, {2, 3, 5, 6}},
6791 {ISD::UMIN, MVT::v2i8, {2, 3, 4, 4}},
6792 {ISD::SMIN, MVT::v4i8, {4, 6,12,13}},
6793 {ISD::UMIN, MVT::v4i8, {3, 6, 7, 7}},
6794 {ISD::SMIN, MVT::v8i8, {5, 9,18,19}},
6795 {ISD::UMIN, MVT::v8i8, {4, 8, 9, 9}},
6796 {ISD::SMIN, MVT::v16i8, {7,13,24,25}},
6797 {ISD::UMIN, MVT::v16i8, {3,10,11,11}},
6798 };
6799
6800 static const CostKindTblEntry SSE41CostTbl[] = {
6801 {ISD::SMIN, MVT::v2i64, {3, 4, 4, 6}},
6802 {ISD::UMIN, MVT::v2i64, {3, 4, 4, 6}},
6803 {ISD::SMIN, MVT::v2i32, {2, 2, 3, 3}},
6804 {ISD::UMIN, MVT::v2i32, {2, 2, 3, 3}},
6805 {ISD::SMIN, MVT::v4i32, {3, 4, 5, 5}},
6806 {ISD::UMIN, MVT::v4i32, {3, 4, 5, 5}},
6807 {ISD::UMIN, MVT::v2i16, {2, 3, 4, 4}},
6808 {ISD::SMIN, MVT::v4i16, {3, 5, 6, 6}},
6809 {ISD::UMIN, MVT::v4i16, {3, 5, 6, 6}},
6810 {ISD::SMIN, MVT::v8i16, {2, 8, 4, 5}},
6811 {ISD::UMIN, MVT::v8i16, {2, 5, 2, 2}},
6812 {ISD::SMIN, MVT::v2i8, {2, 3, 4, 4}},
6813 {ISD::SMIN, MVT::v4i8, {3, 6, 7, 7}},
6814 {ISD::SMIN, MVT::v8i8, {4, 8, 9, 9}},
6815 {ISD::SMIN, MVT::v16i8, {3,10, 7, 8}},
6816 {ISD::UMIN, MVT::v16i8, {3, 8, 5, 5}},
6817 };
6818
6819 static const CostKindTblEntry AVX1CostTbl[] = {
6820 {ISD::SMIN, MVT::v4i64, {5,11, 7,10}},
6821 {ISD::UMIN, MVT::v4i64, {6,12,10,13}},
6822 {ISD::SMIN, MVT::v8i32, {4, 9, 7, 7}},
6823 {ISD::UMIN, MVT::v8i32, {4, 9, 7, 7}},
6824 {ISD::SMIN, MVT::v16i16, {3,15, 6, 7}},
6825 {ISD::UMIN, MVT::v16i16, {2, 9, 4, 4}},
6826 {ISD::SMIN, MVT::v32i8, {4,17, 8, 9}},
6827 {ISD::UMIN, MVT::v32i8, {3,11, 6, 6}},
6828 };
6829
6830 static const CostKindTblEntry AVX2CostTbl[] = {
6831 {ISD::SMIN, MVT::v4i64, {4,11, 7,10}},
6832 {ISD::UMIN, MVT::v4i64, {4,12,10,13}},
6833 {ISD::SMIN, MVT::v2i32, {1, 2, 3, 3}},
6834 {ISD::UMIN, MVT::v2i32, {1, 2, 3, 3}},
6835 {ISD::UMIN, MVT::v4i32, {2, 4, 5, 5}},
6836 {ISD::SMIN, MVT::v4i32, {2, 4, 5, 5}},
6837 {ISD::SMIN, MVT::v8i32, {3, 9, 7, 7}},
6838 {ISD::UMIN, MVT::v8i32, {3, 9, 7, 7}},
6839 {ISD::SMIN, MVT::v4i16, {2, 4, 5, 5}},
6840 {ISD::UMIN, MVT::v4i16, {2, 4, 5, 5}},
6841 {ISD::SMIN, MVT::v16i16, {2,15, 6, 7}},
6842 {ISD::SMIN, MVT::v8i8, {3, 6, 7, 7}},
6843 {ISD::UMIN, MVT::v8i8, {3, 6, 7, 7}},
6844 {ISD::SMIN, MVT::v32i8, {3,17, 8, 9}},
6845 };
6846
6847 static const CostKindTblEntry AVX512FCostTbl[] = {
6848 {ISD::SMIN, MVT::v2i64, {2, 4, 3, 3}},
6849 {ISD::UMIN, MVT::v2i64, {2, 4, 3, 3}},
6850 {ISD::SMIN, MVT::v4i64, {3,10, 5, 5}},
6851 {ISD::UMIN, MVT::v4i64, {3,10, 5, 5}},
6852 {ISD::SMIN, MVT::v8i64, {5,16, 7, 7}},
6853 {ISD::UMIN, MVT::v8i64, {5,16, 7, 7}},
6854 {ISD::SMIN, MVT::v16i32, {4,12, 9, 9}},
6855 {ISD::UMIN, MVT::v16i32, {4,12, 9, 9}},
6856 };
6857
6858 static const CostKindTblEntry AVX512BWCostTbl[] = {
6859 {ISD::SMIN, MVT::v2i16, {1, 2, 3, 3}},
6860 {ISD::UMIN, MVT::v2i16, {1, 2, 3, 3}},
6861 {ISD::SMIN, MVT::v32i16, {2,19, 8, 9}},
6862 {ISD::UMIN, MVT::v32i16, {2,12, 6, 6}},
6863 {ISD::SMIN, MVT::v2i8, {1, 2, 3, 3}},
6864 {ISD::UMIN, MVT::v2i8, {1, 2, 3, 3}},
6865 {ISD::SMIN, MVT::v4i8, {2, 4, 5, 5}},
6866 {ISD::UMIN, MVT::v4i8, {2, 4, 5, 5}},
6867 {ISD::SMIN, MVT::v16i8, {2,10, 6, 7}},
6868 {ISD::UMIN, MVT::v16i8, {2, 6, 4, 4}},
6869 {ISD::SMIN, MVT::v32i8, {2,17, 8, 9}},
6870 {ISD::UMIN, MVT::v32i8, {2,10, 6, 6}},
6871 {ISD::SMIN, MVT::v64i8, {2,21,10,11}},
6872 {ISD::UMIN, MVT::v64i8, {2,14, 8, 8}},
6873 };
6874
6875 // Before legalizing the type, give a chance to look up illegal narrow types
6876 // in the table.
6877 // FIXME: Is there a better way to do this?
6878 EVT VT = TLI->getValueType(DL, ValTy);
6879 if (VT.isSimple()) {
6880 MVT MTy = VT.getSimpleVT();
6881 if (ST->hasBWI())
6882 if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
6883 if (auto KindCost = Entry->Cost[CostKind])
6884 return *KindCost;
6885
6886 if (ST->hasAVX512())
6887 if (const auto *Entry = CostTableLookup(AVX512FCostTbl, ISD, MTy))
6888 if (auto KindCost = Entry->Cost[CostKind])
6889 return *KindCost;
6890
6891 if (ST->hasAVX2())
6892 if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
6893 if (auto KindCost = Entry->Cost[CostKind])
6894 return *KindCost;
6895
6896 if (ST->hasAVX())
6897 if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
6898 if (auto KindCost = Entry->Cost[CostKind])
6899 return *KindCost;
6900
6901 if (ST->hasSSE41())
6902 if (const auto *Entry = CostTableLookup(SSE41CostTbl, ISD, MTy))
6903 if (auto KindCost = Entry->Cost[CostKind])
6904 return *KindCost;
6905
6906 if (ST->hasSSE2())
6907 if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
6908 if (auto KindCost = Entry->Cost[CostKind])
6909 return *KindCost;
6910 }
6911
6912 auto *ValVTy = cast<FixedVectorType>(ValTy);
6913 unsigned NumVecElts = ValVTy->getNumElements();
6914
6915 auto *Ty = ValVTy;
6916 InstructionCost MinMaxCost = 0;
6917 if (LT.first != 1 && MTy.isVector() &&
6918 MTy.getVectorNumElements() < ValVTy->getNumElements()) {
6919 // Type needs to be split. We need LT.first - 1 operations ops.
6920 Ty = FixedVectorType::get(ValVTy->getElementType(),
6921 MTy.getVectorNumElements());
6922 MinMaxCost = getMinMaxCost(IID, Ty, CostKind, FMF);
6923 MinMaxCost *= LT.first - 1;
6924 NumVecElts = MTy.getVectorNumElements();
6925 }
6926
6927 if (ST->hasBWI())
6928 if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
6929 if (auto KindCost = Entry->Cost[CostKind])
6930 return MinMaxCost + *KindCost;
6931
6932 if (ST->hasAVX512())
6933 if (const auto *Entry = CostTableLookup(AVX512FCostTbl, ISD, MTy))
6934 if (auto KindCost = Entry->Cost[CostKind])
6935 return MinMaxCost + *KindCost;
6936
6937 if (ST->hasAVX2())
6938 if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
6939 if (auto KindCost = Entry->Cost[CostKind])
6940 return MinMaxCost + *KindCost;
6941
6942 if (ST->hasAVX())
6943 if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
6944 if (auto KindCost = Entry->Cost[CostKind])
6945 return MinMaxCost + *KindCost;
6946
6947 if (ST->hasSSE41())
6948 if (const auto *Entry = CostTableLookup(SSE41CostTbl, ISD, MTy))
6949 if (auto KindCost = Entry->Cost[CostKind])
6950 return MinMaxCost + *KindCost;
6951
6952 if (ST->hasSSE2())
6953 if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
6954 if (auto KindCost = Entry->Cost[CostKind])
6955 return MinMaxCost + *KindCost;
6956
6957 unsigned ScalarSize = ValTy->getScalarSizeInBits();
6958
6959 // Special case power of 2 reductions where the scalar type isn't changed
6960 // by type legalization.
6961 if (!isPowerOf2_32(ValVTy->getNumElements()) ||
6962 ScalarSize != MTy.getScalarSizeInBits())
6963 return BaseT::getMinMaxReductionCost(IID, ValTy, FMF, CostKind);
6964
6965 // Now handle reduction with the legal type, taking into account size changes
6966 // at each level.
6967 while (NumVecElts > 1) {
6968 // Determine the size of the remaining vector we need to reduce.
6969 unsigned Size = NumVecElts * ScalarSize;
6970 NumVecElts /= 2;
6971 // If we're reducing from 256/512 bits, use an extract_subvector.
6972 if (Size > 128) {
6973 auto *SubTy = FixedVectorType::get(ValVTy->getElementType(), NumVecElts);
6974 MinMaxCost += getShuffleCost(TTI::SK_ExtractSubvector, Ty, Ty, CostKind,
6975 {}, NumVecElts, SubTy);
6976 Ty = SubTy;
6977 } else if (Size == 128) {
6978 // Reducing from 128 bits is a permute of v2f64/v2i64.
6979 VectorType *ShufTy;
6980 if (ValTy->isFloatingPointTy())
6981 ShufTy =
6983 else
6984 ShufTy = FixedVectorType::get(Type::getInt64Ty(ValTy->getContext()), 2);
6985 MinMaxCost += getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, ShufTy,
6986 CostKind, {}, 0, nullptr);
6987 } else if (Size == 64) {
6988 // Reducing from 64 bits is a shuffle of v4f32/v4i32.
6989 FixedVectorType *ShufTy;
6990 if (ValTy->isFloatingPointTy())
6991 ShufTy = FixedVectorType::get(Type::getFloatTy(ValTy->getContext()), 4);
6992 else
6993 ShufTy = FixedVectorType::get(Type::getInt32Ty(ValTy->getContext()), 4);
6994 MinMaxCost += getShuffleCost(TTI::SK_PermuteSingleSrc, ShufTy, ShufTy,
6995 CostKind, {}, 0, nullptr);
6996 } else {
6997 // Reducing from smaller size is a shift by immediate.
6998 auto *ShiftTy = FixedVectorType::get(
6999 Type::getIntNTy(ValTy->getContext(), Size), 128 / Size);
7000 MinMaxCost += getArithmeticInstrCost(
7001 Instruction::LShr, ShiftTy, TTI::TCK_RecipThroughput,
7004 }
7005
7006 // Add the arithmetic op for this level.
7007 MinMaxCost += getMinMaxCost(IID, Ty, CostKind, FMF);
7008 }
7009
7010 // Add the final extract element to the cost.
7011 return MinMaxCost + getVectorInstrCost(Instruction::ExtractElement, Ty,
7012 CostKind, 0, nullptr, nullptr,
7014}
7015
7016/// Calculate the cost of materializing a 64-bit value. This helper
7017/// method might only calculate a fraction of a larger immediate. Therefore it
7018/// is valid to return a cost of ZERO.
7020 if (Val == 0)
7021 return TTI::TCC_Free;
7022
7023 if (isInt<32>(Val))
7024 return TTI::TCC_Basic;
7025
7026 return 2 * TTI::TCC_Basic;
7027}
7028
7030 TTI::TargetCostKind CostKind) const {
7031 assert(Ty->isIntegerTy());
7032
7033 unsigned BitSize = Ty->getPrimitiveSizeInBits();
7034 if (BitSize == 0)
7035 return ~0U;
7036
7037 // Never hoist constants larger than 128bit, because this might lead to
7038 // incorrect code generation or assertions in codegen.
7039 // Fixme: Create a cost model for types larger than i128 once the codegen
7040 // issues have been fixed.
7041 if (BitSize > 128)
7042 return TTI::TCC_Free;
7043
7044 if (Imm == 0)
7045 return TTI::TCC_Free;
7046
7047 // Sign-extend all constants to a multiple of 64-bit.
7048 APInt ImmVal = Imm;
7049 if (BitSize % 64 != 0)
7050 ImmVal = Imm.sext(alignTo(BitSize, 64));
7051
7052 // Split the constant into 64-bit chunks and calculate the cost for each
7053 // chunk.
7055 for (unsigned ShiftVal = 0; ShiftVal < BitSize; ShiftVal += 64) {
7056 APInt Tmp = ImmVal.ashr(ShiftVal).sextOrTrunc(64);
7057 int64_t Val = Tmp.getSExtValue();
7058 Cost += getIntImmCost(Val);
7059 }
7060 // We need at least one instruction to materialize the constant.
7061 return std::max<InstructionCost>(1, Cost);
7062}
7063
7065 const APInt &Imm, Type *Ty,
7066 TTI::TargetCostKind CostKind,
7067 Instruction *Inst) const {
7068 assert(Ty->isIntegerTy());
7069
7070 unsigned BitSize = Ty->getPrimitiveSizeInBits();
7071 unsigned ImmBitWidth = Imm.getBitWidth();
7072
7073 // There is no cost model for constants with a bit size of 0. Return TCC_Free
7074 // here, so that constant hoisting will ignore this constant.
7075 if (BitSize == 0)
7076 return TTI::TCC_Free;
7077
7078 unsigned ImmIdx = ~0U;
7079 switch (Opcode) {
7080 default:
7081 return TTI::TCC_Free;
7082 case Instruction::GetElementPtr:
7083 // Always hoist the base address of a GetElementPtr. This prevents the
7084 // creation of new constants for every base constant that gets constant
7085 // folded with the offset.
7086 if (Idx == 0)
7087 return 2 * TTI::TCC_Basic;
7088 return TTI::TCC_Free;
7089 case Instruction::Store:
7090 ImmIdx = 0;
7091 break;
7092 case Instruction::ICmp:
7093 // This is an imperfect hack to prevent constant hoisting of
7094 // compares that might be trying to check if a 64-bit value fits in
7095 // 32-bits. The backend can optimize these cases using a right shift by 32.
7096 // There are other predicates and immediates the backend can use shifts for.
7097 if (Idx == 1 && ImmBitWidth == 64) {
7098 uint64_t ImmVal = Imm.getZExtValue();
7099 if (ImmVal == 0x100000000ULL || ImmVal == 0xffffffff)
7100 return TTI::TCC_Free;
7101
7102 if (auto *Cmp = dyn_cast_or_null<CmpInst>(Inst)) {
7103 if (Cmp->isEquality()) {
7104 KnownBits Known = computeKnownBits(Cmp->getOperand(0), DL);
7105 if (Known.countMinTrailingZeros() >= 32)
7106 return TTI::TCC_Free;
7107 }
7108 }
7109 }
7110 ImmIdx = 1;
7111 break;
7112 case Instruction::And:
7113 // We support 64-bit ANDs with immediates with 32-bits of leading zeroes
7114 // by using a 32-bit operation with implicit zero extension. Detect such
7115 // immediates here as the normal path expects bit 31 to be sign extended.
7116 if (Idx == 1 && ImmBitWidth == 64 && Imm.isIntN(32))
7117 return TTI::TCC_Free;
7118 // If we have BMI then we can use BEXTR/BZHI to mask out upper i64 bits.
7119 if (Idx == 1 && ImmBitWidth == 64 && ST->is64Bit() && ST->hasBMI() &&
7120 Imm.isMask())
7121 return X86TTIImpl::getIntImmCost(ST->hasBMI2() ? 255 : 65535);
7122 ImmIdx = 1;
7123 break;
7124 case Instruction::Add:
7125 case Instruction::Sub:
7126 // For add/sub, we can use the opposite instruction for INT32_MIN.
7127 if (Idx == 1 && ImmBitWidth == 64 && Imm.getZExtValue() == 0x80000000)
7128 return TTI::TCC_Free;
7129 ImmIdx = 1;
7130 break;
7131 case Instruction::UDiv:
7132 case Instruction::SDiv:
7133 case Instruction::URem:
7134 case Instruction::SRem:
7135 // Division by constant is typically expanded later into a different
7136 // instruction sequence. This completely changes the constants.
7137 // Report them as "free" to stop ConstantHoist from marking them as opaque.
7138 return TTI::TCC_Free;
7139 case Instruction::Mul:
7140 case Instruction::Or:
7141 case Instruction::Xor:
7142 ImmIdx = 1;
7143 break;
7144 // Always return TCC_Free for the shift value of a shift instruction.
7145 case Instruction::Shl:
7146 case Instruction::LShr:
7147 case Instruction::AShr:
7148 if (Idx == 1)
7149 return TTI::TCC_Free;
7150 break;
7151 case Instruction::Trunc:
7152 case Instruction::ZExt:
7153 case Instruction::SExt:
7154 case Instruction::IntToPtr:
7155 case Instruction::PtrToInt:
7156 case Instruction::BitCast:
7157 case Instruction::PHI:
7158 case Instruction::Call:
7159 case Instruction::Select:
7160 case Instruction::Ret:
7161 case Instruction::Load:
7162 break;
7163 }
7164
7165 if (Idx == ImmIdx) {
7166 uint64_t NumConstants = divideCeil(BitSize, 64);
7168 return (Cost <= NumConstants * TTI::TCC_Basic)
7169 ? static_cast<int>(TTI::TCC_Free)
7170 : Cost;
7171 }
7172
7173 return X86TTIImpl::getIntImmCost(Imm, Ty, CostKind);
7174}
7175
7178 const APInt &Imm, Type *Ty,
7179 TTI::TargetCostKind CostKind) const {
7180 assert(Ty->isIntegerTy());
7181
7182 unsigned BitSize = Ty->getPrimitiveSizeInBits();
7183 // There is no cost model for constants with a bit size of 0. Return TCC_Free
7184 // here, so that constant hoisting will ignore this constant.
7185 if (BitSize == 0)
7186 return TTI::TCC_Free;
7187
7188 switch (IID) {
7189 default:
7190 return TTI::TCC_Free;
7191 case Intrinsic::sadd_with_overflow:
7192 case Intrinsic::uadd_with_overflow:
7193 case Intrinsic::ssub_with_overflow:
7194 case Intrinsic::usub_with_overflow:
7195 case Intrinsic::smul_with_overflow:
7196 case Intrinsic::umul_with_overflow:
7197 if ((Idx == 1) && Imm.getBitWidth() <= 64 && Imm.isSignedIntN(32))
7198 return TTI::TCC_Free;
7199 break;
7200 case Intrinsic::experimental_stackmap:
7201 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && Imm.isSignedIntN(64)))
7202 return TTI::TCC_Free;
7203 break;
7204 case Intrinsic::experimental_patchpoint_void:
7205 case Intrinsic::experimental_patchpoint:
7206 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && Imm.isSignedIntN(64)))
7207 return TTI::TCC_Free;
7208 break;
7209 }
7210 return X86TTIImpl::getIntImmCost(Imm, Ty, CostKind);
7211}
7212
7214 TTI::TargetCostKind CostKind,
7215 const Instruction *I) const {
7216 if (CostKind != TTI::TCK_RecipThroughput)
7217 return Opcode == Instruction::PHI ? TTI::TCC_Free : TTI::TCC_Basic;
7218 // Branches are assumed to be predicted.
7219 return TTI::TCC_Free;
7220}
7221
7222int X86TTIImpl::getGatherOverhead() const {
7223 // Some CPUs have more overhead for gather. The specified overhead is relative
7224 // to the Load operation. "2" is the number provided by Intel architects. This
7225 // parameter is used for cost estimation of Gather Op and comparison with
7226 // other alternatives.
7227 // TODO: Remove the explicit hasAVX512()?, That would mean we would only
7228 // enable gather with a -march.
7229 if (ST->hasAVX512() || (ST->hasAVX2() && ST->hasFastGather()))
7230 return 2;
7231
7232 return 1024;
7233}
7234
7235int X86TTIImpl::getScatterOverhead() const {
7236 if (ST->hasAVX512())
7237 return 2;
7238
7239 return 1024;
7240}
7241
7242// Return an average cost of Gather / Scatter instruction, maybe improved later.
7243InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
7244 TTI::TargetCostKind CostKind,
7245 Type *SrcVTy, const Value *Ptr,
7246 Align Alignment,
7247 unsigned AddressSpace) const {
7248
7249 assert(isa<VectorType>(SrcVTy) && "Unexpected type in getGSVectorCost");
7250 unsigned VF = cast<FixedVectorType>(SrcVTy)->getNumElements();
7251
7252 // Try to reduce index size from 64 bit (default for GEP)
7253 // to 32. It is essential for VF 16. If the index can't be reduced to 32, the
7254 // operation will use 16 x 64 indices which do not fit in a zmm and needs
7255 // to split. Also check that the base pointer is the same for all lanes,
7256 // and that there's at most one variable index.
7257 auto getIndexSizeInBits = [](const Value *Ptr, const DataLayout &DL) {
7258 unsigned IndexSize = DL.getPointerSizeInBits();
7259 const GetElementPtrInst *GEP = dyn_cast_or_null<GetElementPtrInst>(Ptr);
7260 if (IndexSize < 64 || !GEP)
7261 return IndexSize;
7262
7263 unsigned NumOfVarIndices = 0;
7264 const Value *Ptrs = GEP->getPointerOperand();
7265 if (Ptrs->getType()->isVectorTy() && !getSplatValue(Ptrs))
7266 return IndexSize;
7267 for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I) {
7268 if (isa<Constant>(GEP->getOperand(I)))
7269 continue;
7270 Type *IndxTy = GEP->getOperand(I)->getType();
7271 if (auto *IndexVTy = dyn_cast<VectorType>(IndxTy))
7272 IndxTy = IndexVTy->getElementType();
7273 if ((IndxTy->getPrimitiveSizeInBits() == 64 &&
7274 !isa<SExtInst>(GEP->getOperand(I))) ||
7275 ++NumOfVarIndices > 1)
7276 return IndexSize; // 64
7277 }
7278 return (unsigned)32;
7279 };
7280
7281 // Trying to reduce IndexSize to 32 bits for vector 16.
7282 // By default the IndexSize is equal to pointer size.
7283 unsigned IndexSize = (ST->hasAVX512() && VF >= 16)
7284 ? getIndexSizeInBits(Ptr, DL)
7285 : DL.getPointerSizeInBits();
7286
7287 auto *IndexVTy = FixedVectorType::get(
7288 IntegerType::get(SrcVTy->getContext(), IndexSize), VF);
7289 std::pair<InstructionCost, MVT> IdxsLT = getTypeLegalizationCost(IndexVTy);
7290 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(SrcVTy);
7291 InstructionCost::CostType SplitFactor =
7292 std::max(IdxsLT.first, SrcLT.first).getValue();
7293 if (SplitFactor > 1) {
7294 // Handle splitting of vector of pointers
7295 auto *SplitSrcTy =
7296 FixedVectorType::get(SrcVTy->getScalarType(), VF / SplitFactor);
7297 return SplitFactor * getGSVectorCost(Opcode, CostKind, SplitSrcTy, Ptr,
7298 Alignment, AddressSpace);
7299 }
7300
7301 // If we didn't split, this will be a single gather/scatter instruction.
7302 if (CostKind == TTI::TCK_CodeSize)
7303 return 1;
7304
7305 // The gather / scatter cost is given by Intel architects. It is a rough
7306 // number since we are looking at one instruction in a time.
7307 const int GSOverhead = (Opcode == Instruction::Load) ? getGatherOverhead()
7308 : getScatterOverhead();
7309 return GSOverhead + VF * getMemoryOpCost(Opcode, SrcVTy->getScalarType(),
7310 Alignment, AddressSpace, CostKind);
7311}
7312
7313/// Calculate the cost of Gather / Scatter operation
7316 TTI::TargetCostKind CostKind) const {
7317 bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
7318 MICA.getID() == Intrinsic::vp_gather;
7319 unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
7320 Type *SrcVTy = MICA.getDataType();
7321 const Value *Ptr = MICA.getPointer();
7322 Align Alignment = MICA.getAlignment();
7323 if ((Opcode == Instruction::Load &&
7324 (!isLegalMaskedGather(SrcVTy, Align(Alignment)) ||
7326 Align(Alignment)))) ||
7327 (Opcode == Instruction::Store &&
7328 (!isLegalMaskedScatter(SrcVTy, Align(Alignment)) ||
7330 Align(Alignment)))))
7331 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
7332
7333 assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
7334 unsigned AddressSpace = MICA.getAddressSpace();
7336 getGSVectorCost(Opcode, CostKind, SrcVTy, Ptr, Alignment, AddressSpace);
7337
7338 // Predicate fanout (see getPredicateFanoutCost); variable mask only.
7339 if (MICA.getVariableMask()) {
7340 auto *MaskVecTy =
7342 cast<FixedVectorType>(SrcVTy)->getNumElements());
7343 Cost += getPredicateFanoutCost(*this, SrcVTy, MaskVecTy);
7344 }
7345 return Cost;
7346}
7347
7349 const TargetTransformInfo::LSRCost &C2) const {
7350 // X86 specific here are "instruction number 1st priority".
7351 return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost, C1.NumIVMuls,
7352 C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) <
7353 std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost, C2.NumIVMuls,
7354 C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost);
7355}
7356
7358 return ST->hasMacroFusion() || ST->hasBranchFusion();
7359}
7360
7361static bool isLegalMaskedLoadStore(Type *ScalarTy, const X86Subtarget *ST) {
7362 if (!ST->hasAVX())
7363 return false;
7364
7365 if (ScalarTy->isPointerTy())
7366 return true;
7367
7368 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7369 return true;
7370
7371 if ((ScalarTy->isHalfTy() || ScalarTy->isBFloatTy()) && ST->hasBWI())
7372 return true;
7373
7374 if (!ScalarTy->isIntegerTy())
7375 return false;
7376
7377 unsigned IntWidth = ScalarTy->getIntegerBitWidth();
7378 return IntWidth == 32 || IntWidth == 64 ||
7379 ((IntWidth == 8 || IntWidth == 16) && ST->hasBWI());
7380}
7381
7383 unsigned AddressSpace,
7384 TTI::MaskKind MaskKind) const {
7385 Type *ScalarTy = DataTy->getScalarType();
7386
7387 // The backend can't handle a single element vector w/o CFCMOV.
7388 if (isa<VectorType>(DataTy) &&
7389 cast<FixedVectorType>(DataTy)->getNumElements() == 1)
7390 return ST->hasCF() &&
7391 hasConditionalLoadStoreForType(ScalarTy, /*IsStore=*/false);
7392
7393 return isLegalMaskedLoadStore(ScalarTy, ST);
7394}
7395
7397 unsigned AddressSpace,
7398 TTI::MaskKind MaskKind) const {
7399 Type *ScalarTy = DataTy->getScalarType();
7400
7401 // The backend can't handle a single element vector w/o CFCMOV.
7402 if (isa<VectorType>(DataTy) &&
7403 cast<FixedVectorType>(DataTy)->getNumElements() == 1)
7404 return ST->hasCF() &&
7405 hasConditionalLoadStoreForType(ScalarTy, /*IsStore=*/true);
7406
7407 return isLegalMaskedLoadStore(ScalarTy, ST);
7408}
7409
7410bool X86TTIImpl::isLegalNTLoad(Type *DataType, Align Alignment) const {
7411 unsigned DataSize = DL.getTypeStoreSize(DataType);
7412 // The only supported nontemporal loads are for aligned vectors of 16 or 32
7413 // bytes. Note that 32-byte nontemporal vector loads are supported by AVX2
7414 // (the equivalent stores only require AVX).
7415 if (Alignment >= DataSize && (DataSize == 16 || DataSize == 32))
7416 return DataSize == 16 ? ST->hasSSE1() : ST->hasAVX2();
7417
7418 return false;
7419}
7420
7421bool X86TTIImpl::isLegalNTStore(Type *DataType, Align Alignment) const {
7422 unsigned DataSize = DL.getTypeStoreSize(DataType);
7423
7424 // SSE4A supports nontemporal stores of float and double at arbitrary
7425 // alignment.
7426 if (ST->hasSSE4A() && (DataType->isFloatTy() || DataType->isDoubleTy()))
7427 return true;
7428
7429 // Besides the SSE4A subtarget exception above, only aligned stores are
7430 // available nontemporaly on any other subtarget. And only stores with a size
7431 // of 4..32 bytes (powers of 2, only) are permitted.
7432 if (Alignment < DataSize || DataSize < 4 || DataSize > 32 ||
7433 !isPowerOf2_32(DataSize))
7434 return false;
7435
7436 // 32-byte vector nontemporal stores are supported by AVX (the equivalent
7437 // loads require AVX2).
7438 if (DataSize == 32)
7439 return ST->hasAVX();
7440 if (DataSize == 16)
7441 return ST->hasSSE1();
7442 return true;
7443}
7444
7446 ElementCount NumElements) const {
7447 // movddup
7448 return ST->hasSSE3() && !NumElements.isScalable() &&
7449 NumElements.getFixedValue() == 2 &&
7450 ElementTy == Type::getDoubleTy(ElementTy->getContext());
7451}
7452
7453bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) const {
7454 if (!isa<VectorType>(DataTy))
7455 return false;
7456
7457 if (!ST->hasAVX512())
7458 return false;
7459
7460 // The backend can't handle a single element vector.
7461 if (cast<FixedVectorType>(DataTy)->getNumElements() == 1)
7462 return false;
7463
7464 Type *ScalarTy = cast<VectorType>(DataTy)->getElementType();
7465
7466 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7467 return true;
7468
7469 if (ScalarTy->isHalfTy() || ScalarTy->isBFloatTy())
7470 return ST->hasVBMI2();
7471
7472 if (!ScalarTy->isIntegerTy())
7473 return false;
7474
7475 unsigned IntWidth = ScalarTy->getIntegerBitWidth();
7476 return IntWidth == 32 || IntWidth == 64 ||
7477 ((IntWidth == 8 || IntWidth == 16) && ST->hasVBMI2());
7478}
7479
7481 Align Alignment) const {
7482 return isLegalMaskedExpandLoad(DataTy, Alignment);
7483}
7484
7485bool X86TTIImpl::supportsGather() const {
7486 // Some CPUs have better gather performance than others.
7487 // TODO: Remove the explicit ST->hasAVX512()?, That would mean we would only
7488 // enable gather with a -march.
7489 return ST->hasAVX512() || (ST->hasFastGather() && ST->hasAVX2());
7490}
7491
7493 Align Alignment) const {
7494 // Gather / Scatter for vector 2 is not profitable on KNL / SKX
7495 // Vector-4 of gather/scatter instruction does not exist on KNL. We can extend
7496 // it to 8 elements, but zeroing upper bits of the mask vector will add more
7497 // instructions. Right now we give the scalar cost of vector-4 for KNL. TODO:
7498 // Check, maybe the gather/scatter instruction is better in the VariableMask
7499 // case.
7500 unsigned NumElts = cast<FixedVectorType>(VTy)->getNumElements();
7501 return NumElts == 1 ||
7502 (ST->hasAVX512() && (NumElts == 2 || (NumElts == 4 && !ST->hasVLX())));
7503}
7504
7506 Align Alignment) const {
7507 Type *ScalarTy = DataTy->getScalarType();
7508 if (ScalarTy->isPointerTy())
7509 return true;
7510
7511 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7512 return true;
7513
7514 if (!ScalarTy->isIntegerTy())
7515 return false;
7516
7517 unsigned IntWidth = ScalarTy->getIntegerBitWidth();
7518 return IntWidth == 32 || IntWidth == 64;
7519}
7520
7521bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, Align Alignment) const {
7522 if (!supportsGather() || !ST->preferGather())
7523 return false;
7524 return isLegalMaskedGatherScatter(DataTy, Alignment);
7525}
7526
7527bool X86TTIImpl::isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
7528 unsigned Opcode1,
7529 const SmallBitVector &OpcodeMask,
7530 ArrayRef<const Value *> Scalars) const {
7531 // ADDSUBPS 4xf32 SSE3
7532 // VADDSUBPS 4xf32 AVX
7533 // VADDSUBPS 8xf32 AVX2
7534 // ADDSUBPD 2xf64 SSE3
7535 // VADDSUBPD 2xf64 AVX
7536 // VADDSUBPD 4xf64 AVX2
7537
7538 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
7539 assert(OpcodeMask.size() == NumElements && "Mask and VecTy are incompatible");
7540 if (!isPowerOf2_32(NumElements))
7541 return false;
7542 // Check the opcode pattern. We apply the mask on the opcode arguments and
7543 // then check if it is what we expect.
7544 auto IsLaneOrder = [&](unsigned EvenOpc, unsigned OddOpc) {
7545 return all_of(seq<unsigned>(0, NumElements), [&](unsigned Lane) {
7546 return (OpcodeMask.test(Lane) ? Opcode1 : Opcode0) ==
7547 (Lane % 2 == 0 ? EvenOpc : OddOpc);
7548 });
7549 };
7550 // We expect FSub for even lanes and FAdd for odd lanes.
7551 const bool IsAddSub = IsLaneOrder(Instruction::FSub, Instruction::FAdd);
7552 // Now check that the pattern is supported by the target ISA.
7553 Type *ElemTy = cast<VectorType>(VecTy)->getElementType();
7554 if (IsAddSub && ST->hasSSE3() &&
7555 ((ElemTy->isFloatTy() && NumElements % 4 == 0) ||
7556 (ElemTy->isDoubleTy() && NumElements % 2 == 0)))
7557 return true;
7558 // The multiplication is fused into every lane: fmaddsub (the even lanes
7559 // subtract and the odd lanes add the product) or fmsubadd (the other way
7560 // round). The subtracted product is the first operand of the subtraction.
7561 using namespace PatternMatch;
7562 auto IsFusedWithFMul = [](const Value *V) {
7563 const auto *I = dyn_cast<Instruction>(V);
7564 if (!I || !I->hasAllowContract())
7565 return false;
7566 auto IsFMul = [&](const Value *Op) {
7567 return match(Op,
7569 cast<Instruction>(Op)->getParent() == I->getParent();
7570 };
7571 return IsFMul(I->getOperand(0)) ||
7572 (I->getOpcode() == Instruction::FAdd && IsFMul(I->getOperand(1)));
7573 };
7574 return ST->hasFMA() && !Scalars.empty() &&
7575 (ElemTy->isFloatTy() || ElemTy->isDoubleTy()) &&
7576 (IsAddSub || IsLaneOrder(Instruction::FAdd, Instruction::FSub)) &&
7577 all_of(Scalars, IsFusedWithFMul);
7578}
7579
7580bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, Align Alignment) const {
7581 // AVX2 doesn't support scatter
7582 if (!ST->hasAVX512() || !ST->preferScatter())
7583 return false;
7584 return isLegalMaskedGatherScatter(DataType, Alignment);
7585}
7586
7587bool X86TTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) const {
7588 EVT VT = TLI->getValueType(DL, DataType);
7589 return TLI->isOperationLegal(IsSigned ? ISD::SDIVREM : ISD::UDIVREM, VT);
7590}
7591
7593 // FDIV is always expensive, even if it has a very low uop count.
7594 // TODO: Still necessary for recent CPUs with low latency/throughput fdiv?
7595 if (I->getOpcode() == Instruction::FDiv)
7596 return true;
7597
7599}
7600
7601bool X86TTIImpl::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { return false; }
7602
7604 const Function *Callee) const {
7605 const TargetMachine &TM = getTLI()->getTargetMachine();
7606
7607 // Work this as a subsetting of subtarget features.
7608 const X86Subtarget &CallerSubtarget = TM.getSubtarget<X86Subtarget>(*Caller);
7609 const X86Subtarget &CalleeSubtarget = TM.getSubtarget<X86Subtarget>(*Callee);
7610 const FeatureBitset &CallerBits = CallerSubtarget.getFeatureBits();
7611 const FeatureBitset &CalleeBits = CalleeSubtarget.getFeatureBits();
7612
7613 // Check whether callee features are a subset of caller features
7614 // (apart from the ignore list).
7615 const FeatureBitset &InlineIgnoreFeatures =
7616 CallerSubtarget.getInlineIgnoreFeatures();
7617 FeatureBitset RealCallerBits = CallerBits & ~InlineIgnoreFeatures;
7618 FeatureBitset RealCalleeBits = CalleeBits & ~InlineIgnoreFeatures;
7619 if ((RealCallerBits & RealCalleeBits) != RealCalleeBits)
7620 return false;
7621
7622 // If the features are not exactly the same (or there is a difference in
7623 // AVX512 register usage), we need to additionally check for calls
7624 // that may become ABI-incompatible as a result of inlining.
7625 if (RealCallerBits == RealCalleeBits &&
7626 CallerSubtarget.useAVX512Regs() == CalleeSubtarget.useAVX512Regs())
7627 return true;
7628
7629 for (const Instruction &I : instructions(Callee)) {
7630 if (const auto *CB = dyn_cast<CallBase>(&I)) {
7631 // Having more target features is fine for inline ASM and intrinsics.
7632 if (CB->isInlineAsm() || CB->getIntrinsicID() != Intrinsic::not_intrinsic)
7633 continue;
7634
7636 for (Value *Arg : CB->args())
7637 Types.push_back(Arg->getType());
7638 if (!CB->getType()->isVoidTy())
7639 Types.push_back(CB->getType());
7640
7641 // Simple types are always ABI compatible.
7642 auto IsSimpleTy = [](Type *Ty) {
7643 return !Ty->isVectorTy() && !Ty->isAggregateType();
7644 };
7645 if (all_of(Types, IsSimpleTy))
7646 continue;
7647
7648 // Do a precise compatibility check.
7649 if (!areTypesABICompatible(Caller, Callee, Types))
7650 return false;
7651 }
7652 }
7653 return true;
7654}
7655
7657 const Function *Callee,
7658 ArrayRef<Type *> Types) const {
7659 const TargetMachine &TM = getTLI()->getTargetMachine();
7660 const TargetLowering *CallerTLI =
7661 TM.getSubtargetImpl(*Caller)->getTargetLowering();
7662 const TargetLowering *CalleeTLI =
7663 TM.getSubtargetImpl(*Callee)->getTargetLowering();
7664
7665 LLVMContext &Ctx = Caller->getContext();
7666 const DataLayout &DL = Caller->getDataLayout();
7667 CallingConv::ID CC = Callee->getCallingConv();
7668 return all_of(Types, [&](Type *Ty) {
7669 SmallVector<EVT> VTs;
7670 ComputeValueVTs(*CallerTLI, DL, Ty, VTs);
7671 return all_of(VTs, [&](EVT VT) {
7672 return CallerTLI->getRegisterTypeForCallingConv(Ctx, CC, VT) ==
7673 CalleeTLI->getRegisterTypeForCallingConv(Ctx, CC, VT);
7674 });
7675 });
7676}
7677
7679X86TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
7681 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
7682 Options.NumLoadsPerBlock = IsZeroCmp ? 2 : 1;
7683 // All GPR and vector loads can be unaligned.
7684 Options.AllowOverlappingLoads = true;
7685 if (IsZeroCmp) {
7686 // Only enable vector loads for equality comparison. Right now the vector
7687 // version is not as fast for three way compare (see #33329).
7688 const unsigned PreferredWidth = ST->getPreferVectorWidth();
7689 if (PreferredWidth >= 512 && ST->hasAVX512())
7690 Options.LoadSizes.push_back(64);
7691 if (PreferredWidth >= 256 && ST->hasAVX()) Options.LoadSizes.push_back(32);
7692 if (PreferredWidth >= 128 && ST->hasSSE2()) Options.LoadSizes.push_back(16);
7693 }
7694 if (ST->is64Bit()) {
7695 Options.LoadSizes.push_back(8);
7696 }
7697 Options.LoadSizes.push_back(4);
7698 Options.LoadSizes.push_back(2);
7699 Options.LoadSizes.push_back(1);
7700 return Options;
7701}
7702
7704 return supportsGather();
7705}
7706
7708 return false;
7709}
7710
7712 // TODO: We expect this to be beneficial regardless of arch,
7713 // but there are currently some unexplained performance artifacts on Atom.
7714 // As a temporary solution, disable on Atom.
7715 return !(ST->isAtom());
7716}
7717
7719 switch (II->getIntrinsicID()) {
7720 default:
7721 return true;
7722 case Intrinsic::vector_reduce_and:
7723 case Intrinsic::vector_reduce_or:
7724 case Intrinsic::vector_reduce_xor:
7725 case Intrinsic::vector_reduce_mul:
7726 case Intrinsic::vector_reduce_smax:
7727 case Intrinsic::vector_reduce_smin:
7728 case Intrinsic::vector_reduce_umax:
7729 case Intrinsic::vector_reduce_umin:
7730 return false;
7731 }
7732}
7733
7734// Get estimation for interleaved load/store operations and strided load.
7735// \p Indices contains indices for strided load.
7736// \p Factor - the factor of interleaving.
7737// AVX-512 provides 3-src shuffles that significantly reduces the cost.
7739 unsigned Opcode, FixedVectorType *VecTy, unsigned Factor,
7740 ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace,
7741 TTI::TargetCostKind CostKind, bool UseMaskForCond,
7742 bool UseMaskForGaps) const {
7743 // VecTy for interleave memop is <VF*Factor x Elt>.
7744 // So, for VF=4, Interleave Factor = 3, Element type = i32 we have
7745 // VecTy = <12 x i32>.
7746
7747 // Calculate the number of memory operations (NumOfMemOps), required
7748 // for load/store the VecTy.
7749 MVT LegalVT = getTypeLegalizationCost(VecTy).second;
7750 unsigned VecTySize = DL.getTypeStoreSize(VecTy);
7751 unsigned LegalVTSize = LegalVT.getStoreSize();
7752 unsigned NumOfMemOps = (VecTySize + LegalVTSize - 1) / LegalVTSize;
7753
7754 // Get the cost of one memory operation.
7755 auto *SingleMemOpTy = FixedVectorType::get(VecTy->getElementType(),
7756 LegalVT.getVectorNumElements());
7757 InstructionCost MemOpCost;
7758 bool UseMaskedMemOp = UseMaskForCond || UseMaskForGaps;
7759 if (UseMaskedMemOp) {
7760 unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
7761 : Intrinsic::masked_store;
7762 MemOpCost = getMaskedMemoryOpCost(
7763 {IID, SingleMemOpTy, Alignment, AddressSpace}, CostKind);
7764 } else
7765 MemOpCost = getMemoryOpCost(Opcode, SingleMemOpTy, Alignment, AddressSpace,
7766 CostKind);
7767
7768 unsigned VF = VecTy->getNumElements() / Factor;
7769 MVT VT =
7770 MVT::getVectorVT(TLI->getSimpleValueType(DL, VecTy->getScalarType()), VF);
7771
7772 InstructionCost MaskCost;
7773 if (UseMaskedMemOp) {
7774 APInt DemandedLoadStoreElts = APInt::getZero(VecTy->getNumElements());
7775 for (unsigned Index : Indices) {
7776 assert(Index < Factor && "Invalid index for interleaved memory op");
7777 for (unsigned Elm = 0; Elm < VF; Elm++)
7778 DemandedLoadStoreElts.setBit(Index + Elm * Factor);
7779 }
7780
7781 Type *I1Type = Type::getInt1Ty(VecTy->getContext());
7782
7783 MaskCost = getReplicationShuffleCost(
7784 I1Type, Factor, VF,
7785 UseMaskForGaps ? DemandedLoadStoreElts
7787 CostKind);
7788
7789 // The Gaps mask is invariant and created outside the loop, therefore the
7790 // cost of creating it is not accounted for here. However if we have both
7791 // a MaskForGaps and some other mask that guards the execution of the
7792 // memory access, we need to account for the cost of And-ing the two masks
7793 // inside the loop.
7794 if (UseMaskForGaps) {
7795 auto *MaskVT = FixedVectorType::get(I1Type, VecTy->getNumElements());
7796 MaskCost += getArithmeticInstrCost(BinaryOperator::And, MaskVT, CostKind);
7797 }
7798 }
7799
7800 if (Opcode == Instruction::Load) {
7801 // The tables (AVX512InterleavedLoadTbl and AVX512InterleavedStoreTbl)
7802 // contain the cost of the optimized shuffle sequence that the
7803 // X86InterleavedAccess pass will generate.
7804 // The cost of loads and stores are computed separately from the table.
7805
7806 // X86InterleavedAccess support only the following interleaved-access group.
7807 static const CostTblEntry AVX512InterleavedLoadTbl[] = {
7808 {3, MVT::v16i8, 12}, //(load 48i8 and) deinterleave into 3 x 16i8
7809 {3, MVT::v32i8, 14}, //(load 96i8 and) deinterleave into 3 x 32i8
7810 {3, MVT::v64i8, 22}, //(load 96i8 and) deinterleave into 3 x 32i8
7811 };
7812
7813 if (const auto *Entry =
7814 CostTableLookup(AVX512InterleavedLoadTbl, Factor, VT))
7815 return MaskCost + NumOfMemOps * MemOpCost + Entry->Cost;
7816 //If an entry does not exist, fallback to the default implementation.
7817
7818 // Kind of shuffle depends on number of loaded values.
7819 // If we load the entire data in one register, we can use a 1-src shuffle.
7820 // Otherwise, we'll merge 2 sources in each operation.
7821 TTI::ShuffleKind ShuffleKind =
7822 (NumOfMemOps > 1) ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc;
7823
7824 InstructionCost ShuffleCost = getShuffleCost(
7825 ShuffleKind, SingleMemOpTy, SingleMemOpTy, CostKind, {}, 0, nullptr);
7826
7827 unsigned NumOfLoadsInInterleaveGrp =
7828 Indices.size() ? Indices.size() : Factor;
7829 auto *ResultTy = FixedVectorType::get(VecTy->getElementType(),
7830 VecTy->getNumElements() / Factor);
7831 InstructionCost NumOfResults =
7832 getTypeLegalizationCost(ResultTy).first * NumOfLoadsInInterleaveGrp;
7833
7834 // About a half of the loads may be folded in shuffles when we have only
7835 // one result. If we have more than one result, or the loads are masked,
7836 // we do not fold loads at all.
7837 unsigned NumOfUnfoldedLoads =
7838 UseMaskedMemOp || NumOfResults > 1 ? NumOfMemOps : NumOfMemOps / 2;
7839
7840 // Get a number of shuffle operations per result.
7841 unsigned NumOfShufflesPerResult =
7842 std::max((unsigned)1, (unsigned)(NumOfMemOps - 1));
7843
7844 // The SK_MergeTwoSrc shuffle clobbers one of src operands.
7845 // When we have more than one destination, we need additional instructions
7846 // to keep sources.
7847 InstructionCost NumOfMoves = 0;
7848 if (NumOfResults > 1 && ShuffleKind == TTI::SK_PermuteTwoSrc)
7849 NumOfMoves = NumOfResults * NumOfShufflesPerResult / 2;
7850
7851 InstructionCost Cost = NumOfResults * NumOfShufflesPerResult * ShuffleCost +
7852 MaskCost + NumOfUnfoldedLoads * MemOpCost +
7853 NumOfMoves;
7854
7855 return Cost;
7856 }
7857
7858 // Store.
7859 assert(Opcode == Instruction::Store &&
7860 "Expected Store Instruction at this point");
7861 // X86InterleavedAccess support only the following interleaved-access group.
7862 static const CostTblEntry AVX512InterleavedStoreTbl[] = {
7863 {3, MVT::v16i8, 12}, // interleave 3 x 16i8 into 48i8 (and store)
7864 {3, MVT::v32i8, 14}, // interleave 3 x 32i8 into 96i8 (and store)
7865 {3, MVT::v64i8, 26}, // interleave 3 x 64i8 into 96i8 (and store)
7866
7867 {4, MVT::v8i8, 10}, // interleave 4 x 8i8 into 32i8 (and store)
7868 {4, MVT::v16i8, 11}, // interleave 4 x 16i8 into 64i8 (and store)
7869 {4, MVT::v32i8, 14}, // interleave 4 x 32i8 into 128i8 (and store)
7870 {4, MVT::v64i8, 24} // interleave 4 x 32i8 into 256i8 (and store)
7871 };
7872
7873 if (const auto *Entry =
7874 CostTableLookup(AVX512InterleavedStoreTbl, Factor, VT))
7875 return MaskCost + NumOfMemOps * MemOpCost + Entry->Cost;
7876 //If an entry does not exist, fallback to the default implementation.
7877
7878 // There is no strided stores meanwhile. And store can't be folded in
7879 // shuffle.
7880 unsigned NumOfSources = Factor; // The number of values to be merged.
7881 InstructionCost ShuffleCost =
7882 getShuffleCost(TTI::SK_PermuteTwoSrc, SingleMemOpTy, SingleMemOpTy,
7883 CostKind, {}, 0, nullptr);
7884 unsigned NumOfShufflesPerStore = NumOfSources - 1;
7885
7886 // The SK_MergeTwoSrc shuffle clobbers one of src operands.
7887 // We need additional instructions to keep sources.
7888 unsigned NumOfMoves = NumOfMemOps * NumOfShufflesPerStore / 2;
7890 MaskCost +
7891 NumOfMemOps * (MemOpCost + NumOfShufflesPerStore * ShuffleCost) +
7892 NumOfMoves;
7893 return Cost;
7894}
7895
7897 unsigned Opcode, Type *BaseTy, unsigned Factor, ArrayRef<unsigned> Indices,
7898 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
7899 bool UseMaskForCond, bool UseMaskForGaps) const {
7900 auto *VecTy = cast<FixedVectorType>(BaseTy);
7901
7902 auto isSupportedOnAVX512 = [&](Type *VecTy) {
7903 Type *EltTy = cast<VectorType>(VecTy)->getElementType();
7904 if (EltTy->isFloatTy() || EltTy->isDoubleTy() || EltTy->isIntegerTy(64) ||
7905 EltTy->isIntegerTy(32) || EltTy->isPointerTy())
7906 return true;
7907 if (EltTy->isIntegerTy(16) || EltTy->isIntegerTy(8) || EltTy->isHalfTy())
7908 return ST->hasBWI();
7909 if (EltTy->isBFloatTy())
7910 return ST->hasBF16();
7911 return false;
7912 };
7913 if (ST->hasAVX512() && isSupportedOnAVX512(VecTy))
7915 Opcode, VecTy, Factor, Indices, Alignment,
7916 AddressSpace, CostKind, UseMaskForCond, UseMaskForGaps);
7917
7918 if (UseMaskForCond || UseMaskForGaps)
7919 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
7920 Alignment, AddressSpace, CostKind,
7921 UseMaskForCond, UseMaskForGaps);
7922
7923 // Get estimation for interleaved load/store operations for SSE-AVX2.
7924 // As opposed to AVX-512, SSE-AVX2 do not have generic shuffles that allow
7925 // computing the cost using a generic formula as a function of generic
7926 // shuffles. We therefore use a lookup table instead, filled according to
7927 // the instruction sequences that codegen currently generates.
7928
7929 // VecTy for interleave memop is <VF*Factor x Elt>.
7930 // So, for VF=4, Interleave Factor = 3, Element type = i32 we have
7931 // VecTy = <12 x i32>.
7932 MVT LegalVT = getTypeLegalizationCost(VecTy).second;
7933
7934 // This function can be called with VecTy=<6xi128>, Factor=3, in which case
7935 // the VF=2, while v2i128 is an unsupported MVT vector type
7936 // (see MachineValueType.h::getVectorVT()).
7937 if (!LegalVT.isVector())
7938 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
7939 Alignment, AddressSpace, CostKind);
7940
7941 unsigned VF = VecTy->getNumElements() / Factor;
7942 Type *ScalarTy = VecTy->getElementType();
7943 // Deduplicate entries, model floats/pointers as appropriately-sized integers.
7944 if (!ScalarTy->isIntegerTy())
7945 ScalarTy =
7946 Type::getIntNTy(ScalarTy->getContext(), DL.getTypeSizeInBits(ScalarTy));
7947
7948 // Get the cost of all the memory operations.
7949 // FIXME: discount dead loads.
7950 InstructionCost MemOpCosts =
7951 getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind);
7952
7953 auto *VT = FixedVectorType::get(ScalarTy, VF);
7954 EVT ETy = TLI->getValueType(DL, VT);
7955 if (!ETy.isSimple())
7956 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
7957 Alignment, AddressSpace, CostKind);
7958
7959 // TODO: Complete for other data-types and strides.
7960 // Each combination of Stride, element bit width and VF results in a different
7961 // sequence; The cost tables are therefore accessed with:
7962 // Factor (stride) and VectorType=VFxiN.
7963 // The Cost accounts only for the shuffle sequence;
7964 // The cost of the loads/stores is accounted for separately.
7965 //
7966 static const CostTblEntry AVX2InterleavedLoadTbl[] = {
7967 {2, MVT::v2i8, 2}, // (load 4i8 and) deinterleave into 2 x 2i8
7968 {2, MVT::v4i8, 2}, // (load 8i8 and) deinterleave into 2 x 4i8
7969 {2, MVT::v8i8, 2}, // (load 16i8 and) deinterleave into 2 x 8i8
7970 {2, MVT::v16i8, 4}, // (load 32i8 and) deinterleave into 2 x 16i8
7971 {2, MVT::v32i8, 6}, // (load 64i8 and) deinterleave into 2 x 32i8
7972
7973 {2, MVT::v8i16, 6}, // (load 16i16 and) deinterleave into 2 x 8i16
7974 {2, MVT::v16i16, 9}, // (load 32i16 and) deinterleave into 2 x 16i16
7975 {2, MVT::v32i16, 18}, // (load 64i16 and) deinterleave into 2 x 32i16
7976
7977 {2, MVT::v8i32, 4}, // (load 16i32 and) deinterleave into 2 x 8i32
7978 {2, MVT::v16i32, 8}, // (load 32i32 and) deinterleave into 2 x 16i32
7979 {2, MVT::v32i32, 16}, // (load 64i32 and) deinterleave into 2 x 32i32
7980
7981 {2, MVT::v4i64, 4}, // (load 8i64 and) deinterleave into 2 x 4i64
7982 {2, MVT::v8i64, 8}, // (load 16i64 and) deinterleave into 2 x 8i64
7983 {2, MVT::v16i64, 16}, // (load 32i64 and) deinterleave into 2 x 16i64
7984 {2, MVT::v32i64, 32}, // (load 64i64 and) deinterleave into 2 x 32i64
7985
7986 {3, MVT::v2i8, 3}, // (load 6i8 and) deinterleave into 3 x 2i8
7987 {3, MVT::v4i8, 3}, // (load 12i8 and) deinterleave into 3 x 4i8
7988 {3, MVT::v8i8, 6}, // (load 24i8 and) deinterleave into 3 x 8i8
7989 {3, MVT::v16i8, 11}, // (load 48i8 and) deinterleave into 3 x 16i8
7990 {3, MVT::v32i8, 14}, // (load 96i8 and) deinterleave into 3 x 32i8
7991
7992 {3, MVT::v2i16, 5}, // (load 6i16 and) deinterleave into 3 x 2i16
7993 {3, MVT::v4i16, 7}, // (load 12i16 and) deinterleave into 3 x 4i16
7994 {3, MVT::v8i16, 9}, // (load 24i16 and) deinterleave into 3 x 8i16
7995 {3, MVT::v16i16, 28}, // (load 48i16 and) deinterleave into 3 x 16i16
7996 {3, MVT::v32i16, 56}, // (load 96i16 and) deinterleave into 3 x 32i16
7997
7998 {3, MVT::v2i32, 3}, // (load 6i32 and) deinterleave into 3 x 2i32
7999 {3, MVT::v4i32, 3}, // (load 12i32 and) deinterleave into 3 x 4i32
8000 {3, MVT::v8i32, 7}, // (load 24i32 and) deinterleave into 3 x 8i32
8001 {3, MVT::v16i32, 14}, // (load 48i32 and) deinterleave into 3 x 16i32
8002 {3, MVT::v32i32, 32}, // (load 96i32 and) deinterleave into 3 x 32i32
8003
8004 {3, MVT::v2i64, 1}, // (load 6i64 and) deinterleave into 3 x 2i64
8005 {3, MVT::v4i64, 5}, // (load 12i64 and) deinterleave into 3 x 4i64
8006 {3, MVT::v8i64, 10}, // (load 24i64 and) deinterleave into 3 x 8i64
8007 {3, MVT::v16i64, 20}, // (load 48i64 and) deinterleave into 3 x 16i64
8008
8009 {4, MVT::v2i8, 4}, // (load 8i8 and) deinterleave into 4 x 2i8
8010 {4, MVT::v4i8, 4}, // (load 16i8 and) deinterleave into 4 x 4i8
8011 {4, MVT::v8i8, 12}, // (load 32i8 and) deinterleave into 4 x 8i8
8012 {4, MVT::v16i8, 24}, // (load 64i8 and) deinterleave into 4 x 16i8
8013 {4, MVT::v32i8, 56}, // (load 128i8 and) deinterleave into 4 x 32i8
8014
8015 {4, MVT::v2i16, 6}, // (load 8i16 and) deinterleave into 4 x 2i16
8016 {4, MVT::v4i16, 17}, // (load 16i16 and) deinterleave into 4 x 4i16
8017 {4, MVT::v8i16, 33}, // (load 32i16 and) deinterleave into 4 x 8i16
8018 {4, MVT::v16i16, 75}, // (load 64i16 and) deinterleave into 4 x 16i16
8019 {4, MVT::v32i16, 150}, // (load 128i16 and) deinterleave into 4 x 32i16
8020
8021 {4, MVT::v2i32, 4}, // (load 8i32 and) deinterleave into 4 x 2i32
8022 {4, MVT::v4i32, 8}, // (load 16i32 and) deinterleave into 4 x 4i32
8023 {4, MVT::v8i32, 16}, // (load 32i32 and) deinterleave into 4 x 8i32
8024 {4, MVT::v16i32, 32}, // (load 64i32 and) deinterleave into 4 x 16i32
8025 {4, MVT::v32i32, 68}, // (load 128i32 and) deinterleave into 4 x 32i32
8026
8027 {4, MVT::v2i64, 6}, // (load 8i64 and) deinterleave into 4 x 2i64
8028 {4, MVT::v4i64, 8}, // (load 16i64 and) deinterleave into 4 x 4i64
8029 {4, MVT::v8i64, 20}, // (load 32i64 and) deinterleave into 4 x 8i64
8030 {4, MVT::v16i64, 40}, // (load 64i64 and) deinterleave into 4 x 16i64
8031
8032 {6, MVT::v2i8, 6}, // (load 12i8 and) deinterleave into 6 x 2i8
8033 {6, MVT::v4i8, 14}, // (load 24i8 and) deinterleave into 6 x 4i8
8034 {6, MVT::v8i8, 18}, // (load 48i8 and) deinterleave into 6 x 8i8
8035 {6, MVT::v16i8, 43}, // (load 96i8 and) deinterleave into 6 x 16i8
8036 {6, MVT::v32i8, 82}, // (load 192i8 and) deinterleave into 6 x 32i8
8037
8038 {6, MVT::v2i16, 13}, // (load 12i16 and) deinterleave into 6 x 2i16
8039 {6, MVT::v4i16, 9}, // (load 24i16 and) deinterleave into 6 x 4i16
8040 {6, MVT::v8i16, 39}, // (load 48i16 and) deinterleave into 6 x 8i16
8041 {6, MVT::v16i16, 106}, // (load 96i16 and) deinterleave into 6 x 16i16
8042 {6, MVT::v32i16, 212}, // (load 192i16 and) deinterleave into 6 x 32i16
8043
8044 {6, MVT::v2i32, 6}, // (load 12i32 and) deinterleave into 6 x 2i32
8045 {6, MVT::v4i32, 15}, // (load 24i32 and) deinterleave into 6 x 4i32
8046 {6, MVT::v8i32, 31}, // (load 48i32 and) deinterleave into 6 x 8i32
8047 {6, MVT::v16i32, 64}, // (load 96i32 and) deinterleave into 6 x 16i32
8048
8049 {6, MVT::v2i64, 6}, // (load 12i64 and) deinterleave into 6 x 2i64
8050 {6, MVT::v4i64, 18}, // (load 24i64 and) deinterleave into 6 x 4i64
8051 {6, MVT::v8i64, 36}, // (load 48i64 and) deinterleave into 6 x 8i64
8052
8053 {8, MVT::v8i32, 40} // (load 64i32 and) deinterleave into 8 x 8i32
8054 };
8055
8056 static const CostTblEntry SSSE3InterleavedLoadTbl[] = {
8057 {2, MVT::v4i16, 2}, // (load 8i16 and) deinterleave into 2 x 4i16
8058 };
8059
8060 static const CostTblEntry SSE2InterleavedLoadTbl[] = {
8061 {2, MVT::v2i16, 2}, // (load 4i16 and) deinterleave into 2 x 2i16
8062 {2, MVT::v4i16, 7}, // (load 8i16 and) deinterleave into 2 x 4i16
8063
8064 {2, MVT::v2i32, 2}, // (load 4i32 and) deinterleave into 2 x 2i32
8065 {2, MVT::v4i32, 2}, // (load 8i32 and) deinterleave into 2 x 4i32
8066
8067 {2, MVT::v2i64, 2}, // (load 4i64 and) deinterleave into 2 x 2i64
8068 };
8069
8070 static const CostTblEntry AVX2InterleavedStoreTbl[] = {
8071 {2, MVT::v16i8, 3}, // interleave 2 x 16i8 into 32i8 (and store)
8072 {2, MVT::v32i8, 4}, // interleave 2 x 32i8 into 64i8 (and store)
8073
8074 {2, MVT::v8i16, 3}, // interleave 2 x 8i16 into 16i16 (and store)
8075 {2, MVT::v16i16, 4}, // interleave 2 x 16i16 into 32i16 (and store)
8076 {2, MVT::v32i16, 8}, // interleave 2 x 32i16 into 64i16 (and store)
8077
8078 {2, MVT::v4i32, 2}, // interleave 2 x 4i32 into 8i32 (and store)
8079 {2, MVT::v8i32, 4}, // interleave 2 x 8i32 into 16i32 (and store)
8080 {2, MVT::v16i32, 8}, // interleave 2 x 16i32 into 32i32 (and store)
8081 {2, MVT::v32i32, 16}, // interleave 2 x 32i32 into 64i32 (and store)
8082
8083 {2, MVT::v2i64, 2}, // interleave 2 x 2i64 into 4i64 (and store)
8084 {2, MVT::v4i64, 4}, // interleave 2 x 4i64 into 8i64 (and store)
8085 {2, MVT::v8i64, 8}, // interleave 2 x 8i64 into 16i64 (and store)
8086 {2, MVT::v16i64, 16}, // interleave 2 x 16i64 into 32i64 (and store)
8087 {2, MVT::v32i64, 32}, // interleave 2 x 32i64 into 64i64 (and store)
8088
8089 {3, MVT::v2i8, 4}, // interleave 3 x 2i8 into 6i8 (and store)
8090 {3, MVT::v4i8, 4}, // interleave 3 x 4i8 into 12i8 (and store)
8091 {3, MVT::v8i8, 6}, // interleave 3 x 8i8 into 24i8 (and store)
8092 {3, MVT::v16i8, 11}, // interleave 3 x 16i8 into 48i8 (and store)
8093 {3, MVT::v32i8, 13}, // interleave 3 x 32i8 into 96i8 (and store)
8094
8095 {3, MVT::v2i16, 4}, // interleave 3 x 2i16 into 6i16 (and store)
8096 {3, MVT::v4i16, 6}, // interleave 3 x 4i16 into 12i16 (and store)
8097 {3, MVT::v8i16, 12}, // interleave 3 x 8i16 into 24i16 (and store)
8098 {3, MVT::v16i16, 27}, // interleave 3 x 16i16 into 48i16 (and store)
8099 {3, MVT::v32i16, 54}, // interleave 3 x 32i16 into 96i16 (and store)
8100
8101 {3, MVT::v2i32, 4}, // interleave 3 x 2i32 into 6i32 (and store)
8102 {3, MVT::v4i32, 5}, // interleave 3 x 4i32 into 12i32 (and store)
8103 {3, MVT::v8i32, 11}, // interleave 3 x 8i32 into 24i32 (and store)
8104 {3, MVT::v16i32, 22}, // interleave 3 x 16i32 into 48i32 (and store)
8105 {3, MVT::v32i32, 48}, // interleave 3 x 32i32 into 96i32 (and store)
8106
8107 {3, MVT::v2i64, 4}, // interleave 3 x 2i64 into 6i64 (and store)
8108 {3, MVT::v4i64, 6}, // interleave 3 x 4i64 into 12i64 (and store)
8109 {3, MVT::v8i64, 12}, // interleave 3 x 8i64 into 24i64 (and store)
8110 {3, MVT::v16i64, 24}, // interleave 3 x 16i64 into 48i64 (and store)
8111
8112 {4, MVT::v2i8, 4}, // interleave 4 x 2i8 into 8i8 (and store)
8113 {4, MVT::v4i8, 4}, // interleave 4 x 4i8 into 16i8 (and store)
8114 {4, MVT::v8i8, 4}, // interleave 4 x 8i8 into 32i8 (and store)
8115 {4, MVT::v16i8, 8}, // interleave 4 x 16i8 into 64i8 (and store)
8116 {4, MVT::v32i8, 12}, // interleave 4 x 32i8 into 128i8 (and store)
8117
8118 {4, MVT::v2i16, 2}, // interleave 4 x 2i16 into 8i16 (and store)
8119 {4, MVT::v4i16, 6}, // interleave 4 x 4i16 into 16i16 (and store)
8120 {4, MVT::v8i16, 10}, // interleave 4 x 8i16 into 32i16 (and store)
8121 {4, MVT::v16i16, 32}, // interleave 4 x 16i16 into 64i16 (and store)
8122 {4, MVT::v32i16, 64}, // interleave 4 x 32i16 into 128i16 (and store)
8123
8124 {4, MVT::v2i32, 5}, // interleave 4 x 2i32 into 8i32 (and store)
8125 {4, MVT::v4i32, 6}, // interleave 4 x 4i32 into 16i32 (and store)
8126 {4, MVT::v8i32, 16}, // interleave 4 x 8i32 into 32i32 (and store)
8127 {4, MVT::v16i32, 32}, // interleave 4 x 16i32 into 64i32 (and store)
8128 {4, MVT::v32i32, 64}, // interleave 4 x 32i32 into 128i32 (and store)
8129
8130 {4, MVT::v2i64, 6}, // interleave 4 x 2i64 into 8i64 (and store)
8131 {4, MVT::v4i64, 8}, // interleave 4 x 4i64 into 16i64 (and store)
8132 {4, MVT::v8i64, 20}, // interleave 4 x 8i64 into 32i64 (and store)
8133 {4, MVT::v16i64, 40}, // interleave 4 x 16i64 into 64i64 (and store)
8134
8135 {6, MVT::v2i8, 7}, // interleave 6 x 2i8 into 12i8 (and store)
8136 {6, MVT::v4i8, 9}, // interleave 6 x 4i8 into 24i8 (and store)
8137 {6, MVT::v8i8, 16}, // interleave 6 x 8i8 into 48i8 (and store)
8138 {6, MVT::v16i8, 27}, // interleave 6 x 16i8 into 96i8 (and store)
8139 {6, MVT::v32i8, 90}, // interleave 6 x 32i8 into 192i8 (and store)
8140
8141 {6, MVT::v2i16, 10}, // interleave 6 x 2i16 into 12i16 (and store)
8142 {6, MVT::v4i16, 15}, // interleave 6 x 4i16 into 24i16 (and store)
8143 {6, MVT::v8i16, 21}, // interleave 6 x 8i16 into 48i16 (and store)
8144 {6, MVT::v16i16, 58}, // interleave 6 x 16i16 into 96i16 (and store)
8145 {6, MVT::v32i16, 90}, // interleave 6 x 32i16 into 192i16 (and store)
8146
8147 {6, MVT::v2i32, 9}, // interleave 6 x 2i32 into 12i32 (and store)
8148 {6, MVT::v4i32, 12}, // interleave 6 x 4i32 into 24i32 (and store)
8149 {6, MVT::v8i32, 33}, // interleave 6 x 8i32 into 48i32 (and store)
8150 {6, MVT::v16i32, 66}, // interleave 6 x 16i32 into 96i32 (and store)
8151
8152 {6, MVT::v2i64, 8}, // interleave 6 x 2i64 into 12i64 (and store)
8153 {6, MVT::v4i64, 15}, // interleave 6 x 4i64 into 24i64 (and store)
8154 {6, MVT::v8i64, 30}, // interleave 6 x 8i64 into 48i64 (and store)
8155 };
8156
8157 static const CostTblEntry SSE2InterleavedStoreTbl[] = {
8158 {2, MVT::v2i8, 1}, // interleave 2 x 2i8 into 4i8 (and store)
8159 {2, MVT::v4i8, 1}, // interleave 2 x 4i8 into 8i8 (and store)
8160 {2, MVT::v8i8, 1}, // interleave 2 x 8i8 into 16i8 (and store)
8161
8162 {2, MVT::v2i16, 1}, // interleave 2 x 2i16 into 4i16 (and store)
8163 {2, MVT::v4i16, 1}, // interleave 2 x 4i16 into 8i16 (and store)
8164
8165 {2, MVT::v2i32, 1}, // interleave 2 x 2i32 into 4i32 (and store)
8166 };
8167
8168 if (Opcode == Instruction::Load) {
8169 auto GetDiscountedCost = [Factor, NumMembers = Indices.size(),
8170 MemOpCosts](const CostTblEntry *Entry) {
8171 // NOTE: this is just an approximation!
8172 // It can over/under -estimate the cost!
8173 return MemOpCosts + divideCeil(NumMembers * Entry->Cost, Factor);
8174 };
8175
8176 if (ST->hasAVX2())
8177 if (const auto *Entry = CostTableLookup(AVX2InterleavedLoadTbl, Factor,
8178 ETy.getSimpleVT()))
8179 return GetDiscountedCost(Entry);
8180
8181 if (ST->hasSSSE3())
8182 if (const auto *Entry = CostTableLookup(SSSE3InterleavedLoadTbl, Factor,
8183 ETy.getSimpleVT()))
8184 return GetDiscountedCost(Entry);
8185
8186 if (ST->hasSSE2())
8187 if (const auto *Entry = CostTableLookup(SSE2InterleavedLoadTbl, Factor,
8188 ETy.getSimpleVT()))
8189 return GetDiscountedCost(Entry);
8190 } else {
8191 assert(Opcode == Instruction::Store &&
8192 "Expected Store Instruction at this point");
8193 assert((!Indices.size() || Indices.size() == Factor) &&
8194 "Interleaved store only supports fully-interleaved groups.");
8195 if (ST->hasAVX2())
8196 if (const auto *Entry = CostTableLookup(AVX2InterleavedStoreTbl, Factor,
8197 ETy.getSimpleVT()))
8198 return MemOpCosts + Entry->Cost;
8199
8200 if (ST->hasSSE2())
8201 if (const auto *Entry = CostTableLookup(SSE2InterleavedStoreTbl, Factor,
8202 ETy.getSimpleVT()))
8203 return MemOpCosts + Entry->Cost;
8204 }
8205
8206 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
8207 Alignment, AddressSpace, CostKind,
8208 UseMaskForCond, UseMaskForGaps);
8209}
8210
8212 StackOffset BaseOffset,
8213 bool HasBaseReg, int64_t Scale,
8214 unsigned AddrSpace) const {
8215 // Scaling factors are not free at all.
8216 // An indexed folded instruction, i.e., inst (reg1, reg2, scale),
8217 // will take 2 allocations in the out of order engine instead of 1
8218 // for plain addressing mode, i.e. inst (reg1).
8219 // E.g.,
8220 // vaddps (%rsi,%rdx), %ymm0, %ymm1
8221 // Requires two allocations (one for the load, one for the computation)
8222 // whereas:
8223 // vaddps (%rsi), %ymm0, %ymm1
8224 // Requires just 1 allocation, i.e., freeing allocations for other operations
8225 // and having less micro operations to execute.
8226 //
8227 // For some X86 architectures, this is even worse because for instance for
8228 // stores, the complex addressing mode forces the instruction to use the
8229 // "load" ports instead of the dedicated "store" port.
8230 // E.g., on Haswell:
8231 // vmovaps %ymm1, (%r8, %rdi) can use port 2 or 3.
8232 // vmovaps %ymm1, (%r8) can use port 2, 3, or 7.
8234 AM.BaseGV = BaseGV;
8235 AM.BaseOffs = BaseOffset.getFixed();
8236 AM.HasBaseReg = HasBaseReg;
8237 AM.Scale = Scale;
8238 AM.ScalableOffset = BaseOffset.getScalable();
8239 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AddrSpace))
8240 // Scale represents reg2 * scale, thus account for 1
8241 // as soon as we use a second register.
8242 return AM.Scale != 0;
8244}
8245
8247 // TODO: Hook MispredictPenalty of SchedMachineModel into this.
8248 return 14;
8249}
8250
8252 unsigned Bits = Ty->getScalarSizeInBits();
8253
8254 // XOP has v16i8/v8i16/v4i32/v2i64 variable vector shifts.
8255 // Splitting for v32i8/v16i16 on XOP+AVX2 targets is still preferred.
8256 if (ST->hasXOP() && (Bits == 8 || Bits == 16 || Bits == 32 || Bits == 64))
8257 return false;
8258
8259 // AVX2 has vpsllv[dq] instructions (and other shifts) that make variable
8260 // shifts just as cheap as scalar ones.
8261 if (ST->hasAVX2() && (Bits == 32 || Bits == 64))
8262 return false;
8263
8264 // AVX512BW has shifts such as vpsllvw.
8265 if (ST->hasBWI() && Bits == 16)
8266 return false;
8267
8268 // Otherwise, it's significantly cheaper to shift by a scalar amount than by a
8269 // fully general vector.
8270 return true;
8271}
8272
8273unsigned X86TTIImpl::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
8274 Type *ScalarValTy, Align Alignment,
8275 unsigned AddrSpace) const {
8276 if (ST->hasF16C() && ScalarMemTy->isHalfTy()) {
8277 return 4;
8278 }
8279 return BaseT::getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy, Alignment,
8280 AddrSpace);
8281}
8282
8284 SmallVectorImpl<Use *> &Ops) const {
8285 using namespace llvm::PatternMatch;
8286
8287 if (I->getOpcode() == Instruction::And &&
8288 (ST->hasBMI() || (I->getType()->isVectorTy() && ST->hasSSE2()))) {
8289 for (auto &Op : I->operands()) {
8290 // (and X, (not Y)) -> (andn X, Y)
8291 if (match(Op.get(), m_Not(m_Value())) && !I->getType()->isIntegerTy(8)) {
8292 Ops.push_back(&Op);
8293 return true;
8294 }
8295 // (and X, (splat (not Y))) -> (andn X, (splat Y))
8296 if (match(Op.get(),
8298 m_Value(), m_ZeroMask()))) {
8299 Use &InsertElt = cast<Instruction>(Op)->getOperandUse(0);
8300 Use &Not = cast<Instruction>(InsertElt)->getOperandUse(1);
8301 Ops.push_back(&Not);
8302 Ops.push_back(&InsertElt);
8303 Ops.push_back(&Op);
8304 return true;
8305 }
8306 }
8307 }
8308
8309 FixedVectorType *VTy = dyn_cast<FixedVectorType>(I->getType());
8310 if (!VTy)
8311 return false;
8312
8313 if (I->getOpcode() == Instruction::Mul &&
8314 VTy->getElementType()->isIntegerTy(64)) {
8315 for (auto &Op : I->operands()) {
8316 // Make sure we are not already sinking this operand
8317 if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
8318 continue;
8319
8320 // Look for PMULDQ pattern where the input is a sext_inreg from vXi32 or
8321 // the PMULUDQ pattern where the input is a zext_inreg from vXi32.
8322 if (ST->hasSSE41() &&
8323 match(Op.get(), m_AShr(m_Shl(m_Value(), m_SpecificInt(32)),
8324 m_SpecificInt(32)))) {
8325 Ops.push_back(&cast<Instruction>(Op)->getOperandUse(0));
8326 Ops.push_back(&Op);
8327 } else if (ST->hasSSE2() &&
8328 match(Op.get(),
8329 m_And(m_Value(), m_SpecificInt(UINT64_C(0xffffffff))))) {
8330 Ops.push_back(&Op);
8331 }
8332 }
8333
8334 return !Ops.empty();
8335 }
8336
8337 // A uniform shift amount in a vector shift or funnel shift may be much
8338 // cheaper than a generic variable vector shift, so make that pattern visible
8339 // to SDAG by sinking the shuffle instruction next to the shift.
8340 int ShiftAmountOpNum = -1;
8341 if (I->isShift())
8342 ShiftAmountOpNum = 1;
8343 else if (auto *II = dyn_cast<IntrinsicInst>(I)) {
8344 if (II->getIntrinsicID() == Intrinsic::fshl ||
8345 II->getIntrinsicID() == Intrinsic::fshr)
8346 ShiftAmountOpNum = 2;
8347 }
8348
8349 if (ShiftAmountOpNum == -1)
8350 return false;
8351
8352 auto *Shuf = dyn_cast<ShuffleVectorInst>(I->getOperand(ShiftAmountOpNum));
8353 if (Shuf && getSplatIndex(Shuf->getShuffleMask()) >= 0 &&
8354 isVectorShiftByScalarCheap(I->getType())) {
8355 Ops.push_back(&I->getOperandUse(ShiftAmountOpNum));
8356 return true;
8357 }
8358
8359 return false;
8360}
8361
8363 bool HasEGPR = ST->hasEGPR();
8364 bool UseAPXFastCC = ST->useAPXFastCC();
8365 const TargetMachine &TM = getTLI()->getTargetMachine();
8366
8367 for (User *U : F.users()) {
8369 if (!CB || CB->getCalledOperand() != &F)
8370 continue;
8371 Function *CallerFunc = CB->getFunction();
8372 const X86Subtarget &CallerST = TM.getSubtarget<X86Subtarget>(*CallerFunc);
8373 if (CallerST.hasEGPR() != HasEGPR ||
8374 CallerST.useAPXFastCC() != UseAPXFastCC)
8375 return false;
8376 }
8377
8378 return true;
8379}
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t Imm
Expand Atomic instructions
This file provides a helper that implements much of the TTI interface in terms of the target-independ...
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Cost tables and simple lookup functions.
Hexagon Common GEP
iv users
Definition IVUsers.cpp:48
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static LVOptions Options
Definition LVOptions.cpp:25
CostTblEntryT< CostKindCosts > CostKindTblEntry
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
uint64_t IntrinsicInst * II
#define P(N)
int64_t C1
RISCVCC::CondCode CC
This file implements the SmallBitVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This file describes how to lower LLVM code to machine code.
This pass exposes codegen information to IR-level passes.
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
static InstructionCost getPredicateFanoutCost(const X86TTIImpl &TTI, Type *DataVTy, Type *MaskVTy)
static bool isFoldedIntoRsqrtEstimate(const Value *V, const X86TargetLowering &TLI, MVT VT, TTI::TargetCostKind CostKind)
Returns true if V is the square root, divided into with the reciprocal allowed, and the target replac...
static bool isLegalMaskedLoadStore(Type *ScalarTy, const X86Subtarget *ST)
TypeConversionCostTblEntryT< CostKindCosts > TypeConversionCostKindTblEntry
This file a TargetTransformInfoImplBase conforming object specific to the X86 target machine.
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static const fltSemantics & IEEEdouble()
Definition APFloat.h:305
static LLVM_ABI unsigned int semanticsPrecision(const fltSemantics &)
Definition APFloat.cpp:335
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
unsigned popcount() const
Count the number of bits set.
Definition APInt.h:1690
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition APInt.h:1350
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition APInt.h:254
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
LLVM_ABI APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition APInt.cpp:1086
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition APInt.h:829
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:196
LLVM_ABI APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition APInt.cpp:478
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond=false, bool UseMaskForGaps=false) const override
InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind) const override
TTI::ShuffleKind improveShuffleKindFromMask(TTI::ShuffleKind Kind, ArrayRef< int > Mask, VectorType *SrcTy, int &Index, VectorType *&SubTy) const
bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace, Instruction *I=nullptr, int64_t ScalableOffset=0) const override
unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, Type *ScalarValTy, Align Alignment, unsigned AddrSpace) const override
InstructionCost getScalarizationOverhead(VectorType *InTy, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={}, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind) const override
InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Op2Info={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Opd1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Opd2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr) const override
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef< int > Mask, int Index, VectorType *SubTp, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const override
std::pair< InstructionCost, MVT > getTypeLegalizationCost(Type *Ty) const
InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, TTI::PartialReductionExtendKind OpAExtend, TTI::PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp, TTI::TargetCostKind CostKind, std::optional< FastMathFlags > FMF) const override
InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, TTI::TargetCostKind CostKind) const override
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const override
InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *, const SCEV *, TTI::TargetCostKind) const override
InstructionCost getGEPCost(Type *PointeeType, const Value *Ptr, ArrayRef< const Value * > Operands, TTI::TargetCostKind CostKind, Type *AccessType) const override
InstructionCost getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Value * getCalledOperand() const
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
Container class for subtarget features.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
static unsigned getPointerOperandIndex()
static InstructionCost getInvalid(CostType Val=0)
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
const SmallVectorImpl< Type * > & getArgTypes() const
const SmallVectorImpl< const Value * > & getArgs() const
const IntrinsicInst * getInst() const
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
Machine Value Type.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
Information for memory intrinsic cost model.
This class represents an analyzed expression in the program.
The main scalar evolution driver.
static LLVM_ABI bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool test(unsigned Idx) const
Returns true if bit Idx is set.
size_type size() const
Returns the number of bits in this bitvector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
Definition TypeSize.h:31
static StackOffset getScalable(int64_t Scalable)
Definition TypeSize.h:41
static StackOffset getFixed(int64_t Fixed)
Definition TypeSize.h:40
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
const STC & getSubtarget(const Function &F) const
This method returns a pointer to the specified type of TargetSubtargetInfo.
virtual const TargetSubtargetInfo * getSubtargetImpl(const Function &) const
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
virtual const TargetLowering * getTargetLowering() const
bool isStridedAccess(const SCEV *Ptr) const
unsigned minRequiredElementSize(const Value *Val, bool &isSigned) const
const SCEVConstant * getConstantStrideStep(ScalarEvolution *SE, const SCEV *Ptr) const
virtual bool isExpensiveToSpeculativelyExecute(const Instruction *I) const
virtual InstructionCost getPointersChainCost(ArrayRef< const Value * > Ptrs, const Value *Base, const TTI::PointersChainInfo &Info, Type *AccessTy, const TTI::TargetCostKind CostKind) const
MaskKind
Some targets only support masked load/store with a constant mask.
TargetCostKind
The kind of cost model.
@ TCK_RecipThroughput
Reciprocal throughput.
@ TCK_CodeSize
Instruction code size.
@ TCK_SizeAndLatency
The weighted sum of size and latency.
@ TCK_Latency
The latency of instruction.
static bool requiresOrderedReduction(std::optional< FastMathFlags > FMF)
A helper function to determine the type of reduction algorithm used for a given Opcode and set of Fas...
PopcntSupportKind
Flags indicating the kind of support for population count.
llvm::VectorInstrContext VectorInstrContext
@ TCC_Free
Expected to fold away in lowering.
@ TCC_Basic
The cost of a typical 'add' instruction.
ShuffleKind
The various kinds of shuffle patterns for vector queries.
@ SK_InsertSubvector
InsertSubvector. Index indicates start offset.
@ SK_Select
Selects elements from the corresponding lane of either source operand.
@ SK_PermuteSingleSrc
Shuffle elements of single source vector with any shuffle mask.
@ SK_Transpose
Transpose two vectors.
@ SK_Splice
Concatenates elements from the first input vector with elements of the second input vector.
@ SK_Broadcast
Broadcast element 0 to all other elements.
@ SK_PermuteTwoSrc
Merge elements from two source vectors into one with any shuffle mask.
@ SK_Reverse
Reverse the order of the vector.
@ SK_ExtractSubvector
ExtractSubvector Index indicates start offset.
CastContextHint
Represents a hint about the context in which a cast is used.
@ None
The cast is not used with a load/store of any kind.
CacheLevel
The possible cache levels.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Definition TypeSize.h:355
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
Definition Type.cpp:295
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition Type.h:258
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:155
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
Definition Type.h:147
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:292
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition Type.h:144
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition Type.h:158
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:291
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
LLVM_ABI bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
Definition Type.cpp:61
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
Definition Type.cpp:272
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition Type.h:222
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Definition Type.h:399
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:271
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
Base class of all SIMD vector types.
static VectorType * getExtendedElementVectorType(VectorType *VTy)
This static method is like getInteger except that the element types are twice as wide as the elements...
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
static VectorType * getDoubleElementsVectorType(VectorType *VTy)
This static method returns a VectorType with twice as many elements as the input type and the same el...
Type * getElementType() const
bool useAVX512Regs() const
bool hasAVX512() const
bool useAPXFastCC() const
bool hasAVX2() const
bool useFastCCForInternalCall(Function &F) const override
bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, const SmallBitVector &OpcodeMask, ArrayRef< const Value * > Scalars) const override
InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, TTI::TargetCostKind CostKind) const override
bool isLegalNTLoad(Type *DataType, Align Alignment) const override
std::optional< unsigned > getCacheAssociativity(TargetTransformInfo::CacheLevel Level) const override
InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, TTI::TargetCostKind CostKind) const override
Try to calculate op costs for min/max reduction operations.
bool isLegalBroadcastLoad(Type *ElementTy, ElementCount NumElements) const override
unsigned getRegisterClassForType(bool Vector, Type *Ty) const override
InstructionCost getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override
Get memory intrinsic cost based on arguments.
unsigned getMaxInterleaveFactor(ElementCount VF, bool HasUnorderedReductions) const override
InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, TTI::PartialReductionExtendKind OpAExtend, TTI::PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp, TTI::TargetCostKind CostKind, std::optional< FastMathFlags > FMF) const override
bool isLegalNTStore(Type *DataType, Align Alignment) const override
InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const override
InstructionCost getInterleavedMemoryOpCostAVX512(unsigned Opcode, FixedVectorType *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond=false, bool UseMaskForGaps=false) const
TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override
bool isVectorShiftByScalarCheap(Type *Ty) const override
bool isLegalMaskedGather(Type *DataType, Align Alignment) const override
bool shouldExpandReduction(const IntrinsicInst *II) const override
InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const override
Return the cost of the scaling factor used in the addressing mode represented by AM for this target,...
unsigned getAtomicMemIntrinsicMaxElementSize() const override
InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Op2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr) const override
bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) const override
InstructionCost getBranchMispredictPenalty() const override
bool isExpensiveToSpeculativelyExecute(const Instruction *I) const override
bool hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const override
InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind, ArrayRef< const Value * > Scalars) const override
bool isLegalMaskedStore(Type *DataType, Align Alignment, unsigned AddressSpace, TTI::MaskKind MaskKind=TTI::MaskKind::VariableOrConstantMask) const override
std::optional< unsigned > getCacheSize(TargetTransformInfo::CacheLevel Level) const override
bool isLegalMaskedGatherScatter(Type *DataType, Align Alignment) const
bool isLegalMaskedLoad(Type *DataType, Align Alignment, unsigned AddressSpace, TTI::MaskKind MaskKind=TTI::MaskKind::VariableOrConstantMask) const override
bool enableInterleavedAccessVectorization() const override
unsigned getLoadStoreVecRegBitWidth(unsigned AS) const override
unsigned getNumberOfRegisters(unsigned ClassID) const override
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
bool isLegalMaskedScatter(Type *DataType, Align Alignment) const override
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, TTI::TargetCostKind CostKind, ArrayRef< int > Mask, int Index, VectorType *SubTp, ArrayRef< const Value * > Args={}, const Instruction *CtxI=nullptr, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
unsigned getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, Type *ScalarValTy, Align Alignment, unsigned AddrSpace) const override
bool hasDivRemOp(Type *DataType, bool IsSigned) const override
bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) const override
InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind) const override
bool supportsEfficientVectorElementLoadStore() const override
InstructionCost getIntImmCostInst(unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind, Instruction *Inst=nullptr) const override
bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) const override
TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override
bool isFCmpOrdCheaperThanFCmpZero(Type *Ty) const override
TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const override
InstructionCost getPointersChainCost(ArrayRef< const Value * > Ptrs, const Value *Base, const TTI::PointersChainInfo &Info, Type *AccessTy, const TTI::TargetCostKind CostKind) const override
InstructionCost getIntImmCost(int64_t) const
Calculate the cost of materializing a 64-bit value.
InstructionCost getMinMaxCost(Intrinsic::ID IID, Type *Ty, TTI::TargetCostKind CostKind, FastMathFlags FMF) const
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I=nullptr) const override
InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond=false, bool UseMaskForGaps=false) const override
bool canMacroFuseCmp() const override
bool areInlineCompatible(const Function *Caller, const Function *Callee) const override
InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const
bool prefersVectorizedAddressing() const override
bool areTypesABICompatible(const Function *Caller, const Function *Callee, ArrayRef< Type * > Type) const override
bool forceScalarizeMaskedScatter(VectorType *VTy, Align Alignment) const override
InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const override
Get intrinsic cost based on arguments.
bool isProfitableToSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
InstructionCost getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={}, TTI::VectorInstrContext VIC=TTI::VectorInstrContext::None) const override
Estimate the overhead of scalarizing an instruction.
InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind) const override
InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const
Calculate the cost of Gather / Scatter operation.
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr, TTI::TargetCostKind CostKind) const override
InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Op2Info={TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I=nullptr) const override
bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, const TargetTransformInfo::LSRCost &C2) const override
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
Definition TypeSize.h:181
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:253
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
Definition APInt.cpp:3052
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
ISD namespace - This namespace contains an enum which represents all of the SelectionDAG node types a...
Definition ISDOpcodes.h:26
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition ISDOpcodes.h:47
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition ISDOpcodes.h:282
@ CLMUL
Carry-less multiplication operations.
Definition ISDOpcodes.h:802
@ CTLZ_ZERO_POISON
Definition ISDOpcodes.h:820
@ PARTIAL_REDUCE_UMLA
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ PARTIAL_REDUCE_FMLA
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition ISDOpcodes.h:351
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SMULO
Same for multiplication.
Definition ISDOpcodes.h:359
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
Definition ISDOpcodes.h:819
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ PARTIAL_REDUCE_SUMLA
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
ap_match< APInt > m_APIntAllowPoison(const APInt *&Res)
Match APInt while allowing poison in splat vector constants.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
BinaryOp_match< LHS, RHS, Instruction::FMul > m_FMul(const LHS &L, const RHS &R)
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
auto m_Value()
Match an arbitrary value and ignore it.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)
Matches LoadInst.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::FDiv > m_FDiv(const LHS &L, const RHS &R)
AllowFmf_match< T, FastMathFlags::AllowContract > m_AllowContract(const T &SubPattern)
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
This is an optimization pass for GlobalISel generic memory operations.
constexpr auto not_equal_to(T &&Arg)
Functor variant of std::not_equal_to that can be used as a UnaryPredicate in functional algorithms li...
Definition STLExtras.h:2196
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
const CostTblEntryT< CostType > * CostTableLookup(ArrayRef< CostTblEntryT< CostType > > Tbl, int ISD, MVT Ty)
Find in cost table.
Definition CostTable.h:36
InstructionCost Cost
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
@ Known
Known to have no common set bits.
LLVM_ABI void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs=nullptr, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition Analysis.cpp:121
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:676
LLVM_ABI unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Return the number of times the sign bit of the register is replicated into the other bits.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Definition MathExtras.h:380
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
constexpr int PoisonMaskElem
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
TargetTransformInfo TTI
DWARFExpression::Operation Op
OutputIt copy(R &&Range, OutputIt Out)
Definition STLExtras.h:1901
CostTblEntryT< uint16_t > CostTblEntry
Definition CostTable.h:31
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:2035
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
LLVM_ABI void processShuffleMasks(ArrayRef< int > Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, unsigned NumOfUsedRegs, function_ref< void()> NoInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned)> SingleInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned, bool)> ManyInputsAction)
Splits and processes shuffle mask depending on the number of input and output registers.
const TypeConversionCostTblEntryT< CostType > * ConvertCostTableLookup(ArrayRef< TypeConversionCostTblEntryT< CostType > > Tbl, int ISD, MVT Dst, MVT Src)
Find in type conversion cost table.
Definition CostTable.h:67
LLVM_ABI int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Cost Table Entry.
Definition CostTable.h:26
Extended Value Type.
Definition ValueTypes.h:35
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
unsigned Insns
TODO: Some of these could be merged.
Returns options for expansion of memcmp. IsZeroCmp is.
Describe known properties for a set of pointers.
Type Conversion Cost Table.
Definition CostTable.h:56