LLVM 20.0.0git
X86ISelLowering.cpp
Go to the documentation of this file.
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that X86 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "X86ISelLowering.h"
15#include "MCTargetDesc/X86ShuffleDecode.h"
16#include "X86.h"
17#include "X86CallingConv.h"
18#include "X86FrameLowering.h"
19#include "X86InstrBuilder.h"
20#include "X86IntrinsicsInfo.h"
21#include "X86MachineFunctionInfo.h"
22#include "X86TargetMachine.h"
23#include "X86TargetObjectFile.h"
24#include "llvm/ADT/SmallBitVector.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/ADT/StringSwitch.h"
29#include "llvm/Analysis/BlockFrequencyInfo.h"
30#include "llvm/Analysis/ObjCARCUtil.h"
31#include "llvm/Analysis/ProfileSummaryInfo.h"
32#include "llvm/Analysis/VectorUtils.h"
33#include "llvm/CodeGen/IntrinsicLowering.h"
34#include "llvm/CodeGen/MachineFrameInfo.h"
35#include "llvm/CodeGen/MachineFunction.h"
36#include "llvm/CodeGen/MachineInstrBuilder.h"
37#include "llvm/CodeGen/MachineJumpTableInfo.h"
38#include "llvm/CodeGen/MachineLoopInfo.h"
39#include "llvm/CodeGen/MachineModuleInfo.h"
40#include "llvm/CodeGen/MachineRegisterInfo.h"
41#include "llvm/CodeGen/SDPatternMatch.h"
42#include "llvm/CodeGen/TargetLowering.h"
43#include "llvm/CodeGen/WinEHFuncInfo.h"
44#include "llvm/IR/CallingConv.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/EHPersonalities.h"
48#include "llvm/IR/Function.h"
49#include "llvm/IR/GlobalAlias.h"
50#include "llvm/IR/GlobalVariable.h"
51#include "llvm/IR/IRBuilder.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/Intrinsics.h"
54#include "llvm/IR/PatternMatch.h"
55#include "llvm/MC/MCAsmInfo.h"
56#include "llvm/MC/MCContext.h"
57#include "llvm/MC/MCExpr.h"
58#include "llvm/MC/MCSymbol.h"
59#include "llvm/Support/CommandLine.h"
60#include "llvm/Support/Debug.h"
61#include "llvm/Support/ErrorHandling.h"
62#include "llvm/Support/KnownBits.h"
63#include "llvm/Support/MathExtras.h"
64#include "llvm/Target/TargetOptions.h"
65#include <algorithm>
66#include <bitset>
67#include <cctype>
68#include <numeric>
69using namespace llvm;
70
71#define DEBUG_TYPE "x86-isel"
72
73static cl::opt<int> ExperimentalPrefInnermostLoopAlignment(
74 "x86-experimental-pref-innermost-loop-alignment", cl::init(4),
75 cl::desc(
76 "Sets the preferable loop alignment for experiments (as log2 bytes) "
77 "for innermost loops only. If specified, this option overrides "
78 "alignment set by x86-experimental-pref-loop-alignment."),
79 cl::Hidden);
80
81static cl::opt<int> BrMergingBaseCostThresh(
82 "x86-br-merging-base-cost", cl::init(2),
83 cl::desc(
84 "Sets the cost threshold for when multiple conditionals will be merged "
85 "into one branch versus be split in multiple branches. Merging "
86 "conditionals saves branches at the cost of additional instructions. "
87 "This value sets the instruction cost limit, below which conditionals "
88 "will be merged, and above which conditionals will be split. Set to -1 "
89 "to never merge branches."),
90 cl::Hidden);
91
92static cl::opt<int> BrMergingCcmpBias(
93 "x86-br-merging-ccmp-bias", cl::init(6),
94 cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target "
95 "supports conditional compare instructions."),
96 cl::Hidden);
97
98static cl::opt<int> BrMergingLikelyBias(
99 "x86-br-merging-likely-bias", cl::init(0),
100 cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely "
101 "that all conditionals will be executed. For example for merging "
102 "the conditionals (a == b && c > d), if its known that a == b is "
103 "likely, then it is likely that if the conditionals are split "
104 "both sides will be executed, so it may be desirable to increase "
105 "the instruction cost threshold. Set to -1 to never merge likely "
106 "branches."),
107 cl::Hidden);
108
109static cl::opt<int> BrMergingUnlikelyBias(
110 "x86-br-merging-unlikely-bias", cl::init(-1),
111 cl::desc(
112 "Decreases 'x86-br-merging-base-cost' in cases that it is unlikely "
113 "that all conditionals will be executed. For example for merging "
114 "the conditionals (a == b && c > d), if its known that a == b is "
115 "unlikely, then it is unlikely that if the conditionals are split "
116 "both sides will be executed, so it may be desirable to decrease "
117 "the instruction cost threshold. Set to -1 to never merge unlikely "
118 "branches."),
119 cl::Hidden);
120
121static cl::opt<bool> MulConstantOptimization(
122 "mul-constant-optimization", cl::init(true),
123 cl::desc("Replace 'mul x, Const' with more effective instructions like "
124 "SHIFT, LEA, etc."),
125 cl::Hidden);
126
127X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
128 const X86Subtarget &STI)
129 : TargetLowering(TM), Subtarget(STI) {
130 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
131 MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
132
133 // Set up the TargetLowering object.
134
135 // X86 is weird. It always uses i8 for shift amounts and setcc results.
136 setBooleanContents(ZeroOrOneBooleanContent);
137 // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
138 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
139
140 // X86 instruction cache is coherent with its data cache so we can use the
141 // default expansion to a no-op.
142 setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Expand);
143
144 // For 64-bit, since we have so many registers, use the ILP scheduler.
145 // For 32-bit, use the register pressure specific scheduling.
146 // For Atom, always use ILP scheduling.
147 if (Subtarget.isAtom())
148 setSchedulingPreference(Sched::ILP);
149 else if (Subtarget.is64Bit())
150 setSchedulingPreference(Sched::ILP);
151 else
152 setSchedulingPreference(Sched::RegPressure);
153 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
154 setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
155
156 // Bypass expensive divides and use cheaper ones.
157 if (TM.getOptLevel() >= CodeGenOptLevel::Default) {
158 if (Subtarget.hasSlowDivide32())
159 addBypassSlowDiv(32, 8);
160 if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit())
161 addBypassSlowDiv(64, 32);
162 }
163
164 // Setup Windows compiler runtime calls.
165 if (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()) {
166 static const struct {
167 const RTLIB::Libcall Op;
168 const char * const Name;
169 const CallingConv::ID CC;
170 } LibraryCalls[] = {
171 { RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall },
172 { RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall },
173 { RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall },
174 { RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall },
175 { RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall },
176 };
177
178 for (const auto &LC : LibraryCalls) {
179 setLibcallName(LC.Op, LC.Name);
180 setLibcallCallingConv(LC.Op, LC.CC);
181 }
182 }
183
184 if (Subtarget.canUseCMPXCHG16B())
185 setMaxAtomicSizeInBitsSupported(128);
186 else if (Subtarget.canUseCMPXCHG8B())
187 setMaxAtomicSizeInBitsSupported(64);
188 else
189 setMaxAtomicSizeInBitsSupported(32);
190
191 setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
192
193 setMaxLargeFPConvertBitWidthSupported(128);
194
195 // Set up the register classes.
196 addRegisterClass(MVT::i8, &X86::GR8RegClass);
197 addRegisterClass(MVT::i16, &X86::GR16RegClass);
198 addRegisterClass(MVT::i32, &X86::GR32RegClass);
199 if (Subtarget.is64Bit())
200 addRegisterClass(MVT::i64, &X86::GR64RegClass);
201
202 for (MVT VT : MVT::integer_valuetypes())
203 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
204
205 // We don't accept any truncstore of integer registers.
206 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
207 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
208 setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
209 setTruncStoreAction(MVT::i32, MVT::i16, Expand);
210 setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
211 setTruncStoreAction(MVT::i16, MVT::i8, Expand);
212
213 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
214
215 // SETOEQ and SETUNE require checking two conditions.
216 for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) {
217 setCondCodeAction(ISD::SETOEQ, VT, Expand);
218 setCondCodeAction(ISD::SETUNE, VT, Expand);
219 }
220
221 // Integer absolute.
222 if (Subtarget.canUseCMOV()) {
223 setOperationAction(ISD::ABS , MVT::i16 , Custom);
224 setOperationAction(ISD::ABS , MVT::i32 , Custom);
225 if (Subtarget.is64Bit())
226 setOperationAction(ISD::ABS , MVT::i64 , Custom);
227 }
228
229 // Absolute difference.
230 for (auto Op : {ISD::ABDS, ISD::ABDU}) {
231 setOperationAction(Op , MVT::i8 , Custom);
232 setOperationAction(Op , MVT::i16 , Custom);
233 setOperationAction(Op , MVT::i32 , Custom);
234 if (Subtarget.is64Bit())
235 setOperationAction(Op , MVT::i64 , Custom);
236 }
237
238 // Signed saturation subtraction.
239 setOperationAction(ISD::SSUBSAT , MVT::i8 , Custom);
240 setOperationAction(ISD::SSUBSAT , MVT::i16 , Custom);
241 setOperationAction(ISD::SSUBSAT , MVT::i32 , Custom);
242 if (Subtarget.is64Bit())
243 setOperationAction(ISD::SSUBSAT , MVT::i64 , Custom);
244
245 // Funnel shifts.
246 for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
247 // For slow shld targets we only lower for code size.
248 LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal;
249
250 setOperationAction(ShiftOp , MVT::i8 , Custom);
251 setOperationAction(ShiftOp , MVT::i16 , Custom);
252 setOperationAction(ShiftOp , MVT::i32 , ShiftDoubleAction);
253 if (Subtarget.is64Bit())
254 setOperationAction(ShiftOp , MVT::i64 , ShiftDoubleAction);
255 }
256
257 if (!Subtarget.useSoftFloat()) {
258 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
259 // operation.
260 setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
261 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);
262 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
263 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);
264 // We have an algorithm for SSE2, and we turn this into a 64-bit
265 // FILD or VCVTUSI2SS/SD for other targets.
266 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
267 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
268 // We have an algorithm for SSE2->double, and we turn this into a
269 // 64-bit FILD followed by conditional FADD for other targets.
270 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
271 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
272
273 // Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
274 // this operation.
275 setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
276 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);
277 // SSE has no i16 to fp conversion, only i32. We promote in the handler
278 // to allow f80 to use i16 and f64 to use i16 with sse1 only
279 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom);
280 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);
281 // f32 and f64 cases are Legal with SSE1/SSE2, f80 case is not
282 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
283 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
284 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
285 // are Legal, f80 is custom lowered.
286 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
287 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
288
289 // Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
290 // this operation.
291 setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
292 // FIXME: This doesn't generate invalid exception when it should. PR44019.
293 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8, Promote);
294 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom);
295 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);
296 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
297 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
298 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
299 // are Legal, f80 is custom lowered.
300 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
301 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
302
303 // Handle FP_TO_UINT by promoting the destination to a larger signed
304 // conversion.
305 setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
306 // FIXME: This doesn't generate invalid exception when it should. PR44019.
307 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8, Promote);
308 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
309 // FIXME: This doesn't generate invalid exception when it should. PR44019.
310 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);
311 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
312 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
313 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
314 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
315
316 setOperationAction(ISD::LRINT, MVT::f32, Custom);
317 setOperationAction(ISD::LRINT, MVT::f64, Custom);
318 setOperationAction(ISD::LLRINT, MVT::f32, Custom);
319 setOperationAction(ISD::LLRINT, MVT::f64, Custom);
320
321 if (!Subtarget.is64Bit()) {
322 setOperationAction(ISD::LRINT, MVT::i64, Custom);
323 setOperationAction(ISD::LLRINT, MVT::i64, Custom);
324 }
325 }
326
327 if (Subtarget.hasSSE2()) {
328 // Custom lowering for saturating float to int conversions.
329 // We handle promotion to larger result types manually.
330 for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) {
331 setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
332 setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
333 }
334 if (Subtarget.is64Bit()) {
335 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
336 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
337 }
338 }
339
340 // Handle address space casts between mixed sized pointers.
341 setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
342 setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
343
344 // TODO: when we have SSE, these could be more efficient, by using movd/movq.
345 if (!Subtarget.hasSSE2()) {
346 setOperationAction(ISD::BITCAST , MVT::f32 , Expand);
347 setOperationAction(ISD::BITCAST , MVT::i32 , Expand);
348 if (Subtarget.is64Bit()) {
349 setOperationAction(ISD::BITCAST , MVT::f64 , Expand);
350 // Without SSE, i64->f64 goes through memory.
351 setOperationAction(ISD::BITCAST , MVT::i64 , Expand);
352 }
353 } else if (!Subtarget.is64Bit())
354 setOperationAction(ISD::BITCAST , MVT::i64 , Custom);
355
356 // Scalar integer divide and remainder are lowered to use operations that
357 // produce two results, to match the available instructions. This exposes
358 // the two-result form to trivial CSE, which is able to combine x/y and x%y
359 // into a single instruction.
360 //
361 // Scalar integer multiply-high is also lowered to use two-result
362 // operations, to match the available instructions. However, plain multiply
363 // (low) operations are left as Legal, as there are single-result
364 // instructions for this in x86. Using the two-result multiply instructions
365 // when both high and low results are needed must be arranged by dagcombine.
366 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
367 setOperationAction(ISD::MULHS, VT, Expand);
368 setOperationAction(ISD::MULHU, VT, Expand);
369 setOperationAction(ISD::SDIV, VT, Expand);
370 setOperationAction(ISD::UDIV, VT, Expand);
371 setOperationAction(ISD::SREM, VT, Expand);
372 setOperationAction(ISD::UREM, VT, Expand);
373 }
374
375 setOperationAction(ISD::BR_JT , MVT::Other, Expand);
376 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
377 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
378 MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
379 setOperationAction(ISD::BR_CC, VT, Expand);
380 setOperationAction(ISD::SELECT_CC, VT, Expand);
381 }
382 if (Subtarget.is64Bit())
383 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
384 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
385 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
386 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
387
388 setOperationAction(ISD::FREM , MVT::f32 , Expand);
389 setOperationAction(ISD::FREM , MVT::f64 , Expand);
390 setOperationAction(ISD::FREM , MVT::f80 , Expand);
391 setOperationAction(ISD::FREM , MVT::f128 , Expand);
392
393 if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) {
394 setOperationAction(ISD::GET_ROUNDING , MVT::i32 , Custom);
395 setOperationAction(ISD::SET_ROUNDING , MVT::Other, Custom);
396 setOperationAction(ISD::GET_FPENV_MEM , MVT::Other, Custom);
397 setOperationAction(ISD::SET_FPENV_MEM , MVT::Other, Custom);
398 setOperationAction(ISD::RESET_FPENV , MVT::Other, Custom);
399 }
400
401 // Promote the i8 variants and force them on up to i32 which has a shorter
402 // encoding.
403 setOperationPromotedToType(ISD::CTTZ , MVT::i8 , MVT::i32);
404 setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
405 // Promoted i16. tzcntw has a false dependency on Intel CPUs. For BSF, we emit
406 // a REP prefix to encode it as TZCNT for modern CPUs so it makes sense to
407 // promote that too.
408 setOperationPromotedToType(ISD::CTTZ , MVT::i16 , MVT::i32);
409 setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , MVT::i32);
410
411 if (!Subtarget.hasBMI()) {
412 setOperationAction(ISD::CTTZ , MVT::i32 , Custom);
413 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Legal);
414 if (Subtarget.is64Bit()) {
415 setOperationPromotedToType(ISD::CTTZ , MVT::i32, MVT::i64);
416 setOperationAction(ISD::CTTZ , MVT::i64 , Custom);
417 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal);
418 }
419 }
420
421 if (Subtarget.hasLZCNT()) {
422 // When promoting the i8 variants, force them to i32 for a shorter
423 // encoding.
424 setOperationPromotedToType(ISD::CTLZ , MVT::i8 , MVT::i32);
425 setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
426 } else {
427 for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
428 if (VT == MVT::i64 && !Subtarget.is64Bit())
429 continue;
430 setOperationAction(ISD::CTLZ , VT, Custom);
431 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
432 }
433 }
434
435 for (auto Op : {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP, ISD::FP_TO_FP16,
436 ISD::STRICT_FP_TO_FP16}) {
437 // Special handling for half-precision floating point conversions.
438 // If we don't have F16C support, then lower half float conversions
439 // into library calls.
440 setOperationAction(
441 Op, MVT::f32,
442 (!Subtarget.useSoftFloat() && Subtarget.hasF16C()) ? Custom : Expand);
443 // There's never any support for operations beyond MVT::f32.
444 setOperationAction(Op, MVT::f64, Expand);
445 setOperationAction(Op, MVT::f80, Expand);
446 setOperationAction(Op, MVT::f128, Expand);
447 }
448
449 for (auto VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
450 setOperationAction(ISD::STRICT_FP_TO_BF16, VT, Expand);
451 setOperationAction(ISD::STRICT_BF16_TO_FP, VT, Expand);
452 }
453
454 for (MVT VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
455 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
456 setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
457 setTruncStoreAction(VT, MVT::f16, Expand);
458 setTruncStoreAction(VT, MVT::bf16, Expand);
459
460 setOperationAction(ISD::BF16_TO_FP, VT, Expand);
461 setOperationAction(ISD::FP_TO_BF16, VT, Custom);
462 }
463
464 setOperationAction(ISD::PARITY, MVT::i8, Custom);
465 setOperationAction(ISD::PARITY, MVT::i16, Custom);
466 setOperationAction(ISD::PARITY, MVT::i32, Custom);
467 if (Subtarget.is64Bit())
468 setOperationAction(ISD::PARITY, MVT::i64, Custom);
469 if (Subtarget.hasPOPCNT()) {
470 setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32);
471 // popcntw is longer to encode than popcntl and also has a false dependency
472 // on the dest that popcntl hasn't had since Cannon Lake.
473 setOperationPromotedToType(ISD::CTPOP, MVT::i16, MVT::i32);
474 } else {
475 setOperationAction(ISD::CTPOP , MVT::i8 , Custom);
476 setOperationAction(ISD::CTPOP , MVT::i16 , Custom);
477 setOperationAction(ISD::CTPOP , MVT::i32 , Custom);
478 setOperationAction(ISD::CTPOP , MVT::i64 , Custom);
479 }
480
481 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
482
483 if (!Subtarget.hasMOVBE())
484 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
485
486 // X86 wants to expand cmov itself.
487 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
488 setOperationAction(ISD::SELECT, VT, Custom);
489 setOperationAction(ISD::SETCC, VT, Custom);
490 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
491 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
492 }
493 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
494 if (VT == MVT::i64 && !Subtarget.is64Bit())
495 continue;
496 setOperationAction(ISD::SELECT, VT, Custom);
497 setOperationAction(ISD::SETCC, VT, Custom);
498 }
499
500 // Custom action for SELECT MMX and expand action for SELECT_CC MMX
501 setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
502 setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
503
504 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
505 // NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
506 // LLVM/Clang supports zero-cost DWARF and SEH exception handling.
507 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
508 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
509 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
510 if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
511 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
512
513 // Darwin ABI issue.
514 for (auto VT : { MVT::i32, MVT::i64 }) {
515 if (VT == MVT::i64 && !Subtarget.is64Bit())
516 continue;
517 setOperationAction(ISD::ConstantPool , VT, Custom);
518 setOperationAction(ISD::JumpTable , VT, Custom);
519 setOperationAction(ISD::GlobalAddress , VT, Custom);
520 setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
521 setOperationAction(ISD::ExternalSymbol , VT, Custom);
522 setOperationAction(ISD::BlockAddress , VT, Custom);
523 }
524
525 // 64-bit shl, sra, srl (iff 32-bit x86)
526 for (auto VT : { MVT::i32, MVT::i64 }) {
527 if (VT == MVT::i64 && !Subtarget.is64Bit())
528 continue;
529 setOperationAction(ISD::SHL_PARTS, VT, Custom);
530 setOperationAction(ISD::SRA_PARTS, VT, Custom);
531 setOperationAction(ISD::SRL_PARTS, VT, Custom);
532 }
533
534 if (Subtarget.hasSSEPrefetch())
535 setOperationAction(ISD::PREFETCH , MVT::Other, Custom);
536
537 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
538
539 // Expand certain atomics
540 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
541 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
542 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
543 setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
544 setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom);
545 setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);
546 setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);
547 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
548 }
549
550 if (!Subtarget.is64Bit())
551 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
552
553 if (Subtarget.is64Bit() && Subtarget.hasAVX()) {
554 // All CPUs supporting AVX will atomically load/store aligned 128-bit
555 // values, so we can emit [V]MOVAPS/[V]MOVDQA.
556 setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
557 setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
558 }
559
560 if (Subtarget.canUseCMPXCHG16B())
561 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
562
563 // FIXME - use subtarget debug flags
564 if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() &&
565 !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() &&
566 TM.Options.ExceptionModel != ExceptionHandling::SjLj) {
567 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
568 }
569
570 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
571 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
572
573 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
574 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
575
576 setOperationAction(ISD::TRAP, MVT::Other, Legal);
577 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
578 if (Subtarget.isTargetPS())
579 setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);
580 else
581 setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
582
583 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
584 setOperationAction(ISD::VASTART , MVT::Other, Custom);
585 setOperationAction(ISD::VAEND , MVT::Other, Expand);
586 bool Is64Bit = Subtarget.is64Bit();
587 setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand);
588 setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
589
590 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
591 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
592
593 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
594
595 // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering.
596 setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);
597 setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);
598
599 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
600
601 auto setF16Action = [&] (MVT VT, LegalizeAction Action) {
602 setOperationAction(ISD::FABS, VT, Action);
603 setOperationAction(ISD::FNEG, VT, Action);
604 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
605 setOperationAction(ISD::FREM, VT, Action);
606 setOperationAction(ISD::FMA, VT, Action);
607 setOperationAction(ISD::FMINNUM, VT, Action);
608 setOperationAction(ISD::FMAXNUM, VT, Action);
609 setOperationAction(ISD::FMINIMUM, VT, Action);
610 setOperationAction(ISD::FMAXIMUM, VT, Action);
611 setOperationAction(ISD::FSIN, VT, Action);
612 setOperationAction(ISD::FCOS, VT, Action);
613 setOperationAction(ISD::FSINCOS, VT, Action);
614 setOperationAction(ISD::FTAN, VT, Action);
615 setOperationAction(ISD::FSQRT, VT, Action);
616 setOperationAction(ISD::FPOW, VT, Action);
617 setOperationAction(ISD::FLOG, VT, Action);
618 setOperationAction(ISD::FLOG2, VT, Action);
619 setOperationAction(ISD::FLOG10, VT, Action);
620 setOperationAction(ISD::FEXP, VT, Action);
621 setOperationAction(ISD::FEXP2, VT, Action);
622 setOperationAction(ISD::FEXP10, VT, Action);
623 setOperationAction(ISD::FCEIL, VT, Action);
624 setOperationAction(ISD::FFLOOR, VT, Action);
625 setOperationAction(ISD::FNEARBYINT, VT, Action);
626 setOperationAction(ISD::FRINT, VT, Action);
627 setOperationAction(ISD::BR_CC, VT, Action);
628 setOperationAction(ISD::SETCC, VT, Action);
629 setOperationAction(ISD::SELECT, VT, Custom);
630 setOperationAction(ISD::SELECT_CC, VT, Action);
631 setOperationAction(ISD::FROUND, VT, Action);
632 setOperationAction(ISD::FROUNDEVEN, VT, Action);
633 setOperationAction(ISD::FTRUNC, VT, Action);
634 setOperationAction(ISD::FLDEXP, VT, Action);
635 };
636
637 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
638 // f16, f32 and f64 use SSE.
639 // Set up the FP register classes.
640 addRegisterClass(MVT::f16, Subtarget.hasAVX512() ? &X86::FR16XRegClass
641 : &X86::FR16RegClass);
642 addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass
643 : &X86::FR32RegClass);
644 addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass
645 : &X86::FR64RegClass);
646
647 // Disable f32->f64 extload as we can only generate this in one instruction
648 // under optsize. So its easier to pattern match (fpext (load)) for that
649 // case instead of needing to emit 2 instructions for extload in the
650 // non-optsize case.
651 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
652
653 for (auto VT : { MVT::f32, MVT::f64 }) {
654 // Use ANDPD to simulate FABS.
655 setOperationAction(ISD::FABS, VT, Custom);
656
657 // Use XORP to simulate FNEG.
658 setOperationAction(ISD::FNEG, VT, Custom);
659
660 // Use ANDPD and ORPD to simulate FCOPYSIGN.
661 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
662
663 // These might be better off as horizontal vector ops.
664 setOperationAction(ISD::FADD, VT, Custom);
665 setOperationAction(ISD::FSUB, VT, Custom);
666
667 // We don't support sin/cos/fmod
668 setOperationAction(ISD::FSIN , VT, Expand);
669 setOperationAction(ISD::FCOS , VT, Expand);
670 setOperationAction(ISD::FSINCOS, VT, Expand);
671 }
672
673 // Half type will be promoted by default.
674 setF16Action(MVT::f16, Promote);
675 setOperationAction(ISD::FADD, MVT::f16, Promote);
676 setOperationAction(ISD::FSUB, MVT::f16, Promote);
677 setOperationAction(ISD::FMUL, MVT::f16, Promote);
678 setOperationAction(ISD::FDIV, MVT::f16, Promote);
679 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
680 setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);
681 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
682
683 setOperationAction(ISD::STRICT_FADD, MVT::f16, Promote);
684 setOperationAction(ISD::STRICT_FSUB, MVT::f16, Promote);
685 setOperationAction(ISD::STRICT_FMUL, MVT::f16, Promote);
686 setOperationAction(ISD::STRICT_FDIV, MVT::f16, Promote);
687 setOperationAction(ISD::STRICT_FMA, MVT::f16, Promote);
688 setOperationAction(ISD::STRICT_FMINNUM, MVT::f16, Promote);
689 setOperationAction(ISD::STRICT_FMAXNUM, MVT::f16, Promote);
690 setOperationAction(ISD::STRICT_FMINIMUM, MVT::f16, Promote);
691 setOperationAction(ISD::STRICT_FMAXIMUM, MVT::f16, Promote);
692 setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Promote);
693 setOperationAction(ISD::STRICT_FPOW, MVT::f16, Promote);
694 setOperationAction(ISD::STRICT_FLDEXP, MVT::f16, Promote);
695 setOperationAction(ISD::STRICT_FLOG, MVT::f16, Promote);
696 setOperationAction(ISD::STRICT_FLOG2, MVT::f16, Promote);
697 setOperationAction(ISD::STRICT_FLOG10, MVT::f16, Promote);
698 setOperationAction(ISD::STRICT_FEXP, MVT::f16, Promote);
699 setOperationAction(ISD::STRICT_FEXP2, MVT::f16, Promote);
700 setOperationAction(ISD::STRICT_FCEIL, MVT::f16, Promote);
701 setOperationAction(ISD::STRICT_FFLOOR, MVT::f16, Promote);
702 setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f16, Promote);
703 setOperationAction(ISD::STRICT_FRINT, MVT::f16, Promote);
704 setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Promote);
705 setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Promote);
706 setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
707 setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Promote);
708 setOperationAction(ISD::STRICT_FTRUNC, MVT::f16, Promote);
709 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
710 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom);
711 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
712
713 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
714 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
715
716 // Lower this to MOVMSK plus an AND.
717 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
718 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
719
720 } else if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1() &&
721 (UseX87 || Is64Bit)) {
722 // Use SSE for f32, x87 for f64.
723 // Set up the FP register classes.
724 addRegisterClass(MVT::f32, &X86::FR32RegClass);
725 if (UseX87)
726 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
727
728 // Use ANDPS to simulate FABS.
729 setOperationAction(ISD::FABS , MVT::f32, Custom);
730
731 // Use XORP to simulate FNEG.
732 setOperationAction(ISD::FNEG , MVT::f32, Custom);
733
734 if (UseX87)
735 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
736
737 // Use ANDPS and ORPS to simulate FCOPYSIGN.
738 if (UseX87)
739 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
740 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
741
742 // We don't support sin/cos/fmod
743 setOperationAction(ISD::FSIN , MVT::f32, Expand);
744 setOperationAction(ISD::FCOS , MVT::f32, Expand);
745 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
746
747 if (UseX87) {
748 // Always expand sin/cos functions even though x87 has an instruction.
749 setOperationAction(ISD::FSIN, MVT::f64, Expand);
750 setOperationAction(ISD::FCOS, MVT::f64, Expand);
751 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
752 }
753 } else if (UseX87) {
754 // f32 and f64 in x87.
755 // Set up the FP register classes.
756 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
757 addRegisterClass(MVT::f32, &X86::RFP32RegClass);
758
759 for (auto VT : { MVT::f32, MVT::f64 }) {
760 setOperationAction(ISD::UNDEF, VT, Expand);
761 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
762
763 // Always expand sin/cos functions even though x87 has an instruction.
764 setOperationAction(ISD::FSIN , VT, Expand);
765 setOperationAction(ISD::FCOS , VT, Expand);
766 setOperationAction(ISD::FSINCOS, VT, Expand);
767 }
768 }
769
770 // Expand FP32 immediates into loads from the stack, save special cases.
771 if (isTypeLegal(MVT::f32)) {
772 if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {
773 addLegalFPImmediate(APFloat(+0.0f)); // FLD0
774 addLegalFPImmediate(APFloat(+1.0f)); // FLD1
775 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
776 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
777 } else // SSE immediates.
778 addLegalFPImmediate(APFloat(+0.0f)); // xorps
779 }
780 // Expand FP64 immediates into loads from the stack, save special cases.
781 if (isTypeLegal(MVT::f64)) {
782 if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {
783 addLegalFPImmediate(APFloat(+0.0)); // FLD0
784 addLegalFPImmediate(APFloat(+1.0)); // FLD1
785 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
786 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
787 } else // SSE immediates.
788 addLegalFPImmediate(APFloat(+0.0)); // xorpd
789 }
790 // Support fp16 0 immediate.
791 if (isTypeLegal(MVT::f16))
792 addLegalFPImmediate(APFloat::getZero(APFloat::IEEEhalf()));
793
794 // Handle constrained floating-point operations of scalar.
795 setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
796 setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
797 setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
798 setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
799 setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
800 setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
801 setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
802 setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
803 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
804 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
805 setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
806 setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
807
808 // We don't support FMA.
809 setOperationAction(ISD::FMA, MVT::f64, Expand);
810 setOperationAction(ISD::FMA, MVT::f32, Expand);
811
812 // f80 always uses X87.
813 if (UseX87) {
814 addRegisterClass(MVT::f80, &X86::RFP80RegClass);
815 setOperationAction(ISD::UNDEF, MVT::f80, Expand);
816 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
817 {
818 APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
819 addLegalFPImmediate(TmpFlt); // FLD0
820 TmpFlt.changeSign();
821 addLegalFPImmediate(TmpFlt); // FLD0/FCHS
822
823 bool ignored;
824 APFloat TmpFlt2(+1.0);
825 TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
826 &ignored);
827 addLegalFPImmediate(TmpFlt2); // FLD1
828 TmpFlt2.changeSign();
829 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
830 }
831
832 // Always expand sin/cos functions even though x87 has an instruction.
833 // clang-format off
834 setOperationAction(ISD::FSIN , MVT::f80, Expand);
835 setOperationAction(ISD::FCOS , MVT::f80, Expand);
836 setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
837 setOperationAction(ISD::FTAN , MVT::f80, Expand);
838 setOperationAction(ISD::FASIN , MVT::f80, Expand);
839 setOperationAction(ISD::FACOS , MVT::f80, Expand);
840 setOperationAction(ISD::FATAN , MVT::f80, Expand);
841 setOperationAction(ISD::FSINH , MVT::f80, Expand);
842 setOperationAction(ISD::FCOSH , MVT::f80, Expand);
843 setOperationAction(ISD::FTANH , MVT::f80, Expand);
844 // clang-format on
845
846 setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
847 setOperationAction(ISD::FCEIL, MVT::f80, Expand);
848 setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
849 setOperationAction(ISD::FRINT, MVT::f80, Expand);
850 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
851 setOperationAction(ISD::FROUNDEVEN, MVT::f80, Expand);
852 setOperationAction(ISD::FMA, MVT::f80, Expand);
853 setOperationAction(ISD::LROUND, MVT::f80, LibCall);
854 setOperationAction(ISD::LLROUND, MVT::f80, LibCall);
855 setOperationAction(ISD::LRINT, MVT::f80, Custom);
856 setOperationAction(ISD::LLRINT, MVT::f80, Custom);
857
858 // Handle constrained floating-point operations of scalar.
859 setOperationAction(ISD::STRICT_FADD , MVT::f80, Legal);
860 setOperationAction(ISD::STRICT_FSUB , MVT::f80, Legal);
861 setOperationAction(ISD::STRICT_FMUL , MVT::f80, Legal);
862 setOperationAction(ISD::STRICT_FDIV , MVT::f80, Legal);
863 setOperationAction(ISD::STRICT_FSQRT , MVT::f80, Legal);
864 if (isTypeLegal(MVT::f16)) {
865 setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);
866 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Custom);
867 } else {
868 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Legal);
869 }
870 // FIXME: When the target is 64-bit, STRICT_FP_ROUND will be overwritten
871 // as Custom.
872 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Legal);
873 }
874
875 // f128 uses xmm registers, but most operations require libcalls.
876 if (!Subtarget.useSoftFloat() && Subtarget.is64Bit() && Subtarget.hasSSE1()) {
877 addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass
878 : &X86::VR128RegClass);
879
880 addLegalFPImmediate(APFloat::getZero(APFloat::IEEEquad())); // xorps
881
882 setOperationAction(ISD::FADD, MVT::f128, LibCall);
883 setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);
884 setOperationAction(ISD::FSUB, MVT::f128, LibCall);
885 setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);
886 setOperationAction(ISD::FDIV, MVT::f128, LibCall);
887 setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);
888 setOperationAction(ISD::FMUL, MVT::f128, LibCall);
889 setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);
890 setOperationAction(ISD::FMA, MVT::f128, LibCall);
891 setOperationAction(ISD::STRICT_FMA, MVT::f128, LibCall);
892
893 setOperationAction(ISD::FABS, MVT::f128, Custom);
894 setOperationAction(ISD::FNEG, MVT::f128, Custom);
895 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);
896
897 // clang-format off
898 setOperationAction(ISD::FSIN, MVT::f128, LibCall);
899 setOperationAction(ISD::STRICT_FSIN, MVT::f128, LibCall);
900 setOperationAction(ISD::FCOS, MVT::f128, LibCall);
901 setOperationAction(ISD::STRICT_FCOS, MVT::f128, LibCall);
902 setOperationAction(ISD::FSINCOS, MVT::f128, LibCall);
903 setOperationAction(ISD::FTAN, MVT::f128, LibCall);
904 setOperationAction(ISD::STRICT_FTAN, MVT::f128, LibCall);
905 // clang-format on
906 // No STRICT_FSINCOS
907 setOperationAction(ISD::FSQRT, MVT::f128, LibCall);
908 setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
909
910 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
911 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);
912 // We need to custom handle any FP_ROUND with an f128 input, but
913 // LegalizeDAG uses the result type to know when to run a custom handler.
914 // So we have to list all legal floating point result types here.
915 if (isTypeLegal(MVT::f32)) {
916 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
917 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
918 }
919 if (isTypeLegal(MVT::f64)) {
920 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
921 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
922 }
923 if (isTypeLegal(MVT::f80)) {
924 setOperationAction(ISD::FP_ROUND, MVT::f80, Custom);
925 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Custom);
926 }
927
928 setOperationAction(ISD::SETCC, MVT::f128, Custom);
929
930 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
931 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
932 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f80, Expand);
933 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
934 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
935 setTruncStoreAction(MVT::f128, MVT::f80, Expand);
936 }
937
938 // Always use a library call for pow.
939 setOperationAction(ISD::FPOW , MVT::f32 , Expand);
940 setOperationAction(ISD::FPOW , MVT::f64 , Expand);
941 setOperationAction(ISD::FPOW , MVT::f80 , Expand);
942 setOperationAction(ISD::FPOW , MVT::f128 , Expand);
943
944 setOperationAction(ISD::FLOG, MVT::f80, Expand);
945 setOperationAction(ISD::FLOG2, MVT::f80, Expand);
946 setOperationAction(ISD::FLOG10, MVT::f80, Expand);
947 setOperationAction(ISD::FEXP, MVT::f80, Expand);
948 setOperationAction(ISD::FEXP2, MVT::f80, Expand);
949 setOperationAction(ISD::FEXP10, MVT::f80, Expand);
950 setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
951 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
952
953 // Some FP actions are always expanded for vector types.
954 for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
955 MVT::v4f32, MVT::v8f32, MVT::v16f32,
956 MVT::v2f64, MVT::v4f64, MVT::v8f64 }) {
957 // clang-format off
958 setOperationAction(ISD::FSIN, VT, Expand);
959 setOperationAction(ISD::FSINCOS, VT, Expand);
960 setOperationAction(ISD::FCOS, VT, Expand);
961 setOperationAction(ISD::FTAN, VT, Expand);
962 setOperationAction(ISD::FREM, VT, Expand);
963 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
964 setOperationAction(ISD::FPOW, VT, Expand);
965 setOperationAction(ISD::FLOG, VT, Expand);
966 setOperationAction(ISD::FLOG2, VT, Expand);
967 setOperationAction(ISD::FLOG10, VT, Expand);
968 setOperationAction(ISD::FEXP, VT, Expand);
969 setOperationAction(ISD::FEXP2, VT, Expand);
970 setOperationAction(ISD::FEXP10, VT, Expand);
971 // clang-format on
972 }
973
974 // First set operation action for all vector types to either promote
975 // (for widening) or expand (for scalarization). Then we will selectively
976 // turn on ones that can be effectively codegen'd.
977 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
978 setOperationAction(ISD::SDIV, VT, Expand);
979 setOperationAction(ISD::UDIV, VT, Expand);
980 setOperationAction(ISD::SREM, VT, Expand);
981 setOperationAction(ISD::UREM, VT, Expand);
982 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
983 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
984 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
985 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
986 setOperationAction(ISD::FMA, VT, Expand);
987 setOperationAction(ISD::FFLOOR, VT, Expand);
988 setOperationAction(ISD::FCEIL, VT, Expand);
989 setOperationAction(ISD::FTRUNC, VT, Expand);
990 setOperationAction(ISD::FRINT, VT, Expand);
991 setOperationAction(ISD::FNEARBYINT, VT, Expand);
992 setOperationAction(ISD::FROUNDEVEN, VT, Expand);
993 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
994 setOperationAction(ISD::MULHS, VT, Expand);
995 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
996 setOperationAction(ISD::MULHU, VT, Expand);
997 setOperationAction(ISD::SDIVREM, VT, Expand);
998 setOperationAction(ISD::UDIVREM, VT, Expand);
999 setOperationAction(ISD::CTPOP, VT, Expand);
1000 setOperationAction(ISD::CTTZ, VT, Expand);
1001 setOperationAction(ISD::CTLZ, VT, Expand);
1002 setOperationAction(ISD::ROTL, VT, Expand);
1003 setOperationAction(ISD::ROTR, VT, Expand);
1004 setOperationAction(ISD::BSWAP, VT, Expand);
1005 setOperationAction(ISD::SETCC, VT, Expand);
1006 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
1007 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
1008 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
1009 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
1010 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
1011 setOperationAction(ISD::TRUNCATE, VT, Expand);
1012 setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
1013 setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
1014 setOperationAction(ISD::ANY_EXTEND, VT, Expand);
1015 setOperationAction(ISD::SELECT_CC, VT, Expand);
1016 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
1017 setTruncStoreAction(InnerVT, VT, Expand);
1018
1019 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
1020 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
1021
1022 // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
1023 // types, we have to deal with them whether we ask for Expansion or not.
1024 // Setting Expand causes its own optimisation problems though, so leave
1025 // them legal.
1026 if (VT.getVectorElementType() == MVT::i1)
1027 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
1028
1029 // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are
1030 // split/scalarized right now.
1031 if (VT.getVectorElementType() == MVT::f16 ||
1032 VT.getVectorElementType() == MVT::bf16)
1033 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
1034 }
1035 }
1036
1037 // FIXME: In order to prevent SSE instructions being expanded to MMX ones
1038 // with -msoft-float, disable use of MMX as well.
1039 if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) {
1040 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
1041 // No operations on x86mmx supported, everything uses intrinsics.
1042 }
1043
1044 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) {
1045 addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
1046 : &X86::VR128RegClass);
1047
1048 setOperationAction(ISD::FMAXIMUM, MVT::f32, Custom);
1049 setOperationAction(ISD::FMINIMUM, MVT::f32, Custom);
1050
1051 setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
1052 setOperationAction(ISD::FABS, MVT::v4f32, Custom);
1053 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom);
1054 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
1055 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
1056 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
1057 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
1058 setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
1059
1060 setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
1061 setOperationAction(ISD::STORE, MVT::v2f32, Custom);
1062
1063 setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
1064 setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
1065 setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
1066 setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
1067 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
1068 }
1069
1070 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
1071 addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1072 : &X86::VR128RegClass);
1073
1074 // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
1075 // registers cannot be used even for integer operations.
1076 addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass
1077 : &X86::VR128RegClass);
1078 addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1079 : &X86::VR128RegClass);
1080 addRegisterClass(MVT::v8f16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1081 : &X86::VR128RegClass);
1082 addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass
1083 : &X86::VR128RegClass);
1084 addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1085 : &X86::VR128RegClass);
1086
1087 for (auto VT : { MVT::f64, MVT::v4f32, MVT::v2f64 }) {
1088 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1089 setOperationAction(ISD::FMINIMUM, VT, Custom);
1090 }
1091
1092 for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,
1093 MVT::v2i16, MVT::v4i16, MVT::v2i32 }) {
1094 setOperationAction(ISD::SDIV, VT, Custom);
1095 setOperationAction(ISD::SREM, VT, Custom);
1096 setOperationAction(ISD::UDIV, VT, Custom);
1097 setOperationAction(ISD::UREM, VT, Custom);
1098 }
1099
1100 setOperationAction(ISD::MUL, MVT::v2i8, Custom);
1101 setOperationAction(ISD::MUL, MVT::v4i8, Custom);
1102 setOperationAction(ISD::MUL, MVT::v8i8, Custom);
1103
1104 setOperationAction(ISD::MUL, MVT::v16i8, Custom);
1105 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
1106 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1107 setOperationAction(ISD::MULHU, MVT::v4i32, Custom);
1108 setOperationAction(ISD::MULHS, MVT::v4i32, Custom);
1109 setOperationAction(ISD::MULHU, MVT::v16i8, Custom);
1110 setOperationAction(ISD::MULHS, MVT::v16i8, Custom);
1111 setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
1112 setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
1113 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
1114 setOperationAction(ISD::AVGCEILU, MVT::v16i8, Legal);
1115 setOperationAction(ISD::AVGCEILU, MVT::v8i16, Legal);
1116
1117 setOperationAction(ISD::SMULO, MVT::v16i8, Custom);
1118 setOperationAction(ISD::UMULO, MVT::v16i8, Custom);
1119 setOperationAction(ISD::UMULO, MVT::v2i32, Custom);
1120
1121 setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
1122 setOperationAction(ISD::FABS, MVT::v2f64, Custom);
1123 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom);
1124
1125 setOperationAction(ISD::LRINT, MVT::v4f32, Custom);
1126
1127 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1128 setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
1129 setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
1130 setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
1131 setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
1132 }
1133
1134 setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
1135 setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
1136 setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
1137 setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal);
1138 setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal);
1139 setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal);
1140 setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal);
1141 setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal);
1142 setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom);
1143 setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom);
1144
1145 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
1146 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
1147 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
1148 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
1149
1150 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1151 setOperationAction(ISD::SETCC, VT, Custom);
1152 setOperationAction(ISD::CTPOP, VT, Custom);
1153 setOperationAction(ISD::ABS, VT, Custom);
1154 setOperationAction(ISD::ABDS, VT, Custom);
1155 setOperationAction(ISD::ABDU, VT, Custom);
1156
1157 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1158 // setcc all the way to isel and prefer SETGT in some isel patterns.
1159 setCondCodeAction(ISD::SETLT, VT, Custom);
1160 setCondCodeAction(ISD::SETLE, VT, Custom);
1161 }
1162
1163 setOperationAction(ISD::SETCC, MVT::v2f64, Custom);
1164 setOperationAction(ISD::SETCC, MVT::v4f32, Custom);
1165 setOperationAction(ISD::STRICT_FSETCC, MVT::v2f64, Custom);
1166 setOperationAction(ISD::STRICT_FSETCC, MVT::v4f32, Custom);
1167 setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
1168 setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
1169
1170 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
1171 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1172 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1173 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1174 setOperationAction(ISD::VSELECT, VT, Custom);
1175 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1176 }
1177
1178 for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) {
1179 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1180 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1181 setOperationAction(ISD::VSELECT, VT, Custom);
1182
1183 if (VT == MVT::v2i64 && !Subtarget.is64Bit())
1184 continue;
1185
1186 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1187 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1188 }
1189 setF16Action(MVT::v8f16, Expand);
1190 setOperationAction(ISD::FADD, MVT::v8f16, Expand);
1191 setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
1192 setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
1193 setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
1194 setOperationAction(ISD::FNEG, MVT::v8f16, Custom);
1195 setOperationAction(ISD::FABS, MVT::v8f16, Custom);
1196 setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Custom);
1197
1198 // Custom lower v2i64 and v2f64 selects.
1199 setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
1200 setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
1201 setOperationAction(ISD::SELECT, MVT::v4i32, Custom);
1202 setOperationAction(ISD::SELECT, MVT::v8i16, Custom);
1203 setOperationAction(ISD::SELECT, MVT::v8f16, Custom);
1204 setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
1205
1206 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Custom);
1207 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Custom);
1208 setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
1209 setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom);
1210 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Custom);
1211 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i32, Custom);
1212
1213 // Custom legalize these to avoid over promotion or custom promotion.
1214 for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {
1215 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1216 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1217 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1218 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1219 }
1220
1221 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
1222 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Custom);
1223 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
1224 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i32, Custom);
1225
1226 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
1227 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i32, Custom);
1228
1229 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
1230 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Custom);
1231
1232 // Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
1233 setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
1234 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f32, Custom);
1235 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
1236 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f32, Custom);
1237
1238 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
1239 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f32, Custom);
1240 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
1241 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f32, Custom);
1242
1243 // We want to legalize this to an f64 load rather than an i64 load on
1244 // 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
1245 // store.
1246 setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
1247 setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
1248 setOperationAction(ISD::LOAD, MVT::v8i8, Custom);
1249 setOperationAction(ISD::STORE, MVT::v2i32, Custom);
1250 setOperationAction(ISD::STORE, MVT::v4i16, Custom);
1251 setOperationAction(ISD::STORE, MVT::v8i8, Custom);
1252
1253 // Add 32-bit vector stores to help vectorization opportunities.
1254 setOperationAction(ISD::STORE, MVT::v2i16, Custom);
1255 setOperationAction(ISD::STORE, MVT::v4i8, Custom);
1256
1257 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
1258 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
1259 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
1260 if (!Subtarget.hasAVX512())
1261 setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
1262
1263 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);
1264 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
1265 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
1266
1267 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
1268
1269 setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
1270 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
1271 setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom);
1272 setOperationAction(ISD::TRUNCATE, MVT::v2i64, Custom);
1273 setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
1274 setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
1275 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
1276 setOperationAction(ISD::TRUNCATE, MVT::v4i64, Custom);
1277 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
1278 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
1279 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
1280 setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);
1281 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
1282 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
1283 setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);
1284 setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);
1285
1286 // In the customized shift lowering, the legal v4i32/v2i64 cases
1287 // in AVX2 will be recognized.
1288 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1289 setOperationAction(ISD::SRL, VT, Custom);
1290 setOperationAction(ISD::SHL, VT, Custom);
1291 setOperationAction(ISD::SRA, VT, Custom);
1292 if (VT == MVT::v2i64) continue;
1293 setOperationAction(ISD::ROTL, VT, Custom);
1294 setOperationAction(ISD::ROTR, VT, Custom);
1295 setOperationAction(ISD::FSHL, VT, Custom);
1296 setOperationAction(ISD::FSHR, VT, Custom);
1297 }
1298
1299 setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
1300 setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
1301 setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
1302 setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
1303 setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
1304 }
1305
1306 if (Subtarget.hasGFNI()) {
1307 setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
1308 setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
1309 setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
1310 setOperationAction(ISD::BITREVERSE, MVT::i64, Custom);
1311 }
1312
1313 if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
1314 setOperationAction(ISD::ABS, MVT::v16i8, Legal);
1315 setOperationAction(ISD::ABS, MVT::v8i16, Legal);
1316 setOperationAction(ISD::ABS, MVT::v4i32, Legal);
1317
1318 for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
1319 setOperationAction(ISD::BITREVERSE, VT, Custom);
1320 setOperationAction(ISD::CTLZ, VT, Custom);
1321 }
1322
1323 // These might be better off as horizontal vector ops.
1324 setOperationAction(ISD::ADD, MVT::i16, Custom);
1325 setOperationAction(ISD::ADD, MVT::i32, Custom);
1326 setOperationAction(ISD::SUB, MVT::i16, Custom);
1327 setOperationAction(ISD::SUB, MVT::i32, Custom);
1328 }
1329
1330 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
1331 for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
1332 setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
1333 setOperationAction(ISD::STRICT_FFLOOR, RoundedTy, Legal);
1334 setOperationAction(ISD::FCEIL, RoundedTy, Legal);
1335 setOperationAction(ISD::STRICT_FCEIL, RoundedTy, Legal);
1336 setOperationAction(ISD::FTRUNC, RoundedTy, Legal);
1337 setOperationAction(ISD::STRICT_FTRUNC, RoundedTy, Legal);
1338 setOperationAction(ISD::FRINT, RoundedTy, Legal);
1339 setOperationAction(ISD::STRICT_FRINT, RoundedTy, Legal);
1340 setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal);
1341 setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy, Legal);
1342 setOperationAction(ISD::FROUNDEVEN, RoundedTy, Legal);
1343 setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy, Legal);
1344
1345 setOperationAction(ISD::FROUND, RoundedTy, Custom);
1346 }
1347
1348 setOperationAction(ISD::SMAX, MVT::v16i8, Legal);
1349 setOperationAction(ISD::SMAX, MVT::v4i32, Legal);
1350 setOperationAction(ISD::UMAX, MVT::v8i16, Legal);
1351 setOperationAction(ISD::UMAX, MVT::v4i32, Legal);
1352 setOperationAction(ISD::SMIN, MVT::v16i8, Legal);
1353 setOperationAction(ISD::SMIN, MVT::v4i32, Legal);
1354 setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
1355 setOperationAction(ISD::UMIN, MVT::v4i32, Legal);
1356
1357 setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
1358 setOperationAction(ISD::SADDSAT, MVT::v2i64, Custom);
1359 setOperationAction(ISD::SSUBSAT, MVT::v2i64, Custom);
1360
1361 // FIXME: Do we need to handle scalar-to-vector here?
1362 setOperationAction(ISD::MUL, MVT::v4i32, Legal);
1363 setOperationAction(ISD::SMULO, MVT::v2i32, Custom);
1364
1365 // We directly match byte blends in the backend as they match the VSELECT
1366 // condition form.
1367 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
1368
1369 // SSE41 brings specific instructions for doing vector sign extend even in
1370 // cases where we don't have SRA.
1371 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1372 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
1373 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
1374 }
1375
1376 // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
1377 for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1378 setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
1379 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
1380 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
1381 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
1382 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
1383 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
1384 }
1385
1386 if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) {
1387 // We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can
1388 // do the pre and post work in the vector domain.
1389 setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Custom);
1390 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);
1391 // We need to mark SINT_TO_FP as Custom even though we want to expand it
1392 // so that DAG combine doesn't try to turn it into uint_to_fp.
1393 setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Custom);
1394 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);
1395 }
1396 }
1397
1398 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) {
1399 setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom);
1400 }
1401
1402 if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
1403 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1404 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1405 setOperationAction(ISD::ROTL, VT, Custom);
1406 setOperationAction(ISD::ROTR, VT, Custom);
1407 }
1408
1409 // XOP can efficiently perform BITREVERSE with VPPERM.
1410 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
1411 setOperationAction(ISD::BITREVERSE, VT, Custom);
1412 }
1413
1414 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
1415 bool HasInt256 = Subtarget.hasInt256();
1416
1417 addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
1418 : &X86::VR256RegClass);
1419 addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1420 : &X86::VR256RegClass);
1421 addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1422 : &X86::VR256RegClass);
1423 addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1424 : &X86::VR256RegClass);
1425 addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1426 : &X86::VR256RegClass);
1427 addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1428 : &X86::VR256RegClass);
1429 addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1430 : &X86::VR256RegClass);
1431
1432 for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
1433 setOperationAction(ISD::FFLOOR, VT, Legal);
1434 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
1435 setOperationAction(ISD::FCEIL, VT, Legal);
1436 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
1437 setOperationAction(ISD::FTRUNC, VT, Legal);
1438 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
1439 setOperationAction(ISD::FRINT, VT, Legal);
1440 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
1441 setOperationAction(ISD::FNEARBYINT, VT, Legal);
1442 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1443 setOperationAction(ISD::FROUNDEVEN, VT, Legal);
1444 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1445
1446 setOperationAction(ISD::FROUND, VT, Custom);
1447
1448 setOperationAction(ISD::FNEG, VT, Custom);
1449 setOperationAction(ISD::FABS, VT, Custom);
1450 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1451
1452 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1453 setOperationAction(ISD::FMINIMUM, VT, Custom);
1454 }
1455
1456 setOperationAction(ISD::LRINT, MVT::v8f32, Custom);
1457 setOperationAction(ISD::LRINT, MVT::v4f64, Custom);
1458
1459 // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
1460 // even though v8i16 is a legal type.
1461 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1462 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1463 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1464 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1465 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Custom);
1466 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Custom);
1467 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i32, Custom);
1468
1469 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Custom);
1470 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i32, Custom);
1471 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Expand);
1472 setOperationAction(ISD::FP_ROUND, MVT::v8f16, Expand);
1473 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
1474 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Custom);
1475
1476 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f32, Legal);
1477 setOperationAction(ISD::STRICT_FADD, MVT::v8f32, Legal);
1478 setOperationAction(ISD::STRICT_FADD, MVT::v4f64, Legal);
1479 setOperationAction(ISD::STRICT_FSUB, MVT::v8f32, Legal);
1480 setOperationAction(ISD::STRICT_FSUB, MVT::v4f64, Legal);
1481 setOperationAction(ISD::STRICT_FMUL, MVT::v8f32, Legal);
1482 setOperationAction(ISD::STRICT_FMUL, MVT::v4f64, Legal);
1483 setOperationAction(ISD::STRICT_FDIV, MVT::v8f32, Legal);
1484 setOperationAction(ISD::STRICT_FDIV, MVT::v4f64, Legal);
1485 setOperationAction(ISD::STRICT_FSQRT, MVT::v8f32, Legal);
1486 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f64, Legal);
1487
1488 if (!Subtarget.hasAVX512())
1489 setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
1490
1491 // In the customized shift lowering, the legal v8i32/v4i64 cases
1492 // in AVX2 will be recognized.
1493 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1494 setOperationAction(ISD::SRL, VT, Custom);
1495 setOperationAction(ISD::SHL, VT, Custom);
1496 setOperationAction(ISD::SRA, VT, Custom);
1497 setOperationAction(ISD::ABDS, VT, Custom);
1498 setOperationAction(ISD::ABDU, VT, Custom);
1499 if (VT == MVT::v4i64) continue;
1500 setOperationAction(ISD::ROTL, VT, Custom);
1501 setOperationAction(ISD::ROTR, VT, Custom);
1502 setOperationAction(ISD::FSHL, VT, Custom);
1503 setOperationAction(ISD::FSHR, VT, Custom);
1504 }
1505
1506 // These types need custom splitting if their input is a 128-bit vector.
1507 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1508 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1509 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1510 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1511
1512 setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
1513 setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
1514 setOperationAction(ISD::SELECT, MVT::v8i32, Custom);
1515 setOperationAction(ISD::SELECT, MVT::v16i16, Custom);
1516 setOperationAction(ISD::SELECT, MVT::v16f16, Custom);
1517 setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
1518 setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
1519
1520 for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1521 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1522 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1523 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1524 }
1525
1526 setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom);
1527 setOperationAction(ISD::TRUNCATE, MVT::v32i16, Custom);
1528 setOperationAction(ISD::TRUNCATE, MVT::v32i32, Custom);
1529 setOperationAction(ISD::TRUNCATE, MVT::v32i64, Custom);
1530
1531 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1532 setOperationAction(ISD::SETCC, VT, Custom);
1533 setOperationAction(ISD::CTPOP, VT, Custom);
1534 setOperationAction(ISD::CTLZ, VT, Custom);
1535 setOperationAction(ISD::BITREVERSE, VT, Custom);
1536
1537 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1538 // setcc all the way to isel and prefer SETGT in some isel patterns.
1539 setCondCodeAction(ISD::SETLT, VT, Custom);
1540 setCondCodeAction(ISD::SETLE, VT, Custom);
1541 }
1542
1543 setOperationAction(ISD::SETCC, MVT::v4f64, Custom);
1544 setOperationAction(ISD::SETCC, MVT::v8f32, Custom);
1545 setOperationAction(ISD::STRICT_FSETCC, MVT::v4f64, Custom);
1546 setOperationAction(ISD::STRICT_FSETCC, MVT::v8f32, Custom);
1547 setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f64, Custom);
1548 setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f32, Custom);
1549
1550 if (Subtarget.hasAnyFMA()) {
1551 for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
1552 MVT::v2f64, MVT::v4f64 }) {
1553 setOperationAction(ISD::FMA, VT, Legal);
1554 setOperationAction(ISD::STRICT_FMA, VT, Legal);
1555 }
1556 }
1557
1558 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1559 setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
1560 setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
1561 }
1562
1563 setOperationAction(ISD::MUL, MVT::v4i64, Custom);
1564 setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom);
1565 setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom);
1566 setOperationAction(ISD::MUL, MVT::v32i8, Custom);
1567
1568 setOperationAction(ISD::MULHU, MVT::v8i32, Custom);
1569 setOperationAction(ISD::MULHS, MVT::v8i32, Custom);
1570 setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom);
1571 setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom);
1572 setOperationAction(ISD::MULHU, MVT::v32i8, Custom);
1573 setOperationAction(ISD::MULHS, MVT::v32i8, Custom);
1574 setOperationAction(ISD::AVGCEILU, MVT::v16i16, HasInt256 ? Legal : Custom);
1575 setOperationAction(ISD::AVGCEILU, MVT::v32i8, HasInt256 ? Legal : Custom);
1576
1577 setOperationAction(ISD::SMULO, MVT::v32i8, Custom);
1578 setOperationAction(ISD::UMULO, MVT::v32i8, Custom);
1579
1580 setOperationAction(ISD::ABS, MVT::v4i64, Custom);
1581 setOperationAction(ISD::SMAX, MVT::v4i64, Custom);
1582 setOperationAction(ISD::UMAX, MVT::v4i64, Custom);
1583 setOperationAction(ISD::SMIN, MVT::v4i64, Custom);
1584 setOperationAction(ISD::UMIN, MVT::v4i64, Custom);
1585
1586 setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1587 setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1588 setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1589 setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1590 setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1591 setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1592 setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1593 setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1594 setOperationAction(ISD::UADDSAT, MVT::v8i32, Custom);
1595 setOperationAction(ISD::USUBSAT, MVT::v8i32, Custom);
1596 setOperationAction(ISD::UADDSAT, MVT::v4i64, Custom);
1597 setOperationAction(ISD::USUBSAT, MVT::v4i64, Custom);
1598
1599 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
1600 setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom);
1601 setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
1602 setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
1603 setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
1604 setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);
1605 }
1606
1607 for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
1608 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1609 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1610 }
1611
1612 if (HasInt256) {
1613 // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
1614 // when we have a 256bit-wide blend with immediate.
1615 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
1616 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);
1617
1618 // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
1619 for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1620 setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
1621 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
1622 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
1623 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
1624 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
1625 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
1626 }
1627 }
1628
1629 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1630 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
1631 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
1632 setOperationAction(ISD::MSTORE, VT, Legal);
1633 }
1634
1635 // Extract subvector is special because the value type
1636 // (result) is 128-bit but the source is 256-bit wide.
1637 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1638 MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
1639 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1640 }
1641
1642 // Custom lower several nodes for 256-bit types.
1643 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1644 MVT::v16f16, MVT::v8f32, MVT::v4f64 }) {
1645 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1646 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1647 setOperationAction(ISD::VSELECT, VT, Custom);
1648 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1649 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1650 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1651 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
1652 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1653 setOperationAction(ISD::STORE, VT, Custom);
1654 }
1655 setF16Action(MVT::v16f16, Expand);
1656 setOperationAction(ISD::FNEG, MVT::v16f16, Custom);
1657 setOperationAction(ISD::FABS, MVT::v16f16, Custom);
1658 setOperationAction(ISD::FCOPYSIGN, MVT::v16f16, Custom);
1659 setOperationAction(ISD::FADD, MVT::v16f16, Expand);
1660 setOperationAction(ISD::FSUB, MVT::v16f16, Expand);
1661 setOperationAction(ISD::FMUL, MVT::v16f16, Expand);
1662 setOperationAction(ISD::FDIV, MVT::v16f16, Expand);
1663
1664 if (HasInt256) {
1665 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
1666
1667 // Custom legalize 2x32 to get a little better code.
1668 setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
1669 setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
1670
1671 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1672 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
1673 setOperationAction(ISD::MGATHER, VT, Custom);
1674 }
1675 }
1676
1677 if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() &&
1678 Subtarget.hasF16C()) {
1679 for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) {
1680 setOperationAction(ISD::FP_ROUND, VT, Custom);
1681 setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom);
1682 }
1683 for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32 }) {
1684 setOperationAction(ISD::FP_EXTEND, VT, Custom);
1685 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
1686 }
1687 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
1688 setOperationPromotedToType(Opc, MVT::v8f16, MVT::v8f32);
1689 setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
1690 }
1691 }
1692
1693 // This block controls legalization of the mask vector sizes that are
1694 // available with AVX512. 512-bit vectors are in a separate block controlled
1695 // by useAVX512Regs.
1696 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1697 addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
1698 addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
1699 addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
1700 addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
1701 addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
1702
1703 setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
1704 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
1705 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
1706
1707 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32);
1708 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32);
1709 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32);
1710 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32);
1711 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1, MVT::v8i32);
1712 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1, MVT::v8i32);
1713 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1, MVT::v4i32);
1714 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1, MVT::v4i32);
1715 setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom);
1716 setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom);
1717 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i1, Custom);
1718 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i1, Custom);
1719
1720 // There is no byte sized k-register load or store without AVX512DQ.
1721 if (!Subtarget.hasDQI()) {
1722 setOperationAction(ISD::LOAD, MVT::v1i1, Custom);
1723 setOperationAction(ISD::LOAD, MVT::v2i1, Custom);
1724 setOperationAction(ISD::LOAD, MVT::v4i1, Custom);
1725 setOperationAction(ISD::LOAD, MVT::v8i1, Custom);
1726
1727 setOperationAction(ISD::STORE, MVT::v1i1, Custom);
1728 setOperationAction(ISD::STORE, MVT::v2i1, Custom);
1729 setOperationAction(ISD::STORE, MVT::v4i1, Custom);
1730 setOperationAction(ISD::STORE, MVT::v8i1, Custom);
1731 }
1732
1733 // Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
1734 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1735 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1736 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1737 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1738 }
1739
1740 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 })
1741 setOperationAction(ISD::VSELECT, VT, Expand);
1742
1743 for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
1744 setOperationAction(ISD::SETCC, VT, Custom);
1745 setOperationAction(ISD::SELECT, VT, Custom);
1746 setOperationAction(ISD::TRUNCATE, VT, Custom);
1747
1748 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1749 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1750 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1751 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1752 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1753 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1754 }
1755
1756 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
1757 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1758 }
1759 if (Subtarget.hasDQI() && Subtarget.hasVLX()) {
1760 for (MVT VT : {MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
1761 setOperationAction(ISD::LRINT, VT, Legal);
1762 setOperationAction(ISD::LLRINT, VT, Legal);
1763 }
1764 }
1765
1766 // This block controls legalization for 512-bit operations with 8/16/32/64 bit
1767 // elements. 512-bits can be disabled based on prefer-vector-width and
1768 // required-vector-width function attributes.
1769 if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) {
1770 bool HasBWI = Subtarget.hasBWI();
1771
1772 addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1773 addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1774 addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
1775 addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
1776 addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1777 addRegisterClass(MVT::v32f16, &X86::VR512RegClass);
1778 addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
1779
1780 for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
1781 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal);
1782 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
1783 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal);
1784 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal);
1785 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal);
1786 if (HasBWI)
1787 setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
1788 }
1789
1790 for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
1791 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1792 setOperationAction(ISD::FMINIMUM, VT, Custom);
1793 setOperationAction(ISD::FNEG, VT, Custom);
1794 setOperationAction(ISD::FABS, VT, Custom);
1795 setOperationAction(ISD::FMA, VT, Legal);
1796 setOperationAction(ISD::STRICT_FMA, VT, Legal);
1797 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1798 }
1799 setOperationAction(ISD::LRINT, MVT::v16f32,
1800 Subtarget.hasDQI() ? Legal : Custom);
1801 setOperationAction(ISD::LRINT, MVT::v8f64,
1802 Subtarget.hasDQI() ? Legal : Custom);
1803 if (Subtarget.hasDQI())
1804 setOperationAction(ISD::LLRINT, MVT::v8f64, Legal);
1805
1806 for (MVT VT : { MVT::v16i1, MVT::v16i8 }) {
1807 setOperationPromotedToType(ISD::FP_TO_SINT , VT, MVT::v16i32);
1808 setOperationPromotedToType(ISD::FP_TO_UINT , VT, MVT::v16i32);
1809 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);
1810 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);
1811 }
1812
1813 for (MVT VT : { MVT::v16i16, MVT::v16i32 }) {
1814 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1815 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1816 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1817 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1818 }
1819
1820 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Custom);
1821 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Custom);
1822 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Custom);
1823 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Custom);
1824 setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
1825 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Custom);
1826
1827 setOperationAction(ISD::STRICT_FADD, MVT::v16f32, Legal);
1828 setOperationAction(ISD::STRICT_FADD, MVT::v8f64, Legal);
1829 setOperationAction(ISD::STRICT_FSUB, MVT::v16f32, Legal);
1830 setOperationAction(ISD::STRICT_FSUB, MVT::v8f64, Legal);
1831 setOperationAction(ISD::STRICT_FMUL, MVT::v16f32, Legal);
1832 setOperationAction(ISD::STRICT_FMUL, MVT::v8f64, Legal);
1833 setOperationAction(ISD::STRICT_FDIV, MVT::v16f32, Legal);
1834 setOperationAction(ISD::STRICT_FDIV, MVT::v8f64, Legal);
1835 setOperationAction(ISD::STRICT_FSQRT, MVT::v16f32, Legal);
1836 setOperationAction(ISD::STRICT_FSQRT, MVT::v8f64, Legal);
1837 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f32, Legal);
1838
1839 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
1840 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
1841 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
1842 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal);
1843 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal);
1844 if (HasBWI)
1845 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
1846
1847 // With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
1848 // to 512-bit rather than use the AVX2 instructions so that we can use
1849 // k-masks.
1850 if (!Subtarget.hasVLX()) {
1851 for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1852 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
1853 setOperationAction(ISD::MLOAD, VT, Custom);
1854 setOperationAction(ISD::MSTORE, VT, Custom);
1855 }
1856 }
1857
1858 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Legal);
1859 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Legal);
1860 setOperationAction(ISD::TRUNCATE, MVT::v32i8, HasBWI ? Legal : Custom);
1861 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
1862 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1863 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1864 setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom);
1865 setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom);
1866 setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom);
1867 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
1868 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1869 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1870
1871 if (HasBWI) {
1872 // Extends from v64i1 masks to 512-bit vectors.
1873 setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom);
1874 setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom);
1875 setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom);
1876 }
1877
1878 for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
1879 setOperationAction(ISD::FFLOOR, VT, Legal);
1880 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
1881 setOperationAction(ISD::FCEIL, VT, Legal);
1882 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
1883 setOperationAction(ISD::FTRUNC, VT, Legal);
1884 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
1885 setOperationAction(ISD::FRINT, VT, Legal);
1886 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
1887 setOperationAction(ISD::FNEARBYINT, VT, Legal);
1888 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1889 setOperationAction(ISD::FROUNDEVEN, VT, Legal);
1890 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1891
1892 setOperationAction(ISD::FROUND, VT, Custom);
1893 }
1894
1895 for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
1896 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1897 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1898 }
1899
1900 setOperationAction(ISD::ADD, MVT::v32i16, HasBWI ? Legal : Custom);
1901 setOperationAction(ISD::SUB, MVT::v32i16, HasBWI ? Legal : Custom);
1902 setOperationAction(ISD::ADD, MVT::v64i8, HasBWI ? Legal : Custom);
1903 setOperationAction(ISD::SUB, MVT::v64i8, HasBWI ? Legal : Custom);
1904
1905 setOperationAction(ISD::MUL, MVT::v8i64, Custom);
1906 setOperationAction(ISD::MUL, MVT::v16i32, Legal);
1907 setOperationAction(ISD::MUL, MVT::v32i16, HasBWI ? Legal : Custom);
1908 setOperationAction(ISD::MUL, MVT::v64i8, Custom);
1909
1910 setOperationAction(ISD::MULHU, MVT::v16i32, Custom);
1911 setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
1912 setOperationAction(ISD::MULHS, MVT::v32i16, HasBWI ? Legal : Custom);
1913 setOperationAction(ISD::MULHU, MVT::v32i16, HasBWI ? Legal : Custom);
1914 setOperationAction(ISD::MULHS, MVT::v64i8, Custom);
1915 setOperationAction(ISD::MULHU, MVT::v64i8, Custom);
1916 setOperationAction(ISD::AVGCEILU, MVT::v32i16, HasBWI ? Legal : Custom);
1917 setOperationAction(ISD::AVGCEILU, MVT::v64i8, HasBWI ? Legal : Custom);
1918
1919 setOperationAction(ISD::SMULO, MVT::v64i8, Custom);
1920 setOperationAction(ISD::UMULO, MVT::v64i8, Custom);
1921
1922 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
1923 setOperationAction(ISD::SRL, VT, Custom);
1924 setOperationAction(ISD::SHL, VT, Custom);
1925 setOperationAction(ISD::SRA, VT, Custom);
1926 setOperationAction(ISD::ROTL, VT, Custom);
1927 setOperationAction(ISD::ROTR, VT, Custom);
1928 setOperationAction(ISD::SETCC, VT, Custom);
1929 setOperationAction(ISD::ABDS, VT, Custom);
1930 setOperationAction(ISD::ABDU, VT, Custom);
1931 setOperationAction(ISD::BITREVERSE, VT, Custom);
1932
1933 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1934 // setcc all the way to isel and prefer SETGT in some isel patterns.
1935 setCondCodeAction(ISD::SETLT, VT, Custom);
1936 setCondCodeAction(ISD::SETLE, VT, Custom);
1937 }
1938
1939 setOperationAction(ISD::SETCC, MVT::v8f64, Custom);
1940 setOperationAction(ISD::SETCC, MVT::v16f32, Custom);
1941 setOperationAction(ISD::STRICT_FSETCC, MVT::v8f64, Custom);
1942 setOperationAction(ISD::STRICT_FSETCC, MVT::v16f32, Custom);
1943 setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f64, Custom);
1944 setOperationAction(ISD::STRICT_FSETCCS, MVT::v16f32, Custom);
1945
1946 for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
1947 setOperationAction(ISD::SMAX, VT, Legal);
1948 setOperationAction(ISD::UMAX, VT, Legal);
1949 setOperationAction(ISD::SMIN, VT, Legal);
1950 setOperationAction(ISD::UMIN, VT, Legal);
1951 setOperationAction(ISD::ABS, VT, Legal);
1952 setOperationAction(ISD::CTPOP, VT, Custom);
1953 }
1954
1955 for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1956 setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom);
1957 setOperationAction(ISD::CTPOP, VT, Subtarget.hasBITALG() ? Legal : Custom);
1958 setOperationAction(ISD::CTLZ, VT, Custom);
1959 setOperationAction(ISD::SMAX, VT, HasBWI ? Legal : Custom);
1960 setOperationAction(ISD::UMAX, VT, HasBWI ? Legal : Custom);
1961 setOperationAction(ISD::SMIN, VT, HasBWI ? Legal : Custom);
1962 setOperationAction(ISD::UMIN, VT, HasBWI ? Legal : Custom);
1963 setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom);
1964 setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom);
1965 setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom);
1966 setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom);
1967 }
1968
1969 setOperationAction(ISD::FSHL, MVT::v64i8, Custom);
1970 setOperationAction(ISD::FSHR, MVT::v64i8, Custom);
1971 setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
1972 setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
1973 setOperationAction(ISD::FSHL, MVT::v16i32, Custom);
1974 setOperationAction(ISD::FSHR, MVT::v16i32, Custom);
1975
1976 if (Subtarget.hasDQI()) {
1977 for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
1978 ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
1979 ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
1980 setOperationAction(Opc, MVT::v8i64, Custom);
1981 setOperationAction(ISD::MUL, MVT::v8i64, Legal);
1982 }
1983
1984 if (Subtarget.hasCDI()) {
1985 // NonVLX sub-targets extend 128/256 vectors to use the 512 version.
1986 for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
1987 setOperationAction(ISD::CTLZ, VT, Legal);
1988 }
1989 } // Subtarget.hasCDI()
1990
1991 if (Subtarget.hasVPOPCNTDQ()) {
1992 for (auto VT : { MVT::v16i32, MVT::v8i64 })
1993 setOperationAction(ISD::CTPOP, VT, Legal);
1994 }
1995
1996 // Extract subvector is special because the value type
1997 // (result) is 256-bit but the source is 512-bit wide.
1998 // 128-bit was made Legal under AVX1.
1999 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
2000 MVT::v16f16, MVT::v8f32, MVT::v4f64 })
2001 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
2002
2003 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
2004 MVT::v32f16, MVT::v16f32, MVT::v8f64 }) {
2005 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
2006 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
2007 setOperationAction(ISD::SELECT, VT, Custom);
2008 setOperationAction(ISD::VSELECT, VT, Custom);
2009 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2010 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2011 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2012 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
2013 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
2014 }
2015 setF16Action(MVT::v32f16, Expand);
2016 setOperationAction(ISD::FP_ROUND, MVT::v16f16, Custom);
2017 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Custom);
2018 setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
2019 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Custom);
2020 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
2021 setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);
2022
2023 for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
2024 setOperationAction(ISD::MLOAD, VT, Legal);
2025 setOperationAction(ISD::MSTORE, VT, Legal);
2026 setOperationAction(ISD::MGATHER, VT, Custom);
2027 setOperationAction(ISD::MSCATTER, VT, Custom);
2028 }
2029 if (HasBWI) {
2030 for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
2031 setOperationAction(ISD::MLOAD, VT, Legal);
2032 setOperationAction(ISD::MSTORE, VT, Legal);
2033 }
2034 } else {
2035 setOperationAction(ISD::STORE, MVT::v32i16, Custom);
2036 setOperationAction(ISD::STORE, MVT::v64i8, Custom);
2037 }
2038
2039 if (Subtarget.hasVBMI2()) {
2040 for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
2041 setOperationAction(ISD::FSHL, VT, Custom);
2042 setOperationAction(ISD::FSHR, VT, Custom);
2043 }
2044
2045 setOperationAction(ISD::ROTL, MVT::v32i16, Custom);
2046 setOperationAction(ISD::ROTR, MVT::v32i16, Custom);
2047 }
2048
2049 setOperationAction(ISD::FNEG, MVT::v32f16, Custom);
2050 setOperationAction(ISD::FABS, MVT::v32f16, Custom);
2051 setOperationAction(ISD::FCOPYSIGN, MVT::v32f16, Custom);
2052 }// useAVX512Regs
2053
2054 if (!Subtarget.useSoftFloat() && Subtarget.hasVBMI2()) {
2055 for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v16i16, MVT::v8i32,
2056 MVT::v4i64}) {
2057 setOperationAction(ISD::FSHL, VT, Custom);
2058 setOperationAction(ISD::FSHR, VT, Custom);
2059 }
2060 }
2061
2062 // This block controls legalization for operations that don't have
2063 // pre-AVX512 equivalents. Without VLX we use 512-bit operations for
2064 // narrower widths.
2065 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
2066 // These operations are handled on non-VLX by artificially widening in
2067 // isel patterns.
2068
2069 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i32, Custom);
2070 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Custom);
2071 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i32, Custom);
2072
2073 if (Subtarget.hasDQI()) {
2074 // Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
2075 // v2f32 UINT_TO_FP is already custom under SSE2.
2076 assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) &&
2077 isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&
2078 "Unexpected operation action!");
2079 // v2i64 FP_TO_S/UINT(v2f32) custom conversion.
2080 setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom);
2081 setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom);
2082 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);
2083 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);
2084 }
2085
2086 for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
2087 setOperationAction(ISD::SMAX, VT, Legal);
2088 setOperationAction(ISD::UMAX, VT, Legal);
2089 setOperationAction(ISD::SMIN, VT, Legal);
2090 setOperationAction(ISD::UMIN, VT, Legal);
2091 setOperationAction(ISD::ABS, VT, Legal);
2092 }
2093
2094 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2095 setOperationAction(ISD::ROTL, VT, Custom);
2096 setOperationAction(ISD::ROTR, VT, Custom);
2097 }
2098
2099 // Custom legalize 2x32 to get a little better code.
2100 setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
2101 setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
2102
2103 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
2104 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
2105 setOperationAction(ISD::MSCATTER, VT, Custom);
2106
2107 if (Subtarget.hasDQI()) {
2108 for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
2109 ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
2110 ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT}) {
2111 setOperationAction(Opc, MVT::v2i64, Custom);
2112 setOperationAction(Opc, MVT::v4i64, Custom);
2113 }
2114 setOperationAction(ISD::MUL, MVT::v2i64, Legal);
2115 setOperationAction(ISD::MUL, MVT::v4i64, Legal);
2116 }
2117
2118 if (Subtarget.hasCDI()) {
2119 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2120 setOperationAction(ISD::CTLZ, VT, Legal);
2121 }
2122 } // Subtarget.hasCDI()
2123
2124 if (Subtarget.hasVPOPCNTDQ()) {
2125 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 })
2126 setOperationAction(ISD::CTPOP, VT, Legal);
2127 }
2128 }
2129
2130 // This block control legalization of v32i1/v64i1 which are available with
2131 // AVX512BW..
2132 if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
2133 addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
2134 addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
2135
2136 for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
2137 setOperationAction(ISD::VSELECT, VT, Expand);
2138 setOperationAction(ISD::TRUNCATE, VT, Custom);
2139 setOperationAction(ISD::SETCC, VT, Custom);
2140 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2141 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
2142 setOperationAction(ISD::SELECT, VT, Custom);
2143 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2144 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2145 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
2146 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
2147 }
2148
2149 for (auto VT : { MVT::v16i1, MVT::v32i1 })
2150 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
2151
2152 // Extends from v32i1 masks to 256-bit vectors.
2153 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom);
2154 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom);
2155 setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom);
2156
2157 for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {
2158 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
2159 setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
2160 }
2161
2162 // These operations are handled on non-VLX by artificially widening in
2163 // isel patterns.
2164 // TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
2165
2166 if (Subtarget.hasBITALG()) {
2167 for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
2168 setOperationAction(ISD::CTPOP, VT, Legal);
2169 }
2170 }
2171
2172 if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
2173 auto setGroup = [&] (MVT VT) {
2174 setOperationAction(ISD::FADD, VT, Legal);
2175 setOperationAction(ISD::STRICT_FADD, VT, Legal);
2176 setOperationAction(ISD::FSUB, VT, Legal);
2177 setOperationAction(ISD::STRICT_FSUB, VT, Legal);
2178 setOperationAction(ISD::FMUL, VT, Legal);
2179 setOperationAction(ISD::STRICT_FMUL, VT, Legal);
2180 setOperationAction(ISD::FDIV, VT, Legal);
2181 setOperationAction(ISD::STRICT_FDIV, VT, Legal);
2182 setOperationAction(ISD::FSQRT, VT, Legal);
2183 setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
2184
2185 setOperationAction(ISD::FFLOOR, VT, Legal);
2186 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
2187 setOperationAction(ISD::FCEIL, VT, Legal);
2188 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
2189 setOperationAction(ISD::FTRUNC, VT, Legal);
2190 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
2191 setOperationAction(ISD::FRINT, VT, Legal);
2192 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
2193 setOperationAction(ISD::FNEARBYINT, VT, Legal);
2194 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
2195 setOperationAction(ISD::FROUNDEVEN, VT, Legal);
2196 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
2197
2198 setOperationAction(ISD::FROUND, VT, Custom);
2199
2200 setOperationAction(ISD::LOAD, VT, Legal);
2201 setOperationAction(ISD::STORE, VT, Legal);
2202
2203 setOperationAction(ISD::FMA, VT, Legal);
2204 setOperationAction(ISD::STRICT_FMA, VT, Legal);
2205 setOperationAction(ISD::VSELECT, VT, Legal);
2206 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2207 setOperationAction(ISD::SELECT, VT, Custom);
2208
2209 setOperationAction(ISD::FNEG, VT, Custom);
2210 setOperationAction(ISD::FABS, VT, Custom);
2211 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
2212 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2213 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2214
2215 setOperationAction(ISD::SETCC, VT, Custom);
2216 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
2217 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
2218 };
2219
2220 // AVX512_FP16 scalar operations
2221 setGroup(MVT::f16);
2222 setOperationAction(ISD::FREM, MVT::f16, Promote);
2223 setOperationAction(ISD::STRICT_FREM, MVT::f16, Promote);
2224 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
2225 setOperationAction(ISD::BR_CC, MVT::f16, Expand);
2226 setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
2227 setOperationAction(ISD::FROUNDEVEN, MVT::f16, Legal);
2228 setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Legal);
2229 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
2230 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
2231 setOperationAction(ISD::FMAXIMUM, MVT::f16, Custom);
2232 setOperationAction(ISD::FMINIMUM, MVT::f16, Custom);
2233 setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
2234 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
2235
2236 setCondCodeAction(ISD::SETOEQ, MVT::f16, Expand);
2237 setCondCodeAction(ISD::SETUNE, MVT::f16, Expand);
2238
2239 if (Subtarget.useAVX512Regs()) {
2240 setGroup(MVT::v32f16);
2241 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32f16, Custom);
2242 setOperationAction(ISD::SINT_TO_FP, MVT::v32i16, Legal);
2243 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v32i16, Legal);
2244 setOperationAction(ISD::UINT_TO_FP, MVT::v32i16, Legal);
2245 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v32i16, Legal);
2246 setOperationAction(ISD::FP_ROUND, MVT::v16f16, Legal);
2247 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Legal);
2248 setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
2249 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Legal);
2250 setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
2251 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Legal);
2252 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32f16, Custom);
2253
2254 setOperationAction(ISD::FP_TO_SINT, MVT::v32i16, Custom);
2255 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v32i16, Custom);
2256 setOperationAction(ISD::FP_TO_UINT, MVT::v32i16, Custom);
2257 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v32i16, Custom);
2258 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i8, MVT::v32i16);
2259 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i8,
2260 MVT::v32i16);
2261 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i8, MVT::v32i16);
2262 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i8,
2263 MVT::v32i16);
2264 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i1, MVT::v32i16);
2265 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i1,
2266 MVT::v32i16);
2267 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i1, MVT::v32i16);
2268 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i1,
2269 MVT::v32i16);
2270
2271 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f16, Legal);
2272 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32f16, Legal);
2273 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32f16, Custom);
2274
2275 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Legal);
2276 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal);
2277 }
2278
2279 if (Subtarget.hasVLX()) {
2280 setGroup(MVT::v8f16);
2281 setGroup(MVT::v16f16);
2282
2283 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8f16, Legal);
2284 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16f16, Custom);
2285 setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Legal);
2286 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i16, Legal);
2287 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Legal);
2288 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i16, Legal);
2289 setOperationAction(ISD::UINT_TO_FP, MVT::v16i16, Legal);
2290 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i16, Legal);
2291 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Legal);
2292 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i16, Legal);
2293
2294 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
2295 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i16, Custom);
2296 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
2297 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i16, Custom);
2298 setOperationAction(ISD::FP_ROUND, MVT::v8f16, Legal);
2299 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f16, Legal);
2300 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Custom);
2301 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f32, Legal);
2302 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
2303 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Legal);
2304
2305 // INSERT_VECTOR_ELT v8f16 extended to VECTOR_SHUFFLE
2306 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f16, Custom);
2307 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16f16, Custom);
2308
2309 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f16, Legal);
2310 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16f16, Legal);
2311 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f16, Custom);
2312
2313 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Legal);
2314 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Legal);
2315 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Legal);
2316 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Legal);
2317
2318 // Need to custom widen these to prevent scalarization.
2319 setOperationAction(ISD::LOAD, MVT::v4f16, Custom);
2320 setOperationAction(ISD::STORE, MVT::v4f16, Custom);
2321 }
2322 }
2323
2324 if (!Subtarget.useSoftFloat() &&
2325 (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16())) {
2326 addRegisterClass(MVT::v8bf16, Subtarget.hasAVX512() ? &X86::VR128XRegClass
2327 : &X86::VR128RegClass);
2328 addRegisterClass(MVT::v16bf16, Subtarget.hasAVX512() ? &X86::VR256XRegClass
2329 : &X86::VR256RegClass);
2330 // We set the type action of bf16 to TypeSoftPromoteHalf, but we don't
2331 // provide the method to promote BUILD_VECTOR and INSERT_VECTOR_ELT.
2332 // Set the operation action Custom to do the customization later.
2333 setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
2334 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::bf16, Custom);
2335 for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
2336 setF16Action(VT, Expand);
2337 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2338 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2339 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
2340 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
2341 }
2342 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
2343 setOperationPromotedToType(Opc, MVT::v8bf16, MVT::v8f32);
2344 setOperationPromotedToType(Opc, MVT::v16bf16, MVT::v16f32);
2345 }
2346 setOperationAction(ISD::FP_ROUND, MVT::v8bf16, Custom);
2347 addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
2348 }
2349
2350 if (!Subtarget.useSoftFloat() && Subtarget.hasBF16()) {
2351 addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
2352 setF16Action(MVT::v32bf16, Expand);
2353 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
2354 setOperationPromotedToType(Opc, MVT::v32bf16, MVT::v32f32);
2355 setOperationAction(ISD::BUILD_VECTOR, MVT::v32bf16, Custom);
2356 setOperationAction(ISD::FP_ROUND, MVT::v16bf16, Custom);
2357 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32bf16, Custom);
2358 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32bf16, Legal);
2359 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32bf16, Custom);
2360 }
2361
2362 if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
2363 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal);
2364 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
2365 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
2366 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal);
2367 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
2368
2369 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal);
2370 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
2371 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
2372 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
2373 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
2374
2375 if (Subtarget.hasBWI()) {
2376 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal);
2377 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
2378 }
2379
2380 if (Subtarget.hasFP16()) {
2381 // vcvttph2[u]dq v4f16 -> v4i32/64, v2f16 -> v2i32/64
2382 setOperationAction(ISD::FP_TO_SINT, MVT::v2f16, Custom);
2383 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f16, Custom);
2384 setOperationAction(ISD::FP_TO_UINT, MVT::v2f16, Custom);
2385 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f16, Custom);
2386 setOperationAction(ISD::FP_TO_SINT, MVT::v4f16, Custom);
2387 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f16, Custom);
2388 setOperationAction(ISD::FP_TO_UINT, MVT::v4f16, Custom);
2389 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f16, Custom);
2390 // vcvt[u]dq2ph v4i32/64 -> v4f16, v2i32/64 -> v2f16
2391 setOperationAction(ISD::SINT_TO_FP, MVT::v2f16, Custom);
2392 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f16, Custom);
2393 setOperationAction(ISD::UINT_TO_FP, MVT::v2f16, Custom);
2394 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f16, Custom);
2395 setOperationAction(ISD::SINT_TO_FP, MVT::v4f16, Custom);
2396 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f16, Custom);
2397 setOperationAction(ISD::UINT_TO_FP, MVT::v4f16, Custom);
2398 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f16, Custom);
2399 // vcvtps2phx v4f32 -> v4f16, v2f32 -> v2f16
2400 setOperationAction(ISD::FP_ROUND, MVT::v2f16, Custom);
2401 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f16, Custom);
2402 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
2403 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f16, Custom);
2404 // vcvtph2psx v4f16 -> v4f32, v2f16 -> v2f32
2405 setOperationAction(ISD::FP_EXTEND, MVT::v2f16, Custom);
2406 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f16, Custom);
2407 setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Custom);
2408 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f16, Custom);
2409 }
2410 }
2411
2412 if (!Subtarget.useSoftFloat() && Subtarget.hasAMXTILE()) {
2413 addRegisterClass(MVT::x86amx, &X86::TILERegClass);
2414 }
2415
2416 // We want to custom lower some of our intrinsics.
2417 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
2418 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
2419 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
2420 if (!Subtarget.is64Bit()) {
2421 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
2422 }
2423
2424 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
2425 // handle type legalization for these operations here.
2426 //
2427 // FIXME: We really should do custom legalization for addition and
2428 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better
2429 // than generic legalization for 64-bit multiplication-with-overflow, though.
2430 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
2431 if (VT == MVT::i64 && !Subtarget.is64Bit())
2432 continue;
2433 // Add/Sub/Mul with overflow operations are custom lowered.
2434 setOperationAction(ISD::SADDO, VT, Custom);
2435 setOperationAction(ISD::UADDO, VT, Custom);
2436 setOperationAction(ISD::SSUBO, VT, Custom);
2437 setOperationAction(ISD::USUBO, VT, Custom);
2438 setOperationAction(ISD::SMULO, VT, Custom);
2439 setOperationAction(ISD::UMULO, VT, Custom);
2440
2441 // Support carry in as value rather than glue.
2442 setOperationAction(ISD::UADDO_CARRY, VT, Custom);
2443 setOperationAction(ISD::USUBO_CARRY, VT, Custom);
2444 setOperationAction(ISD::SETCCCARRY, VT, Custom);
2445 setOperationAction(ISD::SADDO_CARRY, VT, Custom);
2446 setOperationAction(ISD::SSUBO_CARRY, VT, Custom);
2447 }
2448
2449 // Combine sin / cos into _sincos_stret if it is available.
2450 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
2451 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
2452 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
2453 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
2454 }
2455
2456 if (Subtarget.isTargetWin64()) {
2457 setOperationAction(ISD::SDIV, MVT::i128, Custom);
2458 setOperationAction(ISD::UDIV, MVT::i128, Custom);
2459 setOperationAction(ISD::SREM, MVT::i128, Custom);
2460 setOperationAction(ISD::UREM, MVT::i128, Custom);
2461 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
2462 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
2463 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
2464 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
2465 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
2466 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
2467 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
2468 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
2469 }
2470
2471 // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)`
2472 // is. We should promote the value to 64-bits to solve this.
2473 // This is what the CRT headers do - `fmodf` is an inline header
2474 // function casting to f64 and calling `fmod`.
2475 if (Subtarget.is32Bit() &&
2476 (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()))
2477 // clang-format off
2478 for (ISD::NodeType Op :
2479 {ISD::FACOS, ISD::STRICT_FACOS,
2480 ISD::FASIN, ISD::STRICT_FASIN,
2481 ISD::FATAN, ISD::STRICT_FATAN,
2482 ISD::FCEIL, ISD::STRICT_FCEIL,
2483 ISD::FCOS, ISD::STRICT_FCOS,
2484 ISD::FCOSH, ISD::STRICT_FCOSH,
2485 ISD::FEXP, ISD::STRICT_FEXP,
2486 ISD::FFLOOR, ISD::STRICT_FFLOOR,
2487 ISD::FREM, ISD::STRICT_FREM,
2488 ISD::FLOG, ISD::STRICT_FLOG,
2489 ISD::FLOG10, ISD::STRICT_FLOG10,
2490 ISD::FPOW, ISD::STRICT_FPOW,
2491 ISD::FSIN, ISD::STRICT_FSIN,
2492 ISD::FSINH, ISD::STRICT_FSINH,
2493 ISD::FTAN, ISD::STRICT_FTAN,
2494 ISD::FTANH, ISD::STRICT_FTANH})
2495 if (isOperationExpand(Op, MVT::f32))
2496 setOperationAction(Op, MVT::f32, Promote);
2497 // clang-format on
2498
2499 // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined. MinGW has
2500 // it, but it's just a wrapper around ldexp.
2501 if (Subtarget.isOSWindows()) {
2502 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
2503 if (isOperationExpand(Op, MVT::f32))
2504 setOperationAction(Op, MVT::f32, Promote);
2505 }
2506
2507 // We have target-specific dag combine patterns for the following nodes:
2508 setTargetDAGCombine({ISD::VECTOR_SHUFFLE,
2509 ISD::SCALAR_TO_VECTOR,
2510 ISD::INSERT_VECTOR_ELT,
2511 ISD::EXTRACT_VECTOR_ELT,
2512 ISD::CONCAT_VECTORS,
2513 ISD::INSERT_SUBVECTOR,
2514 ISD::EXTRACT_SUBVECTOR,
2515 ISD::BITCAST,
2516 ISD::VSELECT,
2517 ISD::SELECT,
2518 ISD::SHL,
2519 ISD::SRA,
2520 ISD::SRL,
2521 ISD::OR,
2522 ISD::AND,
2523 ISD::AVGCEILS,
2524 ISD::AVGCEILU,
2525 ISD::AVGFLOORS,
2526 ISD::AVGFLOORU,
2527 ISD::BITREVERSE,
2528 ISD::ADD,
2529 ISD::FADD,
2530 ISD::FSUB,
2531 ISD::FNEG,
2532 ISD::FMA,
2533 ISD::STRICT_FMA,
2534 ISD::FMINNUM,
2535 ISD::FMAXNUM,
2536 ISD::SUB,
2537 ISD::LOAD,
2538 ISD::LRINT,
2539 ISD::LLRINT,
2540 ISD::MLOAD,
2541 ISD::STORE,
2542 ISD::MSTORE,
2543 ISD::TRUNCATE,
2544 ISD::ZERO_EXTEND,
2545 ISD::ANY_EXTEND,
2546 ISD::SIGN_EXTEND,
2547 ISD::SIGN_EXTEND_INREG,
2548 ISD::ANY_EXTEND_VECTOR_INREG,
2549 ISD::SIGN_EXTEND_VECTOR_INREG,
2550 ISD::ZERO_EXTEND_VECTOR_INREG,
2551 ISD::SINT_TO_FP,
2552 ISD::UINT_TO_FP,
2553 ISD::STRICT_SINT_TO_FP,
2554 ISD::STRICT_UINT_TO_FP,
2555 ISD::SETCC,
2556 ISD::MUL,
2557 ISD::XOR,
2558 ISD::MSCATTER,
2559 ISD::MGATHER,
2560 ISD::FP16_TO_FP,
2561 ISD::FP_EXTEND,
2562 ISD::STRICT_FP_EXTEND,
2563 ISD::FP_ROUND,
2564 ISD::STRICT_FP_ROUND,
2565 ISD::INTRINSIC_VOID,
2566 ISD::INTRINSIC_WO_CHAIN,
2567 ISD::INTRINSIC_W_CHAIN});
2568
2569 computeRegisterProperties(Subtarget.getRegisterInfo());
2570
2571 MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
2572 MaxStoresPerMemsetOptSize = 8;
2573 MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
2574 MaxStoresPerMemcpyOptSize = 4;
2575 MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
2576 MaxStoresPerMemmoveOptSize = 4;
2577
2578 // TODO: These control memcmp expansion in CGP and could be raised higher, but
2579 // that needs to benchmarked and balanced with the potential use of vector
2580 // load/store types (PR33329, PR33914).
2581 MaxLoadsPerMemcmp = 2;
2582 MaxLoadsPerMemcmpOptSize = 2;
2583
2584 // Default loop alignment, which can be overridden by -align-loops.
2585 setPrefLoopAlignment(Align(16));
2586
2587 // An out-of-order CPU can speculatively execute past a predictable branch,
2588 // but a conditional move could be stalled by an expensive earlier operation.
2589 PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder();
2590 EnableExtLdPromotion = true;
2591 setPrefFunctionAlignment(Align(16));
2592
2593 verifyIntrinsicTables();
2594
2595 // Default to having -disable-strictnode-mutation on
2596 IsStrictFPEnabled = true;
2597}
2598
2599// This has so far only been implemented for 64-bit MachO.
2600bool X86TargetLowering::useLoadStackGuardNode() const {
2601 return Subtarget.isTargetMachO() && Subtarget.is64Bit();
2602}
2603
2604bool X86TargetLowering::useStackGuardXorFP() const {
2605 // Currently only MSVC CRTs XOR the frame pointer into the stack guard value.
2606 return Subtarget.getTargetTriple().isOSMSVCRT() && !Subtarget.isTargetMachO();
2607}
2608
2609SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2610 const SDLoc &DL) const {
2611 EVT PtrTy = getPointerTy(DAG.getDataLayout());
2612 unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
2613 MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
2614 return SDValue(Node, 0);
2615}
2616
2617TargetLoweringBase::LegalizeTypeAction
2618X86TargetLowering::getPreferredVectorAction(MVT VT) const {
2619 if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() &&
2620 !Subtarget.hasBWI())
2621 return TypeSplitVector;
2622
2623 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2624 !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16)
2625 return TypeSplitVector;
2626
2627 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2628 VT.getVectorElementType() != MVT::i1)
2629 return TypeWidenVector;
2630
2631 return TargetLoweringBase::getPreferredVectorAction(VT);
2632}
2633
2634FastISel *
2635X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
2636 const TargetLibraryInfo *libInfo) const {
2637 return X86::createFastISel(funcInfo, libInfo);
2638}
2639
2640//===----------------------------------------------------------------------===//
2641// Other Lowering Hooks
2642//===----------------------------------------------------------------------===//
2643
2644bool X86::mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
2645 bool AssumeSingleUse) {
2646 if (!AssumeSingleUse && !Op.hasOneUse())
2647 return false;
2648 if (!ISD::isNormalLoad(Op.getNode()))
2649 return false;
2650
2651 // If this is an unaligned vector, make sure the target supports folding it.
2652 auto *Ld = cast<LoadSDNode>(Op.getNode());
2653 if (!Subtarget.hasAVX() && !Subtarget.hasSSEUnalignedMem() &&
2654 Ld->getValueSizeInBits(0) == 128 && Ld->getAlign() < Align(16))
2655 return false;
2656
2657 // TODO: If this is a non-temporal load and the target has an instruction
2658 // for it, it should not be folded. See "useNonTemporalLoad()".
2659
2660 return true;
2661}
2662
2663bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
2664 const X86Subtarget &Subtarget,
2665 bool AssumeSingleUse) {
2666 assert(Subtarget.hasAVX() && "Expected AVX for broadcast from memory");
2667 if (!X86::mayFoldLoad(Op, Subtarget, AssumeSingleUse))
2668 return false;
2669
2670 // We can not replace a wide volatile load with a broadcast-from-memory,
2671 // because that would narrow the load, which isn't legal for volatiles.
2672 auto *Ld = cast<LoadSDNode>(Op.getNode());
2673 return !Ld->isVolatile() ||
2674 Ld->getValueSizeInBits(0) == EltVT.getScalarSizeInBits();
2675}
2676
2677bool X86::mayFoldIntoStore(SDValue Op) {
2678 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
2679}
2680
2681bool X86::mayFoldIntoZeroExtend(SDValue Op) {
2682 if (Op.hasOneUse()) {
2683 unsigned Opcode = Op.getNode()->use_begin()->getOpcode();
2684 return (ISD::ZERO_EXTEND == Opcode);
2685 }
2686 return false;
2687}
2688
2689static bool isLogicOp(unsigned Opcode) {
2690 // TODO: Add support for X86ISD::FAND/FOR/FXOR/FANDN with test coverage.
2691 return ISD::isBitwiseLogicOp(Opcode) || X86ISD::ANDNP == Opcode;
2692}
2693
2694static bool isTargetShuffle(unsigned Opcode) {
2695 switch(Opcode) {
2696 default: return false;
2697 case X86ISD::BLENDI:
2698 case X86ISD::PSHUFB:
2699 case X86ISD::PSHUFD:
2700 case X86ISD::PSHUFHW:
2701 case X86ISD::PSHUFLW:
2702 case X86ISD::SHUFP:
2703 case X86ISD::INSERTPS:
2704 case X86ISD::EXTRQI:
2705 case X86ISD::INSERTQI:
2706 case X86ISD::VALIGN:
2707 case X86ISD::PALIGNR:
2708 case X86ISD::VSHLDQ:
2709 case X86ISD::VSRLDQ:
2710 case X86ISD::MOVLHPS:
2711 case X86ISD::MOVHLPS:
2712 case X86ISD::MOVSHDUP:
2713 case X86ISD::MOVSLDUP:
2714 case X86ISD::MOVDDUP:
2715 case X86ISD::MOVSS:
2716 case X86ISD::MOVSD:
2717 case X86ISD::MOVSH:
2718 case X86ISD::UNPCKL:
2719 case X86ISD::UNPCKH:
2720 case X86ISD::VBROADCAST:
2721 case X86ISD::VPERMILPI:
2722 case X86ISD::VPERMILPV:
2723 case X86ISD::VPERM2X128:
2724 case X86ISD::SHUF128:
2725 case X86ISD::VPERMIL2:
2726 case X86ISD::VPERMI:
2727 case X86ISD::VPPERM:
2728 case X86ISD::VPERMV:
2729 case X86ISD::VPERMV3:
2730 case X86ISD::VZEXT_MOVL:
2731 return true;
2732 }
2733}
2734
2735static bool isTargetShuffleVariableMask(unsigned Opcode) {
2736 switch (Opcode) {
2737 default: return false;
2738 // Target Shuffles.
2739 case X86ISD::PSHUFB:
2740 case X86ISD::VPERMILPV:
2741 case X86ISD::VPERMIL2:
2742 case X86ISD::VPPERM:
2743 case X86ISD::VPERMV:
2744 case X86ISD::VPERMV3:
2745 return true;
2746 // 'Faux' Target Shuffles.
2747 case ISD::OR:
2748 case ISD::AND:
2749 case X86ISD::ANDNP:
2750 return true;
2751 }
2752}
2753
2754SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
2755 MachineFunction &MF = DAG.getMachineFunction();
2756 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
2757 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2758 int ReturnAddrIndex = FuncInfo->getRAIndex();
2759
2760 if (ReturnAddrIndex == 0) {
2761 // Set up a frame object for the return address.
2762 unsigned SlotSize = RegInfo->getSlotSize();
2763 ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(SlotSize,
2764 -(int64_t)SlotSize,
2765 false);
2766 FuncInfo->setRAIndex(ReturnAddrIndex);
2767 }
2768
2769 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout()));
2770}
2771
2772bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model CM,
2773 bool HasSymbolicDisplacement) {
2774 // Offset should fit into 32 bit immediate field.
2775 if (!isInt<32>(Offset))
2776 return false;
2777
2778 // If we don't have a symbolic displacement - we don't have any extra
2779 // restrictions.
2780 if (!HasSymbolicDisplacement)
2781 return true;
2782
2783 // We can fold large offsets in the large code model because we always use
2784 // 64-bit offsets.
2785 if (CM == CodeModel::Large)
2786 return true;
2787
2788 // For kernel code model we know that all object resist in the negative half
2789 // of 32bits address space. We may not accept negative offsets, since they may
2790 // be just off and we may accept pretty large positive ones.
2791 if (CM == CodeModel::Kernel)
2792 return Offset >= 0;
2793
2794 // For other non-large code models we assume that latest small object is 16MB
2795 // before end of 31 bits boundary. We may also accept pretty large negative
2796 // constants knowing that all objects are in the positive half of address
2797 // space.
2798 return Offset < 16 * 1024 * 1024;
2799}
2800
2801/// Return true if the condition is an signed comparison operation.
2802static bool isX86CCSigned(unsigned X86CC) {
2803 switch (X86CC) {
2804 default:
2805 llvm_unreachable("Invalid integer condition!");
2806 case X86::COND_E:
2807 case X86::COND_NE:
2808 case X86::COND_B:
2809 case X86::COND_A:
2810 case X86::COND_BE:
2811 case X86::COND_AE:
2812 return false;
2813 case X86::COND_G:
2814 case X86::COND_GE:
2815 case X86::COND_L:
2816 case X86::COND_LE:
2817 return true;
2818 }
2819}
2820
2821static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
2822 switch (SetCCOpcode) {
2823 // clang-format off
2824 default: llvm_unreachable("Invalid integer condition!");
2825 case ISD::SETEQ: return X86::COND_E;
2826 case ISD::SETGT: return X86::COND_G;
2827 case ISD::SETGE: return X86::COND_GE;
2828 case ISD::SETLT: return X86::COND_L;
2829 case ISD::SETLE: return X86::COND_LE;
2830 case ISD::SETNE: return X86::COND_NE;
2831 case ISD::SETULT: return X86::COND_B;
2832 case ISD::SETUGT: return X86::COND_A;
2833 case ISD::SETULE: return X86::COND_BE;
2834 case ISD::SETUGE: return X86::COND_AE;
2835 // clang-format on
2836 }
2837}
2838
2839/// Do a one-to-one translation of a ISD::CondCode to the X86-specific
2840/// condition code, returning the condition code and the LHS/RHS of the
2841/// comparison to make.
2842static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
2843 bool isFP, SDValue &LHS, SDValue &RHS,
2844 SelectionDAG &DAG) {
2845 if (!isFP) {
2846 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
2847 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnes()) {
2848 // X > -1 -> X == 0, jump !sign.
2849 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2850 return X86::COND_NS;
2851 }
2852 if (SetCCOpcode == ISD::SETLT && RHSC->isZero()) {
2853 // X < 0 -> X == 0, jump on sign.
2854 return X86::COND_S;
2855 }
2856 if (SetCCOpcode == ISD::SETGE && RHSC->isZero()) {
2857 // X >= 0 -> X == 0, jump on !sign.
2858 return X86::COND_NS;
2859 }
2860 if (SetCCOpcode == ISD::SETLT && RHSC->isOne()) {
2861 // X < 1 -> X <= 0
2862 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2863 return X86::COND_LE;
2864 }
2865 }
2866
2867 return TranslateIntegerX86CC(SetCCOpcode);
2868 }
2869
2870 // First determine if it is required or is profitable to flip the operands.
2871
2872 // If LHS is a foldable load, but RHS is not, flip the condition.
2873 if (ISD::isNON_EXTLoad(LHS.getNode()) &&
2874 !ISD::isNON_EXTLoad(RHS.getNode())) {
2875 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
2876 std::swap(LHS, RHS);
2877 }
2878
2879 switch (SetCCOpcode) {
2880 default: break;
2881 case ISD::SETOLT:
2882 case ISD::SETOLE:
2883 case ISD::SETUGT:
2884 case ISD::SETUGE:
2885 std::swap(LHS, RHS);
2886 break;
2887 }
2888
2889 // On a floating point condition, the flags are set as follows:
2890 // ZF PF CF op
2891 // 0 | 0 | 0 | X > Y
2892 // 0 | 0 | 1 | X < Y
2893 // 1 | 0 | 0 | X == Y
2894 // 1 | 1 | 1 | unordered
2895 switch (SetCCOpcode) {
2896 // clang-format off
2897 default: llvm_unreachable("Condcode should be pre-legalized away");
2898 case ISD::SETUEQ:
2899 case ISD::SETEQ: return X86::COND_E;
2900 case ISD::SETOLT: // flipped
2901 case ISD::SETOGT:
2902 case ISD::SETGT: return X86::COND_A;
2903 case ISD::SETOLE: // flipped
2904 case ISD::SETOGE:
2905 case ISD::SETGE: return X86::COND_AE;
2906 case ISD::SETUGT: // flipped
2907 case ISD::SETULT:
2908 case ISD::SETLT: return X86::COND_B;
2909 case ISD::SETUGE: // flipped
2910 case ISD::SETULE:
2911 case ISD::SETLE: return X86::COND_BE;
2912 case ISD::SETONE:
2913 case ISD::SETNE: return X86::COND_NE;
2914 case ISD::SETUO: return X86::COND_P;
2915 case ISD::SETO: return X86::COND_NP;
2916 case ISD::SETOEQ:
2917 case ISD::SETUNE: return X86::COND_INVALID;
2918 // clang-format on
2919 }
2920}
2921
2922/// Is there a floating point cmov for the specific X86 condition code?
2923/// Current x86 isa includes the following FP cmov instructions:
2924/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
2925static bool hasFPCMov(unsigned X86CC) {
2926 switch (X86CC) {
2927 default:
2928 return false;
2929 case X86::COND_B:
2930 case X86::COND_BE:
2931 case X86::COND_E:
2932 case X86::COND_P:
2933 case X86::COND_A:
2934 case X86::COND_AE:
2935 case X86::COND_NE:
2936 case X86::COND_NP:
2937 return true;
2938 }
2939}
2940
2941static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
2942 return Subtarget.hasVLX() || Subtarget.canExtendTo512DQ() ||
2943 VT.is512BitVector();
2944}
2945
2946bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
2947 const CallInst &I,
2948 MachineFunction &MF,
2949 unsigned Intrinsic) const {
2950 Info.flags = MachineMemOperand::MONone;
2951 Info.offset = 0;
2952
2953 const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic);
2954 if (!IntrData) {
2955 switch (Intrinsic) {
2956 case Intrinsic::x86_aesenc128kl:
2957 case Intrinsic::x86_aesdec128kl:
2958 Info.opc = ISD::INTRINSIC_W_CHAIN;
2959 Info.ptrVal = I.getArgOperand(1);
2960 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2961 Info.align = Align(1);
2962 Info.flags |= MachineMemOperand::MOLoad;
2963 return true;
2964 case Intrinsic::x86_aesenc256kl:
2965 case Intrinsic::x86_aesdec256kl:
2966 Info.opc = ISD::INTRINSIC_W_CHAIN;
2967 Info.ptrVal = I.getArgOperand(1);
2968 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2969 Info.align = Align(1);
2970 Info.flags |= MachineMemOperand::MOLoad;
2971 return true;
2972 case Intrinsic::x86_aesencwide128kl:
2973 case Intrinsic::x86_aesdecwide128kl:
2974 Info.opc = ISD::INTRINSIC_W_CHAIN;
2975 Info.ptrVal = I.getArgOperand(0);
2976 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2977 Info.align = Align(1);
2978 Info.flags |= MachineMemOperand::MOLoad;
2979 return true;
2980 case Intrinsic::x86_aesencwide256kl:
2981 case Intrinsic::x86_aesdecwide256kl:
2982 Info.opc = ISD::INTRINSIC_W_CHAIN;
2983 Info.ptrVal = I.getArgOperand(0);
2984 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2985 Info.align = Align(1);
2986 Info.flags |= MachineMemOperand::MOLoad;
2987 return true;
2988 case Intrinsic::x86_cmpccxadd32:
2989 case Intrinsic::x86_cmpccxadd64:
2990 case Intrinsic::x86_atomic_bts:
2991 case Intrinsic::x86_atomic_btc:
2992 case Intrinsic::x86_atomic_btr: {
2993 Info.opc = ISD::INTRINSIC_W_CHAIN;
2994 Info.ptrVal = I.getArgOperand(0);
2995 unsigned Size = I.getType()->getScalarSizeInBits();
2996 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2997 Info.align = Align(Size);
2998 Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2999 MachineMemOperand::MOVolatile;
3000 return true;
3001 }
3002 case Intrinsic::x86_atomic_bts_rm:
3003 case Intrinsic::x86_atomic_btc_rm:
3004 case Intrinsic::x86_atomic_btr_rm: {
3005 Info.opc = ISD::INTRINSIC_W_CHAIN;
3006 Info.ptrVal = I.getArgOperand(0);
3007 unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
3008 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
3009 Info.align = Align(Size);
3010 Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
3011 MachineMemOperand::MOVolatile;
3012 return true;
3013 }
3014 case Intrinsic::x86_aadd32:
3015 case Intrinsic::x86_aadd64:
3016 case Intrinsic::x86_aand32:
3017 case Intrinsic::x86_aand64:
3018 case Intrinsic::x86_aor32:
3019 case Intrinsic::x86_aor64:
3020 case Intrinsic::x86_axor32:
3021 case Intrinsic::x86_axor64:
3022 case Intrinsic::x86_atomic_add_cc:
3023 case Intrinsic::x86_atomic_sub_cc:
3024 case Intrinsic::x86_atomic_or_cc:
3025 case Intrinsic::x86_atomic_and_cc:
3026 case Intrinsic::x86_atomic_xor_cc: {
3027 Info.opc = ISD::INTRINSIC_W_CHAIN;
3028 Info.ptrVal = I.getArgOperand(0);
3029 unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
3030 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
3031 Info.align = Align(Size);
3032 Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
3033 MachineMemOperand::MOVolatile;
3034 return true;
3035 }
3036 }
3037 return false;
3038 }
3039
3040 switch (IntrData->Type) {
3041 case TRUNCATE_TO_MEM_VI8:
3042 case TRUNCATE_TO_MEM_VI16:
3043 case TRUNCATE_TO_MEM_VI32: {
3044 Info.opc = ISD::INTRINSIC_VOID;
3045 Info.ptrVal = I.getArgOperand(0);
3046 MVT VT = MVT::getVT(I.getArgOperand(1)->getType());
3047 MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
3048 if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
3049 ScalarVT = MVT::i8;
3050 else if (IntrData->Type == TRUNCATE_TO_MEM_VI16)
3051 ScalarVT = MVT::i16;
3052 else if (IntrData->Type == TRUNCATE_TO_MEM_VI32)
3053 ScalarVT = MVT::i32;
3054
3055 Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements());
3056 Info.align = Align(1);
3057 Info.flags |= MachineMemOperand::MOStore;
3058 break;
3059 }
3060 case GATHER:
3061 case GATHER_AVX2: {
3062 Info.opc = ISD::INTRINSIC_W_CHAIN;
3063 Info.ptrVal = nullptr;
3064 MVT DataVT = MVT::getVT(I.getType());
3065 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
3066 unsigned NumElts = std::min(DataVT.getVectorNumElements(),
3067 IndexVT.getVectorNumElements());
3068 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
3069 Info.align = Align(1);
3070 Info.flags |= MachineMemOperand::MOLoad;
3071 break;
3072 }
3073 case SCATTER: {
3074 Info.opc = ISD::INTRINSIC_VOID;
3075 Info.ptrVal = nullptr;
3076 MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
3077 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
3078 unsigned NumElts = std::min(DataVT.getVectorNumElements(),
3079 IndexVT.getVectorNumElements());
3080 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
3081 Info.align = Align(1);
3082 Info.flags |= MachineMemOperand::MOStore;
3083 break;
3084 }
3085 default:
3086 return false;
3087 }
3088
3089 return true;
3090}
3091
3092/// Returns true if the target can instruction select the
3093/// specified FP immediate natively. If false, the legalizer will
3094/// materialize the FP immediate as a load from a constant pool.
3095bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3096 bool ForCodeSize) const {
3097 for (const APFloat &FPImm : LegalFPImmediates)
3098 if (Imm.bitwiseIsEqual(FPImm))
3099 return true;
3100 return false;
3101}
3102
3103bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load,
3104 ISD::LoadExtType ExtTy,
3105 EVT NewVT) const {
3106 assert(cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow");
3107
3108 // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF
3109 // relocation target a movq or addq instruction: don't let the load shrink.
3110 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
3111 if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
3112 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0)))
3113 return GA->getTargetFlags() != X86II::MO_GOTTPOFF;
3114
3115 // If this is an (1) AVX vector load with (2) multiple uses and (3) all of
3116 // those uses are extracted directly into a store, then the extract + store
3117 // can be store-folded. Therefore, it's probably not worth splitting the load.
3118 EVT VT = Load->getValueType(0);
3119 if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) {
3120 for (auto UI = Load->use_begin(), UE = Load->use_end(); UI != UE; ++UI) {
3121 // Skip uses of the chain value. Result 0 of the node is the load value.
3122 if (UI.getUse().getResNo() != 0)
3123 continue;
3124
3125 // If this use is not an extract + store, it's probably worth splitting.
3126 if (UI->getOpcode() != ISD::EXTRACT_SUBVECTOR || !UI->hasOneUse() ||
3127 UI->use_begin()->getOpcode() != ISD::STORE)
3128 return true;
3129 }
3130 // All non-chain uses are extract + store.
3131 return false;
3132 }
3133
3134 return true;
3135}
3136
3137/// Returns true if it is beneficial to convert a load of a constant
3138/// to just the constant itself.
3139bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3140 Type *Ty) const {
3141 assert(Ty->isIntegerTy());
3142
3143 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3144 if (BitSize == 0 || BitSize > 64)
3145 return false;
3146 return true;
3147}
3148
3149bool X86TargetLowering::reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
3150 // If we are using XMM registers in the ABI and the condition of the select is
3151 // a floating-point compare and we have blendv or conditional move, then it is
3152 // cheaper to select instead of doing a cross-register move and creating a
3153 // load that depends on the compare result.
3154 bool IsFPSetCC = CmpOpVT.isFloatingPoint() && CmpOpVT != MVT::f128;
3155 return !IsFPSetCC || !Subtarget.isTarget64BitLP64() || !Subtarget.hasAVX();
3156}
3157
3158bool X86TargetLowering::convertSelectOfConstantsToMath(EVT VT) const {
3159 // TODO: It might be a win to ease or lift this restriction, but the generic
3160 // folds in DAGCombiner conflict with vector folds for an AVX512 target.
3161 if (VT.isVector() && Subtarget.hasAVX512())
3162 return false;
3163
3164 return true;
3165}
3166
3167bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
3168 SDValue C) const {
3169 // TODO: We handle scalars using custom code, but generic combining could make
3170 // that unnecessary.
3171 APInt MulC;
3172 if (!ISD::isConstantSplatVector(C.getNode(), MulC))
3173 return false;
3174
3175 // Find the type this will be legalized too. Otherwise we might prematurely
3176 // convert this to shl+add/sub and then still have to type legalize those ops.
3177 // Another choice would be to defer the decision for illegal types until
3178 // after type legalization. But constant splat vectors of i64 can't make it
3179 // through type legalization on 32-bit targets so we would need to special
3180 // case vXi64.
3181 while (getTypeAction(Context, VT) != TypeLegal)
3182 VT = getTypeToTransformTo(Context, VT);
3183
3184 // If vector multiply is legal, assume that's faster than shl + add/sub.
3185 // Multiply is a complex op with higher latency and lower throughput in
3186 // most implementations, sub-vXi32 vector multiplies are always fast,
3187 // vXi32 mustn't have a SlowMULLD implementation, and anything larger (vXi64)
3188 // is always going to be slow.
3189 unsigned EltSizeInBits = VT.getScalarSizeInBits();
3190 if (isOperationLegal(ISD::MUL, VT) && EltSizeInBits <= 32 &&
3191 (EltSizeInBits != 32 || !Subtarget.isPMULLDSlow()))
3192 return false;
3193
3194 // shl+add, shl+sub, shl+add+neg
3195 return (MulC + 1).isPowerOf2() || (MulC - 1).isPowerOf2() ||
3196 (1 - MulC).isPowerOf2() || (-(MulC + 1)).isPowerOf2();
3197}
3198
3199bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
3200 unsigned Index) const {
3201 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
3202 return false;
3203
3204 // Mask vectors support all subregister combinations and operations that
3205 // extract half of vector.
3206 if (ResVT.getVectorElementType() == MVT::i1)
3207 return Index == 0 || ((ResVT.getSizeInBits() == SrcVT.getSizeInBits()*2) &&
3208 (Index == ResVT.getVectorNumElements()));
3209
3210 return (Index % ResVT.getVectorNumElements()) == 0;
3211}
3212
3213bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
3214 unsigned Opc = VecOp.getOpcode();
3215
3216 // Assume target opcodes can't be scalarized.
3217 // TODO - do we have any exceptions?
3218 if (Opc >= ISD::BUILTIN_OP_END)
3219 return false;
3220
3221 // If the vector op is not supported, try to convert to scalar.
3222 EVT VecVT = VecOp.getValueType();
3223 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
3224 return true;
3225
3226 // If the vector op is supported, but the scalar op is not, the transform may
3227 // not be worthwhile.
3228 EVT ScalarVT = VecVT.getScalarType();
3229 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
3230}
3231
3232bool X86TargetLowering::shouldFormOverflowOp(unsigned Opcode, EVT VT,
3233 bool) const {
3234 // TODO: Allow vectors?
3235 if (VT.isVector())
3236 return false;
3237 return VT.isSimple() || !isOperationExpand(Opcode, VT);
3238}
3239
3240bool X86TargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
3241 // Speculate cttz only if we can directly use TZCNT or can promote to i32/i64.
3242 return Subtarget.hasBMI() ||
3243 (!Ty->isVectorTy() &&
3244 Ty->getScalarSizeInBits() < (Subtarget.is64Bit() ? 64u : 32u));
3245}
3246
3247bool X86TargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
3248 // Speculate ctlz only if we can directly use LZCNT.
3249 return Subtarget.hasLZCNT();
3250}
3251
3252bool X86TargetLowering::ShouldShrinkFPConstant(EVT VT) const {
3253 // Don't shrink FP constpool if SSE2 is available since cvtss2sd is more
3254 // expensive than a straight movsd. On the other hand, it's important to
3255 // shrink long double fp constant since fldt is very slow.
3256 return !Subtarget.hasSSE2() || VT == MVT::f80;
3257}
3258
3259bool X86TargetLowering::isScalarFPTypeInSSEReg(EVT VT) const {
3260 return (VT == MVT::f64 && Subtarget.hasSSE2()) ||
3261 (VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16;
3262}
3263
3264bool X86TargetLowering::isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
3265 const SelectionDAG &DAG,
3266 const MachineMemOperand &MMO) const {
3267 if (!Subtarget.hasAVX512() && !LoadVT.isVector() && BitcastVT.isVector() &&
3268 BitcastVT.getVectorElementType() == MVT::i1)
3269 return false;
3270
3271 if (!Subtarget.hasDQI() && BitcastVT == MVT::v8i1 && LoadVT == MVT::i8)
3272 return false;
3273
3274 // If both types are legal vectors, it's always ok to convert them.
3275 if (LoadVT.isVector() && BitcastVT.isVector() &&
3276 isTypeLegal(LoadVT) && isTypeLegal(BitcastVT))
3277 return true;
3278
3279 return TargetLowering::isLoadBitCastBeneficial(LoadVT, BitcastVT, DAG, MMO);
3280}
3281
3282bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
3283 const MachineFunction &MF) const {
3284 // Do not merge to float value size (128 bytes) if no implicit
3285 // float attribute is set.
3286 bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat);
3287
3288 if (NoFloat) {
3289 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
3290 return (MemVT.getSizeInBits() <= MaxIntSize);
3291 }
3292 // Make sure we don't merge greater than our preferred vector
3293 // width.
3294 if (MemVT.getSizeInBits() > Subtarget.getPreferVectorWidth())
3295 return false;
3296
3297 return true;
3298}
3299
3300bool X86TargetLowering::isCtlzFast() const {
3301 return Subtarget.hasFastLZCNT();
3302}
3303
3304bool X86TargetLowering::isMaskAndCmp0FoldingBeneficial(
3305 const Instruction &AndI) const {
3306 return true;
3307}
3308
3309bool X86TargetLowering::hasAndNotCompare(SDValue Y) const {
3310 EVT VT = Y.getValueType();
3311
3312 if (VT.isVector())
3313 return false;
3314
3315 if (!Subtarget.hasBMI())
3316 return false;
3317
3318 // There are only 32-bit and 64-bit forms for 'andn'.
3319 if (VT != MVT::i32 && VT != MVT::i64)
3320 return false;
3321
3322 return !isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque();
3323}
3324
3325bool X86TargetLowering::hasAndNot(SDValue Y) const {
3326 EVT VT = Y.getValueType();
3327
3328 if (!VT.isVector())
3329 return hasAndNotCompare(Y);
3330
3331 // Vector.
3332
3333 if (!Subtarget.hasSSE1() || VT.getSizeInBits() < 128)
3334 return false;
3335
3336 if (VT == MVT::v4i32)
3337 return true;
3338
3339 return Subtarget.hasSSE2();
3340}
3341
3342bool X86TargetLowering::hasBitTest(SDValue X, SDValue Y) const {
3343 return X.getValueType().isScalarInteger(); // 'bt'
3344}
3345
3346bool X86TargetLowering::
3347 shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3348 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
3349 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
3350 SelectionDAG &DAG) const {
3351 // Does baseline recommend not to perform the fold by default?
3352 if (!TargetLowering::shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3353 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG))
3354 return false;
3355 // For scalars this transform is always beneficial.
3356 if (X.getValueType().isScalarInteger())
3357 return true;
3358 // If all the shift amounts are identical, then transform is beneficial even
3359 // with rudimentary SSE2 shifts.
3360 if (DAG.isSplatValue(Y, /*AllowUndefs=*/true))
3361 return true;
3362 // If we have AVX2 with it's powerful shift operations, then it's also good.
3363 if (Subtarget.hasAVX2())
3364 return true;
3365 // Pre-AVX2 vector codegen for this pattern is best for variant with 'shl'.
3366 return NewShiftOpcode == ISD::SHL;
3367}
3368
3369unsigned X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand(
3370 EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
3371 const APInt &ShiftOrRotateAmt, const std::optional<APInt> &AndMask) const {
3372 if (!VT.isInteger())
3373 return ShiftOpc;
3374
3375 bool PreferRotate = false;
3376 if (VT.isVector()) {
3377 // For vectors, if we have rotate instruction support, then its definetly
3378 // best. Otherwise its not clear what the best so just don't make changed.
3379 PreferRotate = Subtarget.hasAVX512() && (VT.getScalarType() == MVT::i32 ||
3380 VT.getScalarType() == MVT::i64);
3381 } else {
3382 // For scalar, if we have bmi prefer rotate for rorx. Otherwise prefer
3383 // rotate unless we have a zext mask+shr.
3384 PreferRotate = Subtarget.hasBMI2();
3385 if (!PreferRotate) {
3386 unsigned MaskBits =
3387 VT.getScalarSizeInBits() - ShiftOrRotateAmt.getZExtValue();
3388 PreferRotate = (MaskBits != 8) && (MaskBits != 16) && (MaskBits != 32);
3389 }
3390 }
3391
3392 if (ShiftOpc == ISD::SHL || ShiftOpc == ISD::SRL) {
3393 assert(AndMask.has_value() && "Null andmask when querying about shift+and");
3394
3395 if (PreferRotate && MayTransformRotate)
3396 return ISD::ROTL;
3397
3398 // If vector we don't really get much benefit swapping around constants.
3399 // Maybe we could check if the DAG has the flipped node already in the
3400 // future.
3401 if (VT.isVector())
3402 return ShiftOpc;
3403
3404 // See if the beneficial to swap shift type.
3405 if (ShiftOpc == ISD::SHL) {
3406 // If the current setup has imm64 mask, then inverse will have
3407 // at least imm32 mask (or be zext i32 -> i64).
3408 if (VT == MVT::i64)
3409 return AndMask->getSignificantBits() > 32 ? (unsigned)ISD::SRL
3410 : ShiftOpc;
3411
3412 // We can only benefit if req at least 7-bit for the mask. We
3413 // don't want to replace shl of 1,2,3 as they can be implemented
3414 // with lea/add.
3415 return ShiftOrRotateAmt.uge(7) ? (unsigned)ISD::SRL : ShiftOpc;
3416 }
3417
3418 if (VT == MVT::i64)
3419 // Keep exactly 32-bit imm64, this is zext i32 -> i64 which is
3420 // extremely efficient.
3421 return AndMask->getSignificantBits() > 33 ? (unsigned)ISD::SHL : ShiftOpc;
3422
3423 // Keep small shifts as shl so we can generate add/lea.
3424 return ShiftOrRotateAmt.ult(7) ? (unsigned)ISD::SHL : ShiftOpc;
3425 }
3426
3427 // We prefer rotate for vectors of if we won't get a zext mask with SRL
3428 // (PreferRotate will be set in the latter case).
3429 if (PreferRotate || VT.isVector())
3430 return ShiftOpc;
3431
3432 // Non-vector type and we have a zext mask with SRL.
3433 return ISD::SRL;
3434}
3435
3436TargetLoweringBase::CondMergingParams
3437X86TargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
3438 const Value *Lhs,
3439 const Value *Rhs) const {
3440 using namespace llvm::PatternMatch;
3441 int BaseCost = BrMergingBaseCostThresh.getValue();
3442 // With CCMP, branches can be merged in a more efficient way.
3443 if (BaseCost >= 0 && Subtarget.hasCCMP())
3444 BaseCost += BrMergingCcmpBias;
3445 // a == b && a == c is a fast pattern on x86.
3446 if (BaseCost >= 0 && Opc == Instruction::And &&
3447 match(Lhs, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Value(), m_Value())) &&
3448 match(Rhs, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Value(), m_Value())))
3449 BaseCost += 1;
3450 return {BaseCost, BrMergingLikelyBias.getValue(),
3451 BrMergingUnlikelyBias.getValue()};
3452}
3453
3454bool X86TargetLowering::preferScalarizeSplat(SDNode *N) const {
3455 return N->getOpcode() != ISD::FP_EXTEND;
3456}
3457
3458bool X86TargetLowering::shouldFoldConstantShiftPairToMask(
3459 const SDNode *N, CombineLevel Level) const {
3460 assert(((N->getOpcode() == ISD::SHL &&
3461 N->getOperand(0).getOpcode() == ISD::SRL) ||
3462 (N->getOpcode() == ISD::SRL &&
3463 N->getOperand(0).getOpcode() == ISD::SHL)) &&
3464 "Expected shift-shift mask");
3465 // TODO: Should we always create i64 masks? Or only folded immediates?
3466 EVT VT = N->getValueType(0);
3467 if ((Subtarget.hasFastVectorShiftMasks() && VT.isVector()) ||
3468 (Subtarget.hasFastScalarShiftMasks() && !VT.isVector())) {
3469 // Only fold if the shift values are equal - so it folds to AND.
3470 // TODO - we should fold if either is a non-uniform vector but we don't do
3471 // the fold for non-splats yet.
3472 return N->getOperand(1) == N->getOperand(0).getOperand(1);
3473 }
3474 return TargetLoweringBase::shouldFoldConstantShiftPairToMask(N, Level);
3475}
3476
3477bool X86TargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
3478 EVT VT = Y.getValueType();
3479
3480 // For vectors, we don't have a preference, but we probably want a mask.
3481 if (VT.isVector())
3482 return false;
3483
3484 // 64-bit shifts on 32-bit targets produce really bad bloated code.
3485 if (VT == MVT::i64 && !Subtarget.is64Bit())
3486 return false;
3487
3488 return true;
3489}
3490
3491TargetLowering::ShiftLegalizationStrategy
3492X86TargetLowering::preferredShiftLegalizationStrategy(
3493 SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const {
3494 if (DAG.getMachineFunction().getFunction().hasMinSize() &&
3495 !Subtarget.isOSWindows())
3496 return ShiftLegalizationStrategy::LowerToLibcall;
3497 return TargetLowering::preferredShiftLegalizationStrategy(DAG, N,
3498 ExpansionFactor);
3499}
3500
3501bool X86TargetLowering::shouldSplatInsEltVarIndex(EVT VT) const {
3502 // Any legal vector type can be splatted more efficiently than
3503 // loading/spilling from memory.
3504 return isTypeLegal(VT);
3505}
3506
3507MVT X86TargetLowering::hasFastEqualityCompare(unsigned NumBits) const {
3508 MVT VT = MVT::getIntegerVT(NumBits);
3509 if (isTypeLegal(VT))
3510 return VT;
3511
3512 // PMOVMSKB can handle this.
3513 if (NumBits == 128 && isTypeLegal(MVT::v16i8))
3514 return MVT::v16i8;
3515
3516 // VPMOVMSKB can handle this.
3517 if (NumBits == 256 && isTypeLegal(MVT::v32i8))
3518 return MVT::v32i8;
3519
3520 // TODO: Allow 64-bit type for 32-bit target.
3521 // TODO: 512-bit types should be allowed, but make sure that those
3522 // cases are handled in combineVectorSizedSetCCEquality().
3523
3524 return MVT::INVALID_SIMPLE_VALUE_TYPE;
3525}
3526
3527/// Val is the undef sentinel value or equal to the specified value.
3528static bool isUndefOrEqual(int Val, int CmpVal) {
3529 return ((Val == SM_SentinelUndef) || (Val == CmpVal));
3530}
3531
3532/// Return true if every element in Mask is the undef sentinel value or equal to
3533/// the specified value.
3534static bool isUndefOrEqual(ArrayRef<int> Mask, int CmpVal) {
3535 return llvm::all_of(Mask, [CmpVal](int M) {
3536 return (M == SM_SentinelUndef) || (M == CmpVal);
3537 });
3538}
3539
3540/// Return true if every element in Mask, beginning from position Pos and ending
3541/// in Pos+Size is the undef sentinel value or equal to the specified value.
3542static bool isUndefOrEqualInRange(ArrayRef<int> Mask, int CmpVal, unsigned Pos,
3543 unsigned Size) {
3544 return llvm::all_of(Mask.slice(Pos, Size),
3545 [CmpVal](int M) { return isUndefOrEqual(M, CmpVal); });
3546}
3547
3548/// Val is either the undef or zero sentinel value.
3549static bool isUndefOrZero(int Val) {
3550 return ((Val == SM_SentinelUndef) || (Val == SM_SentinelZero));
3551}
3552
3553/// Return true if every element in Mask, beginning from position Pos and ending
3554/// in Pos+Size is the undef sentinel value.
3555static bool isUndefInRange(ArrayRef<int> Mask, unsigned Pos, unsigned Size) {
3556 return llvm::all_of(Mask.slice(Pos, Size),
3557 [](int M) { return M == SM_SentinelUndef; });
3558}
3559
3560/// Return true if the mask creates a vector whose lower half is undefined.
3561static bool isUndefLowerHalf(ArrayRef<int> Mask) {
3562 unsigned NumElts = Mask.size();
3563 return isUndefInRange(Mask, 0, NumElts / 2);
3564}
3565
3566/// Return true if the mask creates a vector whose upper half is undefined.
3567static bool isUndefUpperHalf(ArrayRef<int> Mask) {
3568 unsigned NumElts = Mask.size();
3569 return isUndefInRange(Mask, NumElts / 2, NumElts / 2);
3570}
3571
3572/// Return true if Val falls within the specified range (L, H].
3573static bool isInRange(int Val, int Low, int Hi) {
3574 return (Val >= Low && Val < Hi);
3575}
3576
3577/// Return true if the value of any element in Mask falls within the specified
3578/// range (L, H].
3579static bool isAnyInRange(ArrayRef<int> Mask, int Low, int Hi) {
3580 return llvm::any_of(Mask, [Low, Hi](int M) { return isInRange(M, Low, Hi); });
3581}
3582
3583/// Return true if the value of any element in Mask is the zero sentinel value.
3584static bool isAnyZero(ArrayRef<int> Mask) {
3585 return llvm::any_of(Mask, [](int M) { return M == SM_SentinelZero; });
3586}
3587
3588/// Return true if Val is undef or if its value falls within the
3589/// specified range (L, H].
3590static bool isUndefOrInRange(int Val, int Low, int Hi) {
3591 return (Val == SM_SentinelUndef) || isInRange(Val, Low, Hi);
3592}
3593
3594/// Return true if every element in Mask is undef or if its value
3595/// falls within the specified range (L, H].
3596static bool isUndefOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
3597 return llvm::all_of(
3598 Mask, [Low, Hi](int M) { return isUndefOrInRange(M, Low, Hi); });
3599}
3600
3601/// Return true if Val is undef, zero or if its value falls within the
3602/// specified range (L, H].
3603static bool isUndefOrZeroOrInRange(int Val, int Low, int Hi) {
3604 return isUndefOrZero(Val) || isInRange(Val, Low, Hi);
3605}
3606
3607/// Return true if every element in Mask is undef, zero or if its value
3608/// falls within the specified range (L, H].
3609static bool isUndefOrZeroOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
3610 return llvm::all_of(
3611 Mask, [Low, Hi](int M) { return isUndefOrZeroOrInRange(M, Low, Hi); });
3612}
3613
3614/// Return true if every element in Mask, is an in-place blend/select mask or is
3615/// undef.
3616LLVM_ATTRIBUTE_UNUSED static bool isBlendOrUndef(ArrayRef<int> Mask) {
3617 unsigned NumElts = Mask.size();
3618 for (auto [I, M] : enumerate(Mask))
3619 if (!isUndefOrEqual(M, I) && !isUndefOrEqual(M, I + NumElts))
3620 return false;
3621 return true;
3622}
3623
3624/// Return true if every element in Mask, beginning
3625/// from position Pos and ending in Pos + Size, falls within the specified
3626/// sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
3627static bool isSequentialOrUndefInRange(ArrayRef<int> Mask, unsigned Pos,
3628 unsigned Size, int Low, int Step = 1) {
3629 for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
3630 if (!isUndefOrEqual(Mask[i], Low))
3631 return false;
3632 return true;
3633}
3634
3635/// Return true if every element in Mask, beginning
3636/// from position Pos and ending in Pos+Size, falls within the specified
3637/// sequential range (Low, Low+Size], or is undef or is zero.
3638static bool isSequentialOrUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
3639 unsigned Size, int Low,
3640 int Step = 1) {
3641 for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
3642 if (!isUndefOrZero(Mask[i]) && Mask[i] != Low)
3643 return false;
3644 return true;
3645}
3646
3647/// Return true if every element in Mask, beginning
3648/// from position Pos and ending in Pos+Size is undef or is zero.
3649static bool isUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
3650 unsigned Size) {
3651 return llvm::all_of(Mask.slice(Pos, Size), isUndefOrZero);
3652}
3653
3654/// Return true if every element of a single input is referenced by the shuffle
3655/// mask. i.e. it just permutes them all.
3656static bool isCompletePermute(ArrayRef<int> Mask) {
3657 unsigned NumElts = Mask.size();
3658 APInt DemandedElts = APInt::getZero(NumElts);
3659 for (int M : Mask)
3660 if (isInRange(M, 0, NumElts))
3661 DemandedElts.setBit(M);
3662 return DemandedElts.isAllOnes();
3663}
3664
3665/// Helper function to test whether a shuffle mask could be
3666/// simplified by widening the elements being shuffled.
3667///
3668/// Appends the mask for wider elements in WidenedMask if valid. Otherwise
3669/// leaves it in an unspecified state.
3670///
3671/// NOTE: This must handle normal vector shuffle masks and *target* vector
3672/// shuffle masks. The latter have the special property of a '-2' representing
3673/// a zero-ed lane of a vector.
3674static bool canWidenShuffleElements(ArrayRef<int> Mask,
3675 SmallVectorImpl<int> &WidenedMask) {
3676 WidenedMask.assign(Mask.size() / 2, 0);
3677 for (int i = 0, Size = Mask.size(); i < Size; i += 2) {
3678 int M0 = Mask[i];
3679 int M1 = Mask[i + 1];
3680
3681 // If both elements are undef, its trivial.
3682 if (M0 == SM_SentinelUndef && M1 == SM_SentinelUndef) {
3683 WidenedMask[i / 2] = SM_SentinelUndef;
3684 continue;
3685 }
3686
3687 // Check for an undef mask and a mask value properly aligned to fit with
3688 // a pair of values. If we find such a case, use the non-undef mask's value.
3689 if (M0 == SM_SentinelUndef && M1 >= 0 && (M1 % 2) == 1) {
3690 WidenedMask[i / 2] = M1 / 2;
3691 continue;
3692 }
3693 if (M1 == SM_SentinelUndef && M0 >= 0 && (M0 % 2) == 0) {
3694 WidenedMask[i / 2] = M0 / 2;
3695 continue;
3696 }
3697
3698 // When zeroing, we need to spread the zeroing across both lanes to widen.
3699 if (M0 == SM_SentinelZero || M1 == SM_SentinelZero) {
3700 if ((M0 == SM_SentinelZero || M0 == SM_SentinelUndef) &&
3701 (M1 == SM_SentinelZero || M1 == SM_SentinelUndef)) {
3702 WidenedMask[i / 2] = SM_SentinelZero;
3703 continue;
3704 }
3705 return false;
3706 }
3707
3708 // Finally check if the two mask values are adjacent and aligned with
3709 // a pair.
3710 if (M0 != SM_SentinelUndef && (M0 % 2) == 0 && (M0 + 1) == M1) {
3711 WidenedMask[i / 2] = M0 / 2;
3712 continue;
3713 }
3714
3715 // Otherwise we can't safely widen the elements used in this shuffle.
3716 return false;
3717 }
3718 assert(WidenedMask.size() == Mask.size() / 2 &&
3719 "Incorrect size of mask after widening the elements!");
3720
3721 return true;
3722}
3723
3724static bool canWidenShuffleElements(ArrayRef<int> Mask,
3725 const APInt &Zeroable,
3726 bool V2IsZero,
3727 SmallVectorImpl<int> &WidenedMask) {
3728 // Create an alternative mask with info about zeroable elements.
3729 // Here we do not set undef elements as zeroable.
3730 SmallVector<int, 64> ZeroableMask(Mask);
3731 if (V2IsZero) {
3732 assert(!Zeroable.isZero() && "V2's non-undef elements are used?!");
3733 for (int i = 0, Size = Mask.size(); i != Size; ++i)
3734 if (Mask[i] != SM_SentinelUndef && Zeroable[i])
3735 ZeroableMask[i] = SM_SentinelZero;
3736 }
3737 return canWidenShuffleElements(ZeroableMask, WidenedMask);
3738}
3739
3740static bool canWidenShuffleElements(ArrayRef<int> Mask) {
3741 SmallVector<int, 32> WidenedMask;
3742 return canWidenShuffleElements(Mask, WidenedMask);
3743}
3744
3745// Attempt to narrow/widen shuffle mask until it matches the target number of
3746// elements.
3747static bool scaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts,
3748 SmallVectorImpl<int> &ScaledMask) {
3749 unsigned NumSrcElts = Mask.size();
3750 assert(((NumSrcElts % NumDstElts) == 0 || (NumDstElts % NumSrcElts) == 0) &&
3751 "Illegal shuffle scale factor");
3752
3753 // Narrowing is guaranteed to work.
3754 if (NumDstElts >= NumSrcElts) {
3755 int Scale = NumDstElts / NumSrcElts;
3756 llvm::narrowShuffleMaskElts(Scale, Mask, ScaledMask);
3757 return true;
3758 }
3759
3760 // We have to repeat the widening until we reach the target size, but we can
3761 // split out the first widening as it sets up ScaledMask for us.
3762 if (canWidenShuffleElements(Mask, ScaledMask)) {
3763 while (ScaledMask.size() > NumDstElts) {
3764 SmallVector<int, 16> WidenedMask;
3765 if (!canWidenShuffleElements(ScaledMask, WidenedMask))
3766 return false;
3767 ScaledMask = std::move(WidenedMask);
3768 }
3769 return true;
3770 }
3771
3772 return false;
3773}
3774
3775static bool canScaleShuffleElements(ArrayRef<int> Mask, unsigned NumDstElts) {
3776 SmallVector<int, 32> ScaledMask;
3777 return scaleShuffleElements(Mask, NumDstElts, ScaledMask);
3778}
3779
3780/// Returns true if Elt is a constant zero or a floating point constant +0.0.
3781bool X86::isZeroNode(SDValue Elt) {
3782 return isNullConstant(Elt) || isNullFPConstant(Elt);
3783}
3784
3785// Build a vector of constants.
3786// Use an UNDEF node if MaskElt == -1.
3787// Split 64-bit constants in the 32-bit mode.
3788static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
3789 const SDLoc &dl, bool IsMask = false) {
3790
3791 SmallVector<SDValue, 32> Ops;
3792 bool Split = false;
3793
3794 MVT ConstVecVT = VT;
3795 unsigned NumElts = VT.getVectorNumElements();
3796 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
3797 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
3798 ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
3799 Split = true;
3800 }
3801
3802 MVT EltVT = ConstVecVT.getVectorElementType();
3803 for (unsigned i = 0; i < NumElts; ++i) {
3804 bool IsUndef = Values[i] < 0 && IsMask;
3805 SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
3806 DAG.getConstant(Values[i], dl, EltVT);
3807 Ops.push_back(OpNode);
3808 if (Split)
3809 Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT) :
3810 DAG.getConstant(0, dl, EltVT));
3811 }
3812 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
3813 if (Split)
3814 ConstsNode = DAG.getBitcast(VT, ConstsNode);
3815 return ConstsNode;
3816}
3817
3818static SDValue getConstVector(ArrayRef<APInt> Bits, const APInt &Undefs,
3819 MVT VT, SelectionDAG &DAG, const SDLoc &dl) {
3820 assert(Bits.size() == Undefs.getBitWidth() &&
3821 "Unequal constant and undef arrays");
3822 SmallVector<SDValue, 32> Ops;
3823 bool Split = false;
3824
3825 MVT ConstVecVT = VT;
3826 unsigned NumElts = VT.getVectorNumElements();
3827 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
3828 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
3829 ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
3830 Split = true;
3831 }
3832
3833 MVT EltVT = ConstVecVT.getVectorElementType();
3834 for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
3835 if (Undefs[i]) {
3836 Ops.append(Split ? 2 : 1, DAG.getUNDEF(EltVT));
3837 continue;
3838 }
3839 const APInt &V = Bits[i];
3840 assert(V.getBitWidth() == VT.getScalarSizeInBits() && "Unexpected sizes");
3841 if (Split) {
3842 Ops.push_back(DAG.getConstant(V.trunc(32), dl, EltVT));
3843 Ops.push_back(DAG.getConstant(V.lshr(32).trunc(32), dl, EltVT));
3844 } else if (EltVT == MVT::f32) {
3845 APFloat FV(APFloat::IEEEsingle(), V);
3846 Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
3847 } else if (EltVT == MVT::f64) {
3848 APFloat FV(APFloat::IEEEdouble(), V);
3849 Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
3850 } else {
3851 Ops.push_back(DAG.getConstant(V, dl, EltVT));
3852 }
3853 }
3854
3855 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
3856 return DAG.getBitcast(VT, ConstsNode);
3857}
3858
3859static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT,
3860 SelectionDAG &DAG, const SDLoc &dl) {
3861 APInt Undefs = APInt::getZero(Bits.size());
3862 return getConstVector(Bits, Undefs, VT, DAG, dl);
3863}
3864
3865/// Returns a vector of specified type with all zero elements.
3866static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,
3867 SelectionDAG &DAG, const SDLoc &dl) {
3868 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() ||
3869 VT.getVectorElementType() == MVT::i1) &&
3870 "Unexpected vector type");
3871
3872 // Try to build SSE/AVX zero vectors as <N x i32> bitcasted to their dest
3873 // type. This ensures they get CSE'd. But if the integer type is not
3874 // available, use a floating-point +0.0 instead.
3875 SDValue Vec;
3876 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3877 if (!Subtarget.hasSSE2() && VT.is128BitVector()) {
3878 Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32);
3879 } else if (VT.isFloatingPoint() &&
3880 TLI.isTypeLegal(VT.getVectorElementType())) {
3881 Vec = DAG.getConstantFP(+0.0, dl, VT);
3882 } else if (VT.getVectorElementType() == MVT::i1) {
3883 assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) &&
3884 "Unexpected vector type");
3885 Vec = DAG.getConstant(0, dl, VT);
3886 } else {
3887 unsigned Num32BitElts = VT.getSizeInBits() / 32;
3888 Vec = DAG.getConstant(0, dl, MVT::getVectorVT(MVT::i32, Num32BitElts));
3889 }
3890 return DAG.getBitcast(VT, Vec);
3891}
3892
3893// Helper to determine if the ops are all the extracted subvectors come from a
3894// single source. If we allow commute they don't have to be in order (Lo/Hi).
3895static SDValue getSplitVectorSrc(SDValue LHS, SDValue RHS, bool AllowCommute) {
3896 if (LHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
3897 RHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
3898 LHS.getValueType() != RHS.getValueType() ||
3899 LHS.getOperand(0) != RHS.getOperand(0))
3900 return SDValue();
3901
3902 SDValue Src = LHS.getOperand(0);
3903 if (Src.getValueSizeInBits() != (LHS.getValueSizeInBits() * 2))
3904 return SDValue();
3905
3906 unsigned NumElts = LHS.getValueType().getVectorNumElements();
3907 if ((LHS.getConstantOperandAPInt(1) == 0 &&
3908 RHS.getConstantOperandAPInt(1) == NumElts) ||
3909 (AllowCommute && RHS.getConstantOperandAPInt(1) == 0 &&
3910 LHS.getConstantOperandAPInt(1) == NumElts))
3911 return Src;
3912
3913 return SDValue();
3914}
3915
3916static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
3917 const SDLoc &dl, unsigned vectorWidth) {
3918 EVT VT = Vec.getValueType();
3919 EVT ElVT = VT.getVectorElementType();
3920 unsigned Factor = VT.getSizeInBits() / vectorWidth;
3921 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
3922 VT.getVectorNumElements() / Factor);
3923
3924 // Extract the relevant vectorWidth bits. Generate an EXTRACT_SUBVECTOR
3925 unsigned ElemsPerChunk = vectorWidth / ElVT.getSizeInBits();
3926 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
3927
3928 // This is the index of the first element of the vectorWidth-bit chunk
3929 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
3930 IdxVal &= ~(ElemsPerChunk - 1);
3931
3932 // If the input is a buildvector just emit a smaller one.
3933 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
3934 return DAG.getBuildVector(ResultVT, dl,
3935 Vec->ops().slice(IdxVal, ElemsPerChunk));
3936
3937 // Check if we're extracting the upper undef of a widening pattern.
3938 if (Vec.getOpcode() == ISD::INSERT_SUBVECTOR && Vec.getOperand(0).isUndef() &&
3939 Vec.getOperand(1).getValueType().getVectorNumElements() <= IdxVal &&
3940 isNullConstant(Vec.getOperand(2)))
3941 return DAG.getUNDEF(ResultVT);
3942
3943 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
3944 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx);
3945}
3946
3947/// Generate a DAG to grab 128-bits from a vector > 128 bits. This
3948/// sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128
3949/// or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4
3950/// instructions or a simple subregister reference. Idx is an index in the
3951/// 128 bits we want. It need not be aligned to a 128-bit boundary. That makes
3952/// lowering EXTRACT_VECTOR_ELT operations easier.
3953static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
3954 SelectionDAG &DAG, const SDLoc &dl) {
3955 assert((Vec.getValueType().is256BitVector() ||
3956 Vec.getValueType().is512BitVector()) && "Unexpected vector size!");
3957 return extractSubVector(Vec, IdxVal, DAG, dl, 128);
3958}
3959
3960/// Generate a DAG to grab 256-bits from a 512-bit vector.
3961static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal,
3962 SelectionDAG &DAG, const SDLoc &dl) {
3963 assert(Vec.getValueType().is512BitVector() && "Unexpected vector size!");
3964 return extractSubVector(Vec, IdxVal, DAG, dl, 256);
3965}
3966
3967static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
3968 SelectionDAG &DAG, const SDLoc &dl,
3969 unsigned vectorWidth) {
3970 assert((vectorWidth == 128 || vectorWidth == 256) &&
3971 "Unsupported vector width");
3972 // Inserting UNDEF is Result
3973 if (Vec.isUndef())
3974 return Result;
3975 EVT VT = Vec.getValueType();
3976 EVT ElVT = VT.getVectorElementType();
3977 EVT ResultVT = Result.getValueType();
3978
3979 // Insert the relevant vectorWidth bits.
3980 unsigned ElemsPerChunk = vectorWidth/ElVT.getSizeInBits();
3981 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
3982
3983 // This is the index of the first element of the vectorWidth-bit chunk
3984 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
3985 IdxVal &= ~(ElemsPerChunk - 1);
3986
3987 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
3988 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx);
3989}
3990
3991/// Generate a DAG to put 128-bits into a vector > 128 bits. This
3992/// sets things up to match to an AVX VINSERTF128/VINSERTI128 or
3993/// AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a
3994/// simple superregister reference. Idx is an index in the 128 bits
3995/// we want. It need not be aligned to a 128-bit boundary. That makes
3996/// lowering INSERT_VECTOR_ELT operations easier.
3997static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
3998 SelectionDAG &DAG, const SDLoc &dl) {
3999 assert(Vec.getValueType().is128BitVector() && "Unexpected vector size!");
4000 return insertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
4001}
4002
4003/// Widen a vector to a larger size with the same scalar type, with the new
4004/// elements either zero or undef.
4005static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
4006 const X86Subtarget &Subtarget, SelectionDAG &DAG,
4007 const SDLoc &dl) {
4008 assert(Vec.getValueSizeInBits().getFixedValue() <= VT.getFixedSizeInBits() &&
4009 Vec.getValueType().getScalarType() == VT.getScalarType() &&
4010 "Unsupported vector widening type");
4011 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
4012 : DAG.getUNDEF(VT);
4013 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec,
4014 DAG.getIntPtrConstant(0, dl));
4015}
4016
4017/// Widen a vector to a larger size with the same scalar type, with the new
4018/// elements either zero or undef.
4019static SDValue widenSubVector(SDValue Vec, bool ZeroNewElements,
4020 const X86Subtarget &Subtarget, SelectionDAG &DAG,
4021 const SDLoc &dl, unsigned WideSizeInBits) {
4022 assert(Vec.getValueSizeInBits() <= WideSizeInBits &&
4023 (WideSizeInBits % Vec.getScalarValueSizeInBits()) == 0 &&
4024 "Unsupported vector widening type");
4025 unsigned WideNumElts = WideSizeInBits / Vec.getScalarValueSizeInBits();
4026 MVT SVT = Vec.getSimpleValueType().getScalarType();
4027 MVT VT = MVT::getVectorVT(SVT, WideNumElts);
4028 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
4029}
4030
4031/// Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT
4032/// and bitcast with integer types.
4033static MVT widenMaskVectorType(MVT VT, const X86Subtarget &Subtarget) {
4034 assert(VT.getVectorElementType() == MVT::i1 && "Expected bool vector");
4035 unsigned NumElts = VT.getVectorNumElements();
4036 if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8)
4037 return Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
4038 return VT;
4039}
4040
4041/// Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and
4042/// bitcast with integer types.
4043static SDValue widenMaskVector(SDValue Vec, bool ZeroNewElements,
4044 const X86Subtarget &Subtarget, SelectionDAG &DAG,
4045 const SDLoc &dl) {
4046 MVT VT = widenMaskVectorType(Vec.getSimpleValueType(), Subtarget);
4047 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
4048}
4049
4050// Helper function to collect subvector ops that are concatenated together,
4051// either by ISD::CONCAT_VECTORS or a ISD::INSERT_SUBVECTOR series.
4052// The subvectors in Ops are guaranteed to be the same type.
4053static bool collectConcatOps(SDNode *N, SmallVectorImpl<SDValue> &Ops,
4054 SelectionDAG &DAG) {
4055 assert(Ops.empty() && "Expected an empty ops vector");
4056
4057 if (N->getOpcode() == ISD::CONCAT_VECTORS) {
4058 Ops.append(N->op_begin(), N->op_end());
4059 return true;
4060 }
4061
4062 if (N->getOpcode() == ISD::INSERT_SUBVECTOR) {
4063 SDValue Src = N->getOperand(0);
4064 SDValue Sub = N->getOperand(1);
4065 const APInt &Idx = N->getConstantOperandAPInt(2);
4066 EVT VT = Src.getValueType();
4067 EVT SubVT = Sub.getValueType();
4068
4069 if (VT.getSizeInBits() == (SubVT.getSizeInBits() * 2)) {
4070 // insert_subvector(undef, x, lo)
4071 if (Idx == 0 && Src.isUndef()) {
4072 Ops.push_back(Sub);
4073 Ops.push_back(DAG.getUNDEF(SubVT));
4074 return true;
4075 }
4076 if (Idx == (VT.getVectorNumElements() / 2)) {
4077 // insert_subvector(insert_subvector(undef, x, lo), y, hi)
4078 if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
4079 Src.getOperand(1).getValueType() == SubVT &&
4080 isNullConstant(Src.getOperand(2))) {
4081 // Attempt to recurse into inner (matching) concats.
4082 SDValue Lo = Src.getOperand(1);
4083 SDValue Hi = Sub;
4084 SmallVector<SDValue, 2> LoOps, HiOps;
4085 if (collectConcatOps(Lo.getNode(), LoOps, DAG) &&
4086 collectConcatOps(Hi.getNode(), HiOps, DAG) &&
4087 LoOps.size() == HiOps.size()) {
4088 Ops.append(LoOps);
4089 Ops.append(HiOps);
4090 return true;
4091 }
4092 Ops.push_back(Lo);
4093 Ops.push_back(Hi);
4094 return true;
4095 }
4096 // insert_subvector(x, extract_subvector(x, lo), hi)
4097 if (Sub.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
4098 Sub.getOperand(0) == Src && isNullConstant(Sub.getOperand(1))) {
4099 Ops.append(2, Sub);
4100 return true;
4101 }
4102 // insert_subvector(undef, x, hi)
4103 if (Src.isUndef()) {
4104 Ops.push_back(DAG.getUNDEF(SubVT));
4105 Ops.push_back(Sub);
4106 return true;
4107 }
4108 }
4109 }
4110 }
4111
4112 return false;
4113}
4114
4115// Helper to check if \p V can be split into subvectors and the upper subvectors
4116// are all undef. In which case return the lower subvector.
4117static SDValue isUpperSubvectorUndef(SDValue V, const SDLoc &DL,
4118 SelectionDAG &DAG) {
4119 SmallVector<SDValue> SubOps;
4120 if (!collectConcatOps(V.getNode(), SubOps, DAG))
4121 return SDValue();
4122
4123 unsigned NumSubOps = SubOps.size();
4124 unsigned HalfNumSubOps = NumSubOps / 2;
4125 assert((NumSubOps % 2) == 0 && "Unexpected number of subvectors");
4126
4127 ArrayRef<SDValue> UpperOps(SubOps.begin() + HalfNumSubOps, SubOps.end());
4128 if (any_of(UpperOps, [](SDValue Op) { return !Op.isUndef(); }))
4129 return SDValue();
4130
4131 EVT HalfVT = V.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
4132 ArrayRef<SDValue> LowerOps(SubOps.begin(), SubOps.begin() + HalfNumSubOps);
4133 return DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, LowerOps);
4134}
4135
4136// Helper to check if we can access all the constituent subvectors without any
4137// extract ops.
4138static bool isFreeToSplitVector(SDNode *N, SelectionDAG &DAG) {
4139 SmallVector<SDValue> Ops;
4140 return collectConcatOps(N, Ops, DAG);
4141}
4142
4143static std::pair<SDValue, SDValue> splitVector(SDValue Op, SelectionDAG &DAG,
4144 const SDLoc &dl) {
4145 EVT VT = Op.getValueType();
4146 unsigned NumElems = VT.getVectorNumElements();
4147 unsigned SizeInBits = VT.getSizeInBits();
4148 assert((NumElems % 2) == 0 && (SizeInBits % 2) == 0 &&
4149 "Can't split odd sized vector");
4150
4151 // If this is a splat value (with no-undefs) then use the lower subvector,
4152 // which should be a free extraction.
4153 SDValue Lo = extractSubVector(Op, 0, DAG, dl, SizeInBits / 2);
4154 if (DAG.isSplatValue(Op, /*AllowUndefs*/ false))
4155 return std::make_pair(Lo, Lo);
4156
4157 SDValue Hi = extractSubVector(Op, NumElems / 2, DAG, dl, SizeInBits / 2);
4158 return std::make_pair(Lo, Hi);
4159}
4160
4161/// Break an operation into 2 half sized ops and then concatenate the results.
4162static SDValue splitVectorOp(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) {
4163 unsigned NumOps = Op.getNumOperands();
4164 EVT VT = Op.getValueType();
4165
4166 // Extract the LHS Lo/Hi vectors
4167 SmallVector<SDValue> LoOps(NumOps, SDValue());
4168 SmallVector<SDValue> HiOps(NumOps, SDValue());
4169 for (unsigned I = 0; I != NumOps; ++I) {
4170 SDValue SrcOp = Op.getOperand(I);
4171 if (!SrcOp.getValueType().isVector()) {
4172 LoOps[I] = HiOps[I] = SrcOp;
4173 continue;
4174 }
4175 std::tie(LoOps[I], HiOps[I]) = splitVector(SrcOp, DAG, dl);
4176 }
4177
4178 EVT LoVT, HiVT;
4179 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
4180 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
4181 DAG.getNode(Op.getOpcode(), dl, LoVT, LoOps),
4182 DAG.getNode(Op.getOpcode(), dl, HiVT, HiOps));
4183}
4184
4185/// Break an unary integer operation into 2 half sized ops and then
4186/// concatenate the result back.
4187static SDValue splitVectorIntUnary(SDValue Op, SelectionDAG &DAG,
4188 const SDLoc &dl) {
4189 // Make sure we only try to split 256/512-bit types to avoid creating
4190 // narrow vectors.
4191 [[maybe_unused]] EVT VT = Op.getValueType();
4192 assert((Op.getOperand(0).getValueType().is256BitVector() ||
4193 Op.getOperand(0).getValueType().is512BitVector()) &&
4194 (VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!");
4195 assert(Op.getOperand(0).getValueType().getVectorNumElements() ==
4196 VT.getVectorNumElements() &&
4197 "Unexpected VTs!");
4198 return splitVectorOp(Op, DAG, dl);
4199}
4200
4201/// Break a binary integer operation into 2 half sized ops and then
4202/// concatenate the result back.
4203static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG,
4204 const SDLoc &dl) {
4205 // Assert that all the types match.
4206 [[maybe_unused]] EVT VT = Op.getValueType();
4207 assert(Op.getOperand(0).getValueType() == VT &&
4208 Op.getOperand(1).getValueType() == VT && "Unexpected VTs!");
4209 assert((VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!");
4210 return splitVectorOp(Op, DAG, dl);
4211}
4212
4213// Helper for splitting operands of an operation to legal target size and
4214// apply a function on each part.
4215// Useful for operations that are available on SSE2 in 128-bit, on AVX2 in
4216// 256-bit and on AVX512BW in 512-bit. The argument VT is the type used for
4217// deciding if/how to split Ops. Ops elements do *not* have to be of type VT.
4218// The argument Builder is a function that will be applied on each split part:
4219// SDValue Builder(SelectionDAG&G, SDLoc, ArrayRef<SDValue>)
4220template <typename F>
4221SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4222 const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
4223 F Builder, bool CheckBWI = true) {
4224 assert(Subtarget.hasSSE2() && "Target assumed to support at least SSE2");
4225 unsigned NumSubs = 1;
4226 if ((CheckBWI && Subtarget.useBWIRegs()) ||
4227 (!CheckBWI && Subtarget.useAVX512Regs())) {
4228 if (VT.getSizeInBits() > 512) {
4229 NumSubs = VT.getSizeInBits() / 512;
4230 assert((VT.getSizeInBits() % 512) == 0 && "Illegal vector size");
4231 }
4232 } else if (Subtarget.hasAVX2()) {
4233 if (VT.getSizeInBits() > 256) {
4234 NumSubs = VT.getSizeInBits() / 256;
4235 assert((VT.getSizeInBits() % 256) == 0 && "Illegal vector size");
4236 }
4237 } else {
4238 if (VT.getSizeInBits() > 128) {
4239 NumSubs = VT.getSizeInBits() / 128;
4240 assert((VT.getSizeInBits() % 128) == 0 && "Illegal vector size");
4241 }
4242 }
4243
4244 if (NumSubs == 1)
4245 return Builder(DAG, DL, Ops);
4246
4247 SmallVector<SDValue, 4> Subs;
4248 for (unsigned i = 0; i != NumSubs; ++i) {
4249 SmallVector<SDValue, 2> SubOps;
4250 for (SDValue Op : Ops) {
4251 EVT OpVT = Op.getValueType();
4252 unsigned NumSubElts = OpVT.getVectorNumElements() / NumSubs;
4253 unsigned SizeSub = OpVT.getSizeInBits() / NumSubs;
4254 SubOps.push_back(extractSubVector(Op, i * NumSubElts, DAG, DL, SizeSub));
4255 }
4256 Subs.push_back(Builder(DAG, DL, SubOps));
4257 }
4258 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
4259}
4260
4261// Helper function that extends a non-512-bit vector op to 512-bits on non-VLX
4262// targets.
4263static SDValue getAVX512Node(unsigned Opcode, const SDLoc &DL, MVT VT,
4264 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
4265 const X86Subtarget &Subtarget) {
4266 assert(Subtarget.hasAVX512() && "AVX512 target expected");
4267 MVT SVT = VT.getScalarType();
4268
4269 // If we have a 32/64 splatted constant, splat it to DstTy to
4270 // encourage a foldable broadcast'd operand.
4271 auto MakeBroadcastOp = [&](SDValue Op, MVT OpVT, MVT DstVT) {
4272 unsigned OpEltSizeInBits = OpVT.getScalarSizeInBits();
4273 // AVX512 broadcasts 32/64-bit operands.
4274 // TODO: Support float once getAVX512Node is used by fp-ops.
4275 if (!OpVT.isInteger() || OpEltSizeInBits < 32 ||
4276 !DAG.getTargetLoweringInfo().isTypeLegal(SVT))
4277 return SDValue();
4278 // If we're not widening, don't bother if we're not bitcasting.
4279 if (OpVT == DstVT && Op.getOpcode() != ISD::BITCAST)
4280 return SDValue();
4281 if (auto *BV = dyn_cast<BuildVectorSDNode>(peekThroughBitcasts(Op))) {
4282 APInt SplatValue, SplatUndef;
4283 unsigned SplatBitSize;
4284 bool HasAnyUndefs;
4285 if (BV->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
4286 HasAnyUndefs, OpEltSizeInBits) &&
4287 !HasAnyUndefs && SplatValue.getBitWidth() == OpEltSizeInBits)
4288 return DAG.getConstant(SplatValue, DL, DstVT);
4289 }
4290 return SDValue();
4291 };
4292
4293 bool Widen = !(Subtarget.hasVLX() || VT.is512BitVector());
4294
4295 MVT DstVT = VT;
4296 if (Widen)
4297 DstVT = MVT::getVectorVT(SVT, 512 / SVT.getSizeInBits());
4298
4299 // Canonicalize src operands.
4300 SmallVector<SDValue> SrcOps(Ops);
4301 for (SDValue &Op : SrcOps) {
4302 MVT OpVT = Op.getSimpleValueType();
4303 // Just pass through scalar operands.
4304 if (!OpVT.isVector())
4305 continue;
4306 assert(OpVT == VT && "Vector type mismatch");
4307
4308 if (SDValue BroadcastOp = MakeBroadcastOp(Op, OpVT, DstVT)) {
4309 Op = BroadcastOp;
4310 continue;
4311 }
4312
4313 // Just widen the subvector by inserting into an undef wide vector.
4314 if (Widen)
4315 Op = widenSubVector(Op, false, Subtarget, DAG, DL, 512);
4316 }
4317
4318 SDValue Res = DAG.getNode(Opcode, DL, DstVT, SrcOps);
4319
4320 // Perform the 512-bit op then extract the bottom subvector.
4321 if (Widen)
4322 Res = extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
4323 return Res;
4324}
4325
4326/// Insert i1-subvector to i1-vector.
4327static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
4328 const X86Subtarget &Subtarget) {
4329
4330 SDLoc dl(Op);
4331 SDValue Vec = Op.getOperand(0);
4332 SDValue SubVec = Op.getOperand(1);
4333 SDValue Idx = Op.getOperand(2);
4334 unsigned IdxVal = Op.getConstantOperandVal(2);
4335
4336 // Inserting undef is a nop. We can just return the original vector.
4337 if (SubVec.isUndef())
4338 return Vec;
4339
4340 if (IdxVal == 0 && Vec.isUndef()) // the operation is legal
4341 return Op;
4342
4343 MVT OpVT = Op.getSimpleValueType();
4344 unsigned NumElems = OpVT.getVectorNumElements();
4345 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
4346
4347 // Extend to natively supported kshift.
4348 MVT WideOpVT = widenMaskVectorType(OpVT, Subtarget);
4349
4350 // Inserting into the lsbs of a zero vector is legal. ISel will insert shifts
4351 // if necessary.
4352 if (IdxVal == 0 && ISD::isBuildVectorAllZeros(Vec.getNode())) {
4353 // May need to promote to a legal type.
4354 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4355 DAG.getConstant(0, dl, WideOpVT),
4356 SubVec, Idx);
4357 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4358 }
4359
4360 MVT SubVecVT = SubVec.getSimpleValueType();
4361 unsigned SubVecNumElems = SubVecVT.getVectorNumElements();
4362 assert(IdxVal + SubVecNumElems <= NumElems &&
4363 IdxVal % SubVecVT.getSizeInBits() == 0 &&
4364 "Unexpected index value in INSERT_SUBVECTOR");
4365
4366 SDValue Undef = DAG.getUNDEF(WideOpVT);
4367
4368 if (IdxVal == 0) {
4369 // Zero lower bits of the Vec
4370 SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
4371 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec,
4372 ZeroIdx);
4373 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
4374 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
4375 // Merge them together, SubVec should be zero extended.
4376 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4377 DAG.getConstant(0, dl, WideOpVT),
4378 SubVec, ZeroIdx);
4379 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4380 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4381 }
4382
4383 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4384 Undef, SubVec, ZeroIdx);
4385
4386 if (Vec.isUndef()) {
4387 assert(IdxVal != 0 && "Unexpected index");
4388 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4389 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4390 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4391 }
4392
4393 if (ISD::isBuildVectorAllZeros(Vec.getNode())) {
4394 assert(IdxVal != 0 && "Unexpected index");
4395 // If upper elements of Vec are known undef, then just shift into place.
4396 if (llvm::all_of(Vec->ops().slice(IdxVal + SubVecNumElems),
4397 [](SDValue V) { return V.isUndef(); })) {
4398 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4399 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4400 } else {
4401 NumElems = WideOpVT.getVectorNumElements();
4402 unsigned ShiftLeft = NumElems - SubVecNumElems;
4403 unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
4404 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4405 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4406 if (ShiftRight != 0)
4407 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4408 DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4409 }
4410 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4411 }
4412
4413 // Simple case when we put subvector in the upper part
4414 if (IdxVal + SubVecNumElems == NumElems) {
4415 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4416 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
4417 if (SubVecNumElems * 2 == NumElems) {
4418 // Special case, use legal zero extending insert_subvector. This allows
4419 // isel to optimize when bits are known zero.
4420 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx);
4421 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4422 DAG.getConstant(0, dl, WideOpVT),
4423 Vec, ZeroIdx);
4424 } else {
4425 // Otherwise use explicit shifts to zero the bits.
4426 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
4427 Undef, Vec, ZeroIdx);
4428 NumElems = WideOpVT.getVectorNumElements();
4429 SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
4430 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
4431 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
4432 }
4433 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4434 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4435 }
4436
4437 // Inserting into the middle is more complicated.
4438
4439 NumElems = WideOpVT.getVectorNumElements();
4440
4441 // Widen the vector if needed.
4442 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
4443
4444 unsigned ShiftLeft = NumElems - SubVecNumElems;
4445 unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
4446
4447 // Do an optimization for the most frequently used types.
4448 if (WideOpVT != MVT::v64i1 || Subtarget.is64Bit()) {
4449 APInt Mask0 = APInt::getBitsSet(NumElems, IdxVal, IdxVal + SubVecNumElems);
4450 Mask0.flipAllBits();
4451 SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems));
4452 SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0);
4453 Vec = DAG.getNode(ISD::AND, dl, WideOpVT, Vec, VMask0);
4454 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4455 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4456 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4457 DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4458 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
4459
4460 // Reduce to original width if needed.
4461 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
4462 }
4463
4464 // Clear the upper bits of the subvector and move it to its insert position.
4465 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
4466 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
4467 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
4468 DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
4469
4470 // Isolate the bits below the insertion point.
4471 unsigned LowShift = NumElems - IdxVal;
4472 SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
4473 DAG.getTargetConstant(LowShift, dl, MVT::i8));
4474 Low = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Low,
4475 DAG.getTargetConstant(LowShift, dl, MVT::i8));
4476
4477 // Isolate the bits after the last inserted bit.
4478 unsigned HighShift = IdxVal + SubVecNumElems;
4479 SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
4480 DAG.getTargetConstant(HighShift, dl, MVT::i8));
4481 High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High,
4482 DAG.getTargetConstant(HighShift, dl, MVT::i8));
4483
4484 // Now OR all 3 pieces together.
4485 Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High);
4486 SubVec = DAG.getNode(ISD::OR, dl, WideOpVT, SubVec, Vec);
4487
4488 // Reduce to original width if needed.
4489 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
4490}
4491
4492static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
4493 const SDLoc &dl) {
4494 assert(V1.getValueType() == V2.getValueType() && "subvector type mismatch");
4495 EVT SubVT = V1.getValueType();
4496 EVT SubSVT = SubVT.getScalarType();
4497 unsigned SubNumElts = SubVT.getVectorNumElements();
4498 unsigned SubVectorWidth = SubVT.getSizeInBits();
4499 EVT VT = EVT::getVectorVT(*DAG.getContext(), SubSVT, 2 * SubNumElts);
4500 SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth);
4501 return insertSubVector(V, V2, SubNumElts, DAG, dl, SubVectorWidth);
4502}
4503
4504/// Returns a vector of specified type with all bits set.
4505/// Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>.
4506/// Then bitcast to their original type, ensuring they get CSE'd.
4507static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
4508 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
4509 "Expected a 128/256/512-bit vector type");
4510 unsigned NumElts = VT.getSizeInBits() / 32;
4511 SDValue Vec = DAG.getAllOnesConstant(dl, MVT::getVectorVT(MVT::i32, NumElts));
4512 return DAG.getBitcast(VT, Vec);
4513}
4514
4515static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT,
4516 SDValue In, SelectionDAG &DAG) {
4517 EVT InVT = In.getValueType();
4518 assert(VT.isVector() && InVT.isVector() && "Expected vector VTs.");
4519 assert((ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode ||
4520 ISD::ZERO_EXTEND == Opcode) &&
4521 "Unknown extension opcode");
4522
4523 // For 256-bit vectors, we only need the lower (128-bit) input half.
4524 // For 512-bit vectors, we only need the lower input half or quarter.
4525 if (InVT.getSizeInBits() > 128) {
4526 assert(VT.getSizeInBits() == InVT.getSizeInBits() &&
4527 "Expected VTs to be the same size!");
4528 unsigned Scale = VT.getScalarSizeInBits() / InVT.getScalarSizeInBits();
4529 In = extractSubVector(In, 0, DAG, DL,
4530 std::max(128U, (unsigned)VT.getSizeInBits() / Scale));
4531 InVT = In.getValueType();
4532 }
4533
4534 if (VT.getVectorNumElements() != InVT.getVectorNumElements())
4535 Opcode = DAG.getOpcode_EXTEND_VECTOR_INREG(Opcode);
4536
4537 return DAG.getNode(Opcode, DL, VT, In);
4538}
4539
4540// Create OR(AND(LHS,MASK),AND(RHS,~MASK)) bit select pattern
4541static SDValue getBitSelect(const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS,
4542 SDValue Mask, SelectionDAG &DAG) {
4543 LHS = DAG.getNode(ISD::AND, DL, VT, LHS, Mask);
4544 RHS = DAG.getNode(X86ISD::ANDNP, DL, VT, Mask, RHS);
4545 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
4546}
4547
4548void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask,
4549 bool Lo, bool Unary) {
4550 assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 &&
4551 "Illegal vector type to unpack");
4552 assert(Mask.empty() && "Expected an empty shuffle mask vector");
4553 int NumElts = VT.getVectorNumElements();
4554 int NumEltsInLane = 128 / VT.getScalarSizeInBits();
4555 for (int i = 0; i < NumElts; ++i) {
4556 unsigned LaneStart = (i / NumEltsInLane) * NumEltsInLane;
4557 int Pos = (i % NumEltsInLane) / 2 + LaneStart;
4558 Pos += (Unary ? 0 : NumElts * (i % 2));
4559 Pos += (Lo ? 0 : NumEltsInLane / 2);
4560 Mask.push_back(Pos);
4561 }
4562}
4563
4564/// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
4565/// imposed by AVX and specific to the unary pattern. Example:
4566/// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
4567/// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
4568void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
4569 bool Lo) {
4570 assert(Mask.empty() && "Expected an empty shuffle mask vector");
4571 int NumElts = VT.getVectorNumElements();
4572 for (int i = 0; i < NumElts; ++i) {
4573 int Pos = i / 2;
4574 Pos += (Lo ? 0 : NumElts / 2);
4575 Mask.push_back(Pos);
4576 }
4577}
4578
4579// Attempt to constant fold, else just create a VECTOR_SHUFFLE.
4580static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl,
4581 SDValue V1, SDValue V2, ArrayRef<int> Mask) {
4582 if ((ISD::isBuildVectorOfConstantSDNodes(V1.getNode()) || V1.isUndef()) &&
4583 (ISD::isBuildVectorOfConstantSDNodes(V2.getNode()) || V2.isUndef())) {
4584 SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType()));
4585 for (int I = 0, NumElts = Mask.size(); I != NumElts; ++I) {
4586 int M = Mask[I];
4587 if (M < 0)
4588 continue;
4589 SDValue V = (M < NumElts) ? V1 : V2;
4590 if (V.isUndef())
4591 continue;
4592 Ops[I] = V.getOperand(M % NumElts);
4593 }
4594 return DAG.getBuildVector(VT, dl, Ops);
4595 }
4596
4597 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
4598}
4599
4600/// Returns a vector_shuffle node for an unpackl operation.
4601static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4602 SDValue V1, SDValue V2) {
4603 SmallVector<int, 8> Mask;
4604 createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false);
4605 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
4606}
4607
4608/// Returns a vector_shuffle node for an unpackh operation.
4609static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
4610 SDValue V1, SDValue V2) {
4611 SmallVector<int, 8> Mask;
4612 createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false);
4613 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask);
4614}
4615
4616/// Returns a node that packs the LHS + RHS nodes together at half width.
4617/// May return X86ISD::PACKSS/PACKUS, packing the top/bottom half.
4618/// TODO: Add subvector splitting if/when we have a need for it.
4619static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget,
4620 const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS,
4621 bool PackHiHalf = false) {
4622 MVT OpVT = LHS.getSimpleValueType();
4623 unsigned EltSizeInBits = VT.getScalarSizeInBits();
4624 bool UsePackUS = Subtarget.hasSSE41() || EltSizeInBits == 8;
4625 assert(OpVT == RHS.getSimpleValueType() &&
4626 VT.getSizeInBits() == OpVT.getSizeInBits() &&
4627 (EltSizeInBits * 2) == OpVT.getScalarSizeInBits() &&
4628 "Unexpected PACK operand types");
4629 assert((EltSizeInBits == 8 || EltSizeInBits == 16 || EltSizeInBits == 32) &&
4630 "Unexpected PACK result type");
4631
4632 // Rely on vector shuffles for vXi64 -> vXi32 packing.
4633 if (EltSizeInBits == 32) {
4634 SmallVector<int> PackMask;
4635 int Offset = PackHiHalf ? 1 : 0;
4636 int NumElts = VT.getVectorNumElements();
4637 for (int I = 0; I != NumElts; I += 4) {
4638 PackMask.push_back(I + Offset);
4639 PackMask.push_back(I + Offset + 2);
4640 PackMask.push_back(I + Offset + NumElts);
4641 PackMask.push_back(I + Offset + NumElts + 2);
4642 }
4643 return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS),
4644 DAG.getBitcast(VT, RHS), PackMask);
4645 }
4646
4647 // See if we already have sufficient leading bits for PACKSS/PACKUS.
4648 if (!PackHiHalf) {
4649 if (UsePackUS &&
4650 DAG.computeKnownBits(LHS).countMaxActiveBits() <= EltSizeInBits &&
4651 DAG.computeKnownBits(RHS).countMaxActiveBits() <= EltSizeInBits)
4652 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
4653
4654 if (DAG.ComputeMaxSignificantBits(LHS) <= EltSizeInBits &&
4655 DAG.ComputeMaxSignificantBits(RHS) <= EltSizeInBits)
4656 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
4657 }
4658
4659 // Fallback to sign/zero extending the requested half and pack.
4660 SDValue Amt = DAG.getTargetConstant(EltSizeInBits, dl, MVT::i8);
4661 if (UsePackUS) {
4662 if (PackHiHalf) {
4663 LHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, LHS, Amt);
4664 RHS = DAG.getNode(X86ISD::VSRLI, dl, OpVT, RHS, Amt);
4665 } else {
4666 SDValue Mask = DAG.getConstant((1ULL << EltSizeInBits) - 1, dl, OpVT);
4667 LHS = DAG.getNode(ISD::AND, dl, OpVT, LHS, Mask);
4668 RHS = DAG.getNode(ISD::AND, dl, OpVT, RHS, Mask);
4669 };
4670 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS);
4671 };
4672
4673 if (!PackHiHalf) {
4674 LHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, LHS, Amt);
4675 RHS = DAG.getNode(X86ISD::VSHLI, dl, OpVT, RHS, Amt);
4676 }
4677 LHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, LHS, Amt);
4678 RHS = DAG.getNode(X86ISD::VSRAI, dl, OpVT, RHS, Amt);
4679 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS);
4680}
4681
4682/// Return a vector_shuffle of the specified vector of zero or undef vector.
4683/// This produces a shuffle where the low element of V2 is swizzled into the
4684/// zero/undef vector, landing at element Idx.
4685/// This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
4686static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
4687 bool IsZero,
4688 const X86Subtarget &Subtarget,
4689 SelectionDAG &DAG) {
4690 MVT VT = V2.getSimpleValueType();
4691 SDValue V1 = IsZero
4692 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
4693 int NumElems = VT.getVectorNumElements();
4694 SmallVector<int, 16> MaskVec(NumElems);
4695 for (int i = 0; i != NumElems; ++i)
4696 // If this is the insertion idx, put the low elt of V2 here.
4697 MaskVec[i] = (i == Idx) ? NumElems : i;
4698 return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec);
4699}
4700
4701static ConstantPoolSDNode *getTargetConstantPoolFromBasePtr(SDValue Ptr) {
4702 if (Ptr.getOpcode() == X86ISD::Wrapper ||
4703 Ptr.getOpcode() == X86ISD::WrapperRIP)
4704 Ptr = Ptr.getOperand(0);
4705 return dyn_cast<ConstantPoolSDNode>(Ptr);
4706}
4707
4708// TODO: Add support for non-zero offsets.
4709static const Constant *getTargetConstantFromBasePtr(SDValue Ptr) {
4710 ConstantPoolSDNode *CNode = getTargetConstantPoolFromBasePtr(Ptr);
4711 if (!CNode || CNode->isMachineConstantPoolEntry() || CNode->getOffset() != 0)
4712 return nullptr;
4713 return CNode->getConstVal();
4714}
4715
4716static const Constant *getTargetConstantFromNode(LoadSDNode *Load) {
4717 if (!Load || !ISD::isNormalLoad(Load))
4718 return nullptr;
4719 return getTargetConstantFromBasePtr(Load->getBasePtr());
4720}
4721
4722static const Constant *getTargetConstantFromNode(SDValue Op) {
4723 Op = peekThroughBitcasts(Op);
4724 return getTargetConstantFromNode(dyn_cast<LoadSDNode>(Op));
4725}
4726
4727const Constant *
4728X86TargetLowering::getTargetConstantFromLoad(LoadSDNode *LD) const {
4729 assert(LD && "Unexpected null LoadSDNode");
4730 return getTargetConstantFromNode(LD);
4731}
4732
4733// Extract raw constant bits from constant pools.
4734static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
4735 APInt &UndefElts,
4736 SmallVectorImpl<APInt> &EltBits,
4737 bool AllowWholeUndefs = true,
4738 bool AllowPartialUndefs = false) {
4739 assert(EltBits.empty() && "Expected an empty EltBits vector");
4740
4741 Op = peekThroughBitcasts(Op);
4742
4743 EVT VT = Op.getValueType();
4744 unsigned SizeInBits = VT.getSizeInBits();
4745 assert((SizeInBits % EltSizeInBits) == 0 && "Can't split constant!");
4746 unsigned NumElts = SizeInBits / EltSizeInBits;
4747
4748 // Bitcast a source array of element bits to the target size.
4749 auto CastBitData = [&](APInt &UndefSrcElts, ArrayRef<APInt> SrcEltBits) {
4750 unsigned NumSrcElts = UndefSrcElts.getBitWidth();
4751 unsigned SrcEltSizeInBits = SrcEltBits[0].getBitWidth();
4752 assert((NumSrcElts * SrcEltSizeInBits) == SizeInBits &&
4753 "Constant bit sizes don't match");
4754
4755 // Don't split if we don't allow undef bits.
4756 bool AllowUndefs = AllowWholeUndefs || AllowPartialUndefs;
4757 if (UndefSrcElts.getBoolValue() && !AllowUndefs)
4758 return false;
4759
4760 // If we're already the right size, don't bother bitcasting.
4761 if (NumSrcElts == NumElts) {
4762 UndefElts = UndefSrcElts;
4763 EltBits.assign(SrcEltBits.begin(), SrcEltBits.end());
4764 return true;
4765 }
4766
4767 // Extract all the undef/constant element data and pack into single bitsets.
4768 APInt UndefBits(SizeInBits, 0);
4769 APInt MaskBits(SizeInBits, 0);
4770
4771 for (unsigned i = 0; i != NumSrcElts; ++i) {
4772 unsigned BitOffset = i * SrcEltSizeInBits;
4773 if (UndefSrcElts[i])
4774 UndefBits.setBits(BitOffset, BitOffset + SrcEltSizeInBits);
4775 MaskBits.insertBits(SrcEltBits[i], BitOffset);
4776 }
4777
4778 // Split the undef/constant single bitset data into the target elements.
4779 UndefElts = APInt(NumElts, 0);
4780 EltBits.resize(NumElts, APInt(EltSizeInBits, 0));
4781
4782 for (unsigned i = 0; i != NumElts; ++i) {
4783 unsigned BitOffset = i * EltSizeInBits;
4784 APInt UndefEltBits = UndefBits.extractBits(EltSizeInBits, BitOffset);
4785
4786 // Only treat an element as UNDEF if all bits are UNDEF.
4787 if (UndefEltBits.isAllOnes()) {
4788 if (!AllowWholeUndefs)
4789 return false;
4790 UndefElts.setBit(i);
4791 continue;
4792 }
4793
4794 // If only some bits are UNDEF then treat them as zero (or bail if not
4795 // supported).
4796 if (UndefEltBits.getBoolValue() && !AllowPartialUndefs)
4797 return false;
4798
4799 EltBits[i] = MaskBits.extractBits(EltSizeInBits, BitOffset);
4800 }
4801 return true;
4802 };
4803
4804 // Collect constant bits and insert into mask/undef bit masks.
4805 auto CollectConstantBits = [](const Constant *Cst, APInt &Mask, APInt &Undefs,
4806 unsigned UndefBitIndex) {
4807 if (!Cst)
4808 return false;
4809 if (isa<UndefValue>(Cst)) {
4810 Undefs.setBit(UndefBitIndex);
4811 return true;
4812 }
4813 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
4814 Mask = CInt->getValue();
4815 return true;
4816 }
4817 if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
4818 Mask = CFP->getValueAPF().bitcastToAPInt();
4819 return true;
4820 }
4821 if (auto *CDS = dyn_cast<ConstantDataSequential>(Cst)) {
4822 Type *Ty = CDS->getType();
4823 Mask = APInt::getZero(Ty->getPrimitiveSizeInBits());
4824 Type *EltTy = CDS->getElementType();
4825 bool IsInteger = EltTy->isIntegerTy();
4826 bool IsFP =
4827 EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();
4828 if (!IsInteger && !IsFP)
4829 return false;
4830 unsigned EltBits = EltTy->getPrimitiveSizeInBits();
4831 for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I)
4832 if (IsInteger)
4833 Mask.insertBits(CDS->getElementAsAPInt(I), I * EltBits);
4834 else
4835 Mask.insertBits(CDS->getElementAsAPFloat(I).bitcastToAPInt(),
4836 I * EltBits);
4837 return true;
4838 }
4839 return false;
4840 };
4841
4842 // Handle UNDEFs.
4843 if (Op.isUndef()) {
4844 APInt UndefSrcElts = APInt::getAllOnes(NumElts);
4845 SmallVector<APInt, 64> SrcEltBits(NumElts, APInt(EltSizeInBits, 0));
4846 return CastBitData(UndefSrcElts, SrcEltBits);
4847 }
4848
4849 // Extract scalar constant bits.
4850 if (auto *Cst = dyn_cast<ConstantSDNode>(Op)) {
4851 APInt UndefSrcElts = APInt::getZero(1);
4852 SmallVector<APInt, 64> SrcEltBits(1, Cst->getAPIntValue());
4853 return CastBitData(UndefSrcElts, SrcEltBits);
4854 }
4855 if (auto *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
4856 APInt UndefSrcElts = APInt::getZero(1);
4857 APInt RawBits = Cst->getValueAPF().bitcastToAPInt();
4858 SmallVector<APInt, 64> SrcEltBits(1, RawBits);
4859 return CastBitData(UndefSrcElts, SrcEltBits);
4860 }
4861
4862 // Extract constant bits from build vector.
4863 if (auto *BV = dyn_cast<BuildVectorSDNode>(Op)) {
4864 BitVector Undefs;
4865 SmallVector<APInt> SrcEltBits;
4866 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4867 if (BV->getConstantRawBits(true, SrcEltSizeInBits, SrcEltBits, Undefs)) {
4868 APInt UndefSrcElts = APInt::getZero(SrcEltBits.size());
4869 for (unsigned I = 0, E = SrcEltBits.size(); I != E; ++I)
4870 if (Undefs[I])
4871 UndefSrcElts.setBit(I);
4872 return CastBitData(UndefSrcElts, SrcEltBits);
4873 }
4874 }
4875
4876 // Extract constant bits from constant pool vector.
4877 if (auto *Cst = getTargetConstantFromNode(Op)) {
4878 Type *CstTy = Cst->getType();
4879 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
4880 if (!CstTy->isVectorTy() || (CstSizeInBits % SizeInBits) != 0)
4881 return false;
4882
4883 unsigned SrcEltSizeInBits = CstTy->getScalarSizeInBits();
4884 unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
4885 if ((SizeInBits % SrcEltSizeInBits) != 0)
4886 return false;
4887
4888 APInt UndefSrcElts(NumSrcElts, 0);
4889 SmallVector<APInt, 64> SrcEltBits(NumSrcElts, APInt(SrcEltSizeInBits, 0));
4890 for (unsigned i = 0; i != NumSrcElts; ++i)
4891 if (!CollectConstantBits(Cst->getAggregateElement(i), SrcEltBits[i],
4892 UndefSrcElts, i))
4893 return false;
4894
4895 return CastBitData(UndefSrcElts, SrcEltBits);
4896 }
4897
4898 // Extract constant bits from a broadcasted constant pool scalar.
4899 if (Op.getOpcode() == X86ISD::VBROADCAST_LOAD &&
4900 EltSizeInBits <= VT.getScalarSizeInBits()) {
4901 auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
4902 if (MemIntr->getMemoryVT().getStoreSizeInBits() != VT.getScalarSizeInBits())
4903 return false;
4904
4905 SDValue Ptr = MemIntr->getBasePtr();
4906 if (const Constant *C = getTargetConstantFromBasePtr(Ptr)) {
4907 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4908 unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
4909
4910 APInt UndefSrcElts(NumSrcElts, 0);
4911 SmallVector<APInt, 64> SrcEltBits(1, APInt(SrcEltSizeInBits, 0));
4912 if (CollectConstantBits(C, SrcEltBits[0], UndefSrcElts, 0)) {
4913 if (UndefSrcElts[0])
4914 UndefSrcElts.setBits(0, NumSrcElts);
4915 if (SrcEltBits[0].getBitWidth() != SrcEltSizeInBits)
4916 SrcEltBits[0] = SrcEltBits[0].trunc(SrcEltSizeInBits);
4917 SrcEltBits.append(NumSrcElts - 1, SrcEltBits[0]);
4918 return CastBitData(UndefSrcElts, SrcEltBits);
4919 }
4920 }
4921 }
4922
4923 // Extract constant bits from a subvector broadcast.
4924 if (Op.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) {
4925 auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
4926 SDValue Ptr = MemIntr->getBasePtr();
4927 // The source constant may be larger than the subvector broadcast,
4928 // ensure we extract the correct subvector constants.
4929 if (const Constant *Cst = getTargetConstantFromBasePtr(Ptr)) {
4930 Type *CstTy = Cst->getType();
4931 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
4932 unsigned SubVecSizeInBits = MemIntr->getMemoryVT().getStoreSizeInBits();
4933 if (!CstTy->isVectorTy() || (CstSizeInBits % SubVecSizeInBits) != 0 ||
4934 (SizeInBits % SubVecSizeInBits) != 0)
4935 return false;
4936 unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
4937 unsigned NumSubElts = SubVecSizeInBits / CstEltSizeInBits;
4938 unsigned NumSubVecs = SizeInBits / SubVecSizeInBits;
4939 APInt UndefSubElts(NumSubElts, 0);
4940 SmallVector<APInt, 64> SubEltBits(NumSubElts * NumSubVecs,
4941 APInt(CstEltSizeInBits, 0));
4942 for (unsigned i = 0; i != NumSubElts; ++i) {
4943 if (!CollectConstantBits(Cst->getAggregateElement(i), SubEltBits[i],
4944 UndefSubElts, i))
4945 return false;
4946 for (unsigned j = 1; j != NumSubVecs; ++j)
4947 SubEltBits[i + (j * NumSubElts)] = SubEltBits[i];
4948 }
4949 UndefSubElts = APInt::getSplat(NumSubVecs * UndefSubElts.getBitWidth(),
4950 UndefSubElts);
4951 return CastBitData(UndefSubElts, SubEltBits);
4952 }
4953 }
4954
4955 // Extract a rematerialized scalar constant insertion.
4956 if (Op.getOpcode() == X86ISD::VZEXT_MOVL &&
4957 Op.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR &&
4958 isa<ConstantSDNode>(Op.getOperand(0).getOperand(0))) {
4959 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4960 unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
4961
4962 APInt UndefSrcElts(NumSrcElts, 0);
4963 SmallVector<APInt, 64> SrcEltBits;
4964 const APInt &C = Op.getOperand(0).getConstantOperandAPInt(0);
4965 SrcEltBits.push_back(C.zextOrTrunc(SrcEltSizeInBits));
4966 SrcEltBits.append(NumSrcElts - 1, APInt(SrcEltSizeInBits, 0));
4967 return CastBitData(UndefSrcElts, SrcEltBits);
4968 }
4969
4970 // Insert constant bits from a base and sub vector sources.
4971 if (Op.getOpcode() == ISD::INSERT_SUBVECTOR) {
4972 // If bitcasts to larger elements we might lose track of undefs - don't
4973 // allow any to be safe.
4974 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
4975 bool AllowUndefs = EltSizeInBits >= SrcEltSizeInBits;
4976
4977 APInt UndefSrcElts, UndefSubElts;
4978 SmallVector<APInt, 32> EltSrcBits, EltSubBits;
4979 if (getTargetConstantBitsFromNode(Op.getOperand(1), SrcEltSizeInBits,
4980 UndefSubElts, EltSubBits,
4981 AllowWholeUndefs && AllowUndefs,
4982 AllowPartialUndefs && AllowUndefs) &&
4983 getTargetConstantBitsFromNode(Op.getOperand(0), SrcEltSizeInBits,
4984 UndefSrcElts, EltSrcBits,
4985 AllowWholeUndefs && AllowUndefs,
4986 AllowPartialUndefs && AllowUndefs)) {
4987 unsigned BaseIdx = Op.getConstantOperandVal(2);
4988 UndefSrcElts.insertBits(UndefSubElts, BaseIdx);
4989 for (unsigned i = 0, e = EltSubBits.size(); i != e; ++i)
4990 EltSrcBits[BaseIdx + i] = EltSubBits[i];
4991 return CastBitData(UndefSrcElts, EltSrcBits);
4992 }
4993 }
4994
4995 // Extract constant bits from a subvector's source.
4996 if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
4997 // TODO - support extract_subvector through bitcasts.
4998 if (EltSizeInBits != VT.getScalarSizeInBits())
4999 return false;
5000
5001 if (getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
5002 UndefElts, EltBits, AllowWholeUndefs,
5003 AllowPartialUndefs)) {
5004 EVT SrcVT = Op.getOperand(0).getValueType();
5005 unsigned NumSrcElts = SrcVT.getVectorNumElements();
5006 unsigned NumSubElts = VT.getVectorNumElements();
5007 unsigned BaseIdx = Op.getConstantOperandVal(1);
5008 UndefElts = UndefElts.extractBits(NumSubElts, BaseIdx);
5009 if ((BaseIdx + NumSubElts) != NumSrcElts)
5010 EltBits.erase(EltBits.begin() + BaseIdx + NumSubElts, EltBits.end());
5011 if (BaseIdx != 0)
5012 EltBits.erase(EltBits.begin(), EltBits.begin() + BaseIdx);
5013 return true;
5014 }
5015 }
5016
5017 // Extract constant bits from shuffle node sources.
5018 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Op)) {
5019 // TODO - support shuffle through bitcasts.
5020 if (EltSizeInBits != VT.getScalarSizeInBits())
5021 return false;
5022
5023 ArrayRef<int> Mask = SVN->getMask();
5024 if ((!AllowWholeUndefs || !AllowPartialUndefs) &&
5025 llvm::any_of(Mask, [](int M) { return M < 0; }))
5026 return false;
5027
5028 APInt UndefElts0, UndefElts1;
5029 SmallVector<APInt, 32> EltBits0, EltBits1;
5030 if (isAnyInRange(Mask, 0, NumElts) &&
5031 !getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
5032 UndefElts0, EltBits0, AllowWholeUndefs,
5033 AllowPartialUndefs))
5034 return false;
5035 if (isAnyInRange(Mask, NumElts, 2 * NumElts) &&
5036 !getTargetConstantBitsFromNode(Op.getOperand(1), EltSizeInBits,
5037 UndefElts1, EltBits1, AllowWholeUndefs,
5038 AllowPartialUndefs))
5039 return false;
5040
5041 UndefElts = APInt::getZero(NumElts);
5042 for (int i = 0; i != (int)NumElts; ++i) {
5043 int M = Mask[i];
5044 if (M < 0) {
5045 UndefElts.setBit(i);
5046 EltBits.push_back(APInt::getZero(EltSizeInBits));
5047 } else if (M < (int)NumElts) {
5048 if (UndefElts0[M])
5049 UndefElts.setBit(i);
5050 EltBits.push_back(EltBits0[M]);
5051 } else {
5052 if (UndefElts1[M - NumElts])
5053 UndefElts.setBit(i);
5054 EltBits.push_back(EltBits1[M - NumElts]);
5055 }
5056 }
5057 return true;
5058 }
5059
5060 return false;
5061}
5062
5063namespace llvm {
5064namespace X86 {
5065bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs) {
5066 APInt UndefElts;
5067 SmallVector<APInt, 16> EltBits;
5068 if (getTargetConstantBitsFromNode(
5069 Op, Op.getScalarValueSizeInBits(), UndefElts, EltBits,
5070 /*AllowWholeUndefs*/ true, AllowPartialUndefs)) {
5071 int SplatIndex = -1;
5072 for (int i = 0, e = EltBits.size(); i != e; ++i) {
5073 if (UndefElts[i])
5074 continue;
5075 if (0 <= SplatIndex && EltBits[i] != EltBits[SplatIndex]) {
5076 SplatIndex = -1;
5077 break;
5078 }
5079 SplatIndex = i;
5080 }
5081 if (0 <= SplatIndex) {
5082 SplatVal = EltBits[SplatIndex];
5083 return true;
5084 }
5085 }
5086
5087 return false;
5088}
5089} // namespace X86
5090} // namespace llvm
5091
5092static bool getTargetShuffleMaskIndices(SDValue MaskNode,
5093 unsigned MaskEltSizeInBits,
5094 SmallVectorImpl<uint64_t> &RawMask,
5095 APInt &UndefElts) {
5096 // Extract the raw target constant bits.
5097 SmallVector<APInt, 64> EltBits;
5098 if (!getTargetConstantBitsFromNode(MaskNode, MaskEltSizeInBits, UndefElts,
5099 EltBits, /* AllowWholeUndefs */ true,
5100 /* AllowPartialUndefs */ false))
5101 return false;
5102
5103 // Insert the extracted elements into the mask.
5104 for (const APInt &Elt : EltBits)
5105 RawMask.push_back(Elt.getZExtValue());
5106
5107 return true;
5108}
5109
5110// Match not(xor X, -1) -> X.
5111// Match not(pcmpgt(C, X)) -> pcmpgt(X, C - 1).
5112// Match not(extract_subvector(xor X, -1)) -> extract_subvector(X).
5113// Match not(concat_vectors(xor X, -1, xor Y, -1)) -> concat_vectors(X, Y).
5114static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
5115 V = peekThroughBitcasts(V);
5116 if (V.getOpcode() == ISD::XOR &&
5117 (ISD::isBuildVectorAllOnes(V.getOperand(1).getNode()) ||
5118 isAllOnesConstant(V.getOperand(1))))
5119 return V.getOperand(0);
5120 if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5121 (isNullConstant(V.getOperand(1)) || V.getOperand(0).hasOneUse())) {
5122 if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
5123 Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not);
5124 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), V.getValueType(),
5125 Not, V.getOperand(1));
5126 }
5127 }
5128 if (V.getOpcode() == X86ISD::PCMPGT &&
5129 !ISD::isBuildVectorAllZeros(V.getOperand(0).getNode()) &&
5130 !ISD::isBuildVectorAllOnes(V.getOperand(0).getNode()) &&
5131 V.getOperand(0).hasOneUse()) {
5132 APInt UndefElts;
5133 SmallVector<APInt> EltBits;
5134 if (getTargetConstantBitsFromNode(V.getOperand(0),
5135 V.getScalarValueSizeInBits(), UndefElts,
5136 EltBits)) {
5137 // Don't fold min_signed_value -> (min_signed_value - 1)
5138 bool MinSigned = false;
5139 for (APInt &Elt : EltBits) {
5140 MinSigned |= Elt.isMinSignedValue();
5141 Elt -= 1;
5142 }
5143 if (!MinSigned) {
5144 SDLoc DL(V);
5145 MVT VT = V.getSimpleValueType();
5146 return DAG.getNode(X86ISD::PCMPGT, DL, VT, V.getOperand(1),
5147 getConstVector(EltBits, UndefElts, VT, DAG, DL));
5148 }
5149 }
5150 }
5151 SmallVector<SDValue, 2> CatOps;
5152 if (collectConcatOps(V.getNode(), CatOps, DAG)) {
5153 for (SDValue &CatOp : CatOps) {
5154 SDValue NotCat = IsNOT(CatOp, DAG);
5155 if (!NotCat) return SDValue();
5156 CatOp = DAG.getBitcast(CatOp.getValueType(), NotCat);
5157 }
5158 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(V), V.getValueType(), CatOps);
5159 }
5160 return SDValue();
5161}
5162
5163/// Create a shuffle mask that matches the PACKSS/PACKUS truncation.
5164/// A multi-stage pack shuffle mask is created by specifying NumStages > 1.
5165/// Note: This ignores saturation, so inputs must be checked first.
5166static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
5167 bool Unary, unsigned NumStages = 1) {
5168 assert(Mask.empty() && "Expected an empty shuffle mask vector");
5169 unsigned NumElts = VT.getVectorNumElements();
5170 unsigned NumLanes = VT.getSizeInBits() / 128;
5171 unsigned NumEltsPerLane = 128 / VT.getScalarSizeInBits();
5172 unsigned Offset = Unary ? 0 : NumElts;
5173 unsigned Repetitions = 1u << (NumStages - 1);
5174 unsigned Increment = 1u << NumStages;
5175 assert((NumEltsPerLane >> NumStages) > 0 && "Illegal packing compaction");
5176
5177 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
5178 for (unsigned Stage = 0; Stage != Repetitions; ++Stage) {
5179 for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
5180 Mask.push_back(Elt + (Lane * NumEltsPerLane));
5181 for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += Increment)
5182 Mask.push_back(Elt + (Lane * NumEltsPerLane) + Offset);
5183 }
5184 }
5185}
5186
5187// Split the demanded elts of a PACKSS/PACKUS node between its operands.
5188static void getPackDemandedElts(EVT VT, const APInt &DemandedElts,
5189 APInt &DemandedLHS, APInt &DemandedRHS) {
5190 int NumLanes = VT.getSizeInBits() / 128;
5191 int NumElts = DemandedElts.getBitWidth();
5192 int NumInnerElts = NumElts / 2;
5193 int NumEltsPerLane = NumElts / NumLanes;
5194 int NumInnerEltsPerLane = NumInnerElts / NumLanes;
5195
5196 DemandedLHS = APInt::getZero(NumInnerElts);
5197 DemandedRHS = APInt::getZero(NumInnerElts);
5198
5199 // Map DemandedElts to the packed operands.
5200 for (int Lane = 0; Lane != NumLanes; ++Lane) {
5201 for (int Elt = 0; Elt != NumInnerEltsPerLane; ++Elt) {
5202 int OuterIdx = (Lane * NumEltsPerLane) + Elt;
5203 int InnerIdx = (Lane * NumInnerEltsPerLane) + Elt;
5204 if (DemandedElts[OuterIdx])
5205 DemandedLHS.setBit(InnerIdx);
5206 if (DemandedElts[OuterIdx + NumInnerEltsPerLane])
5207 DemandedRHS.setBit(InnerIdx);
5208 }
5209 }
5210}
5211
5212// Split the demanded elts of a HADD/HSUB node between its operands.
5213static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts,
5214 APInt &DemandedLHS, APInt &DemandedRHS) {
5215 getHorizDemandedEltsForFirstOperand(VT.getSizeInBits(), DemandedElts,
5216 DemandedLHS, DemandedRHS);
5217 DemandedLHS |= DemandedLHS << 1;
5218 DemandedRHS |= DemandedRHS << 1;
5219}
5220
5221/// Calculates the shuffle mask corresponding to the target-specific opcode.
5222/// If the mask could be calculated, returns it in \p Mask, returns the shuffle
5223/// operands in \p Ops, and returns true.
5224/// Sets \p IsUnary to true if only one source is used. Note that this will set
5225/// IsUnary for shuffles which use a single input multiple times, and in those
5226/// cases it will adjust the mask to only have indices within that single input.
5227/// It is an error to call this with non-empty Mask/Ops vectors.
5228static bool getTargetShuffleMask(SDValue N, bool AllowSentinelZero,
5229 SmallVectorImpl<SDValue> &Ops,
5230 SmallVectorImpl<int> &Mask, bool &IsUnary) {
5231 if (!isTargetShuffle(N.getOpcode()))
5232 return false;
5233
5234 MVT VT = N.getSimpleValueType();
5235 unsigned NumElems = VT.getVectorNumElements();
5236 unsigned MaskEltSize = VT.getScalarSizeInBits();
5237 SmallVector<uint64_t, 32> RawMask;
5238 APInt RawUndefs;
5239 uint64_t ImmN;
5240
5241 assert(Mask.empty() && "getTargetShuffleMask expects an empty Mask vector");
5242 assert(Ops.empty() && "getTargetShuffleMask expects an empty Ops vector");
5243
5244 IsUnary = false;
5245 bool IsFakeUnary = false;
5246 switch (N.getOpcode()) {
5247 case X86ISD::BLENDI:
5248 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5249 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5250 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5251 DecodeBLENDMask(NumElems, ImmN, Mask);
5252 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5253 break;
5254 case X86ISD::SHUFP:
5255 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5256 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5257 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5258 DecodeSHUFPMask(NumElems, MaskEltSize, ImmN, Mask);
5259 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5260 break;
5261 case X86ISD::INSERTPS:
5262 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5263 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5264 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5265 DecodeINSERTPSMask(ImmN, Mask);
5266 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5267 break;
5268 case X86ISD::EXTRQI:
5269 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5270 if (isa<ConstantSDNode>(N.getOperand(1)) &&
5271 isa<ConstantSDNode>(N.getOperand(2))) {
5272 int BitLen = N.getConstantOperandVal(1);
5273 int BitIdx = N.getConstantOperandVal(2);
5274 DecodeEXTRQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
5275 IsUnary = true;
5276 }
5277 break;
5278 case X86ISD::INSERTQI:
5279 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5280 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5281 if (isa<ConstantSDNode>(N.getOperand(2)) &&
5282 isa<ConstantSDNode>(N.getOperand(3))) {
5283 int BitLen = N.getConstantOperandVal(2);
5284 int BitIdx = N.getConstantOperandVal(3);
5285 DecodeINSERTQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
5286 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5287 }
5288 break;
5289 case X86ISD::UNPCKH:
5290 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5291 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5292 DecodeUNPCKHMask(NumElems, MaskEltSize, Mask);
5293 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5294 break;
5295 case X86ISD::UNPCKL:
5296 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5297 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5298 DecodeUNPCKLMask(NumElems, MaskEltSize, Mask);
5299 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5300 break;
5301 case X86ISD::MOVHLPS:
5302 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5303 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5304 DecodeMOVHLPSMask(NumElems, Mask);
5305 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5306 break;
5307 case X86ISD::MOVLHPS:
5308 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5309 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5310 DecodeMOVLHPSMask(NumElems, Mask);
5311 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5312 break;
5313 case X86ISD::VALIGN:
5314 assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
5315 "Only 32-bit and 64-bit elements are supported!");
5316 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5317 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5318 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5319 DecodeVALIGNMask(NumElems, ImmN, Mask);
5320 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5321 Ops.push_back(N.getOperand(1));
5322 Ops.push_back(N.getOperand(0));
5323 break;
5324 case X86ISD::PALIGNR:
5325 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5326 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5327 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5328 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5329 DecodePALIGNRMask(NumElems, ImmN, Mask);
5330 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5331 Ops.push_back(N.getOperand(1));
5332 Ops.push_back(N.getOperand(0));
5333 break;
5334 case X86ISD::VSHLDQ:
5335 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5336 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5337 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5338 DecodePSLLDQMask(NumElems, ImmN, Mask);
5339 IsUnary = true;
5340 break;
5341 case X86ISD::VSRLDQ:
5342 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5343 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5344 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5345 DecodePSRLDQMask(NumElems, ImmN, Mask);
5346 IsUnary = true;
5347 break;
5348 case X86ISD::PSHUFD:
5349 case X86ISD::VPERMILPI:
5350 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5351 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5352 DecodePSHUFMask(NumElems, MaskEltSize, ImmN, Mask);
5353 IsUnary = true;
5354 break;
5355 case X86ISD::PSHUFHW:
5356 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5357 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5358 DecodePSHUFHWMask(NumElems, ImmN, Mask);
5359 IsUnary = true;
5360 break;
5361 case X86ISD::PSHUFLW:
5362 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5363 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5364 DecodePSHUFLWMask(NumElems, ImmN, Mask);
5365 IsUnary = true;
5366 break;
5367 case X86ISD::VZEXT_MOVL:
5368 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5369 DecodeZeroMoveLowMask(NumElems, Mask);
5370 IsUnary = true;
5371 break;
5372 case X86ISD::VBROADCAST:
5373 // We only decode broadcasts of same-sized vectors, peeking through to
5374 // extracted subvectors is likely to cause hasOneUse issues with
5375 // SimplifyDemandedBits etc.
5376 if (N.getOperand(0).getValueType() == VT) {
5377 DecodeVectorBroadcast(NumElems, Mask);
5378 IsUnary = true;
5379 break;
5380 }
5381 return false;
5382 case X86ISD::VPERMILPV: {
5383 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5384 IsUnary = true;
5385 SDValue MaskNode = N.getOperand(1);
5386 if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5387 RawUndefs)) {
5388 DecodeVPERMILPMask(NumElems, MaskEltSize, RawMask, RawUndefs, Mask);
5389 break;
5390 }
5391 return false;
5392 }
5393 case X86ISD::PSHUFB: {
5394 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");
5395 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5396 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5397 IsUnary = true;
5398 SDValue MaskNode = N.getOperand(1);
5399 if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
5400 DecodePSHUFBMask(RawMask, RawUndefs, Mask);
5401 break;
5402 }
5403 return false;
5404 }
5405 case X86ISD::VPERMI:
5406 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5407 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5408 DecodeVPERMMask(NumElems, ImmN, Mask);
5409 IsUnary = true;
5410 break;
5411 case X86ISD::MOVSS:
5412 case X86ISD::MOVSD:
5413 case X86ISD::MOVSH:
5414 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5415 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5416 DecodeScalarMoveMask(NumElems, /* IsLoad */ false, Mask);
5417 break;
5418 case X86ISD::VPERM2X128:
5419 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5420 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5421 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5422 DecodeVPERM2X128Mask(NumElems, ImmN, Mask);
5423 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5424 break;
5425 case X86ISD::SHUF128:
5426 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5427 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5428 ImmN = N.getConstantOperandVal(N.getNumOperands() - 1);
5429 decodeVSHUF64x2FamilyMask(NumElems, MaskEltSize, ImmN, Mask);
5430 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5431 break;
5432 case X86ISD::MOVSLDUP:
5433 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5434 DecodeMOVSLDUPMask(NumElems, Mask);
5435 IsUnary = true;
5436 break;
5437 case X86ISD::MOVSHDUP:
5438 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5439 DecodeMOVSHDUPMask(NumElems, Mask);
5440 IsUnary = true;
5441 break;
5442 case X86ISD::MOVDDUP:
5443 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5444 DecodeMOVDDUPMask(NumElems, Mask);
5445 IsUnary = true;
5446 break;
5447 case X86ISD::VPERMIL2: {
5448 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5449 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5450 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5451 SDValue MaskNode = N.getOperand(2);
5452 SDValue CtrlNode = N.getOperand(3);
5453 if (ConstantSDNode *CtrlOp = dyn_cast<ConstantSDNode>(CtrlNode)) {
5454 unsigned CtrlImm = CtrlOp->getZExtValue();
5455 if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5456 RawUndefs)) {
5457 DecodeVPERMIL2PMask(NumElems, MaskEltSize, CtrlImm, RawMask, RawUndefs,
5458 Mask);
5459 break;
5460 }
5461 }
5462 return false;
5463 }
5464 case X86ISD::VPPERM: {
5465 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5466 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5467 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(1);
5468 SDValue MaskNode = N.getOperand(2);
5469 if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
5470 DecodeVPPERMMask(RawMask, RawUndefs, Mask);
5471 break;
5472 }
5473 return false;
5474 }
5475 case X86ISD::VPERMV: {
5476 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type");
5477 IsUnary = true;
5478 // Unlike most shuffle nodes, VPERMV's mask operand is operand 0.
5479 Ops.push_back(N.getOperand(1));
5480 SDValue MaskNode = N.getOperand(0);
5481 if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5482 RawUndefs)) {
5483 DecodeVPERMVMask(RawMask, RawUndefs, Mask);
5484 break;
5485 }
5486 return false;
5487 }
5488 case X86ISD::VPERMV3: {
5489 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type");
5490 assert(N.getOperand(2).getValueType() == VT && "Unexpected value type");
5491 IsUnary = IsFakeUnary = N.getOperand(0) == N.getOperand(2);
5492 // Unlike most shuffle nodes, VPERMV3's mask operand is the middle one.
5493 Ops.push_back(N.getOperand(0));
5494 Ops.push_back(N.getOperand(2));
5495 SDValue MaskNode = N.getOperand(1);
5496 if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
5497 RawUndefs)) {
5498 DecodeVPERMV3Mask(RawMask, RawUndefs, Mask);
5499 break;
5500 }
5501 return false;
5502 }
5503 default:
5504 llvm_unreachable("unknown target shuffle node");
5505 }
5506
5507 // Empty mask indicates the decode failed.
5508 if (Mask.empty())
5509 return false;
5510
5511 // Check if we're getting a shuffle mask with zero'd elements.
5512 if (!AllowSentinelZero && isAnyZero(Mask))
5513 return false;
5514
5515 // If we have a fake unary shuffle, the shuffle mask is spread across two
5516 // inputs that are actually the same node. Re-map the mask to always point
5517 // into the first input.
5518 if (IsFakeUnary)
5519 for (int &M : Mask)
5520 if (M >= (int)Mask.size())
5521 M -= Mask.size();
5522
5523 // If we didn't already add operands in the opcode-specific code, default to
5524 // adding 1 or 2 operands starting at 0.
5525 if (Ops.empty()) {
5526 Ops.push_back(N.getOperand(0));
5527 if (!IsUnary || IsFakeUnary)
5528 Ops.push_back(N.getOperand(1));
5529 }
5530
5531 return true;
5532}
5533
5534// Wrapper for getTargetShuffleMask with InUnary;
5535static bool getTargetShuffleMask(SDValue N, bool AllowSentinelZero,
5536 SmallVectorImpl<SDValue> &Ops,
5537 SmallVectorImpl<int> &Mask) {
5538 bool IsUnary;
5539 return getTargetShuffleMask(N, AllowSentinelZero, Ops, Mask, IsUnary);
5540}
5541
5542/// Compute whether each element of a shuffle is zeroable.
5543///
5544/// A "zeroable" vector shuffle element is one which can be lowered to zero.
5545/// Either it is an undef element in the shuffle mask, the element of the input
5546/// referenced is undef, or the element of the input referenced is known to be
5547/// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
5548/// as many lanes with this technique as possible to simplify the remaining
5549/// shuffle.
5550static void computeZeroableShuffleElements(ArrayRef<int> Mask,
5551 SDValue V1, SDValue V2,
5552 APInt &KnownUndef, APInt &KnownZero) {
5553 int Size = Mask.size();
5554 KnownUndef = KnownZero = APInt::getZero(Size);
5555
5556 V1 = peekThroughBitcasts(V1);
5557 V2 = peekThroughBitcasts(V2);
5558
5559 bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
5560 bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
5561
5562 int VectorSizeInBits = V1.getValueSizeInBits();
5563 int ScalarSizeInBits = VectorSizeInBits / Size;
5564 assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size");
5565
5566 for (int i = 0; i < Size; ++i) {
5567 int M = Mask[i];
5568 // Handle the easy cases.
5569 if (M < 0) {
5570 KnownUndef.setBit(i);
5571 continue;
5572 }
5573 if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
5574 KnownZero.setBit(i);
5575 continue;
5576 }
5577
5578 // Determine shuffle input and normalize the mask.
5579 SDValue V = M < Size ? V1 : V2;
5580 M %= Size;
5581
5582 // Currently we can only search BUILD_VECTOR for UNDEF/ZERO elements.
5583 if (V.getOpcode() != ISD::BUILD_VECTOR)
5584 continue;
5585
5586 // If the BUILD_VECTOR has fewer elements then the bitcasted portion of
5587 // the (larger) source element must be UNDEF/ZERO.
5588 if ((Size % V.getNumOperands()) == 0) {
5589 int Scale = Size / V->getNumOperands();
5590 SDValue Op = V.getOperand(M / Scale);
5591 if (Op.isUndef())
5592 KnownUndef.setBit(i);
5593 if (X86::isZeroNode(Op))
5594 KnownZero.setBit(i);
5595 else if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) {
5596 APInt Val = Cst->getAPIntValue();
5597 Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
5598 if (Val == 0)
5599 KnownZero.setBit(i);
5600 } else if (ConstantFPSDNode *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
5601 APInt Val = Cst->getValueAPF().bitcastToAPInt();
5602 Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
5603 if (Val == 0)
5604 KnownZero.setBit(i);
5605 }
5606 continue;
5607 }
5608
5609 // If the BUILD_VECTOR has more elements then all the (smaller) source
5610 // elements must be UNDEF or ZERO.
5611 if ((V.getNumOperands() % Size) == 0) {
5612 int Scale = V->getNumOperands() / Size;
5613 bool AllUndef = true;
5614 bool AllZero = true;
5615 for (int j = 0; j < Scale; ++j) {
5616 SDValue Op = V.getOperand((M * Scale) + j);
5617 AllUndef &= Op.isUndef();
5618 AllZero &= X86::isZeroNode(Op);
5619 }
5620 if (AllUndef)
5621 KnownUndef.setBit(i);
5622 if (AllZero)
5623 KnownZero.setBit(i);
5624 continue;
5625 }
5626 }
5627}
5628
5629/// Decode a target shuffle mask and inputs and see if any values are
5630/// known to be undef or zero from their inputs.
5631/// Returns true if the target shuffle mask was decoded.
5632/// FIXME: Merge this with computeZeroableShuffleElements?
5633static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
5634 SmallVectorImpl<SDValue> &Ops,
5635 APInt &KnownUndef, APInt &KnownZero) {
5636 bool IsUnary;
5637 if (!isTargetShuffle(N.getOpcode()))
5638 return false;
5639
5640 MVT VT = N.getSimpleValueType();
5641 if (!getTargetShuffleMask(N, true, Ops, Mask, IsUnary))
5642 return false;
5643
5644 int Size = Mask.size();
5645 SDValue V1 = Ops[0];
5646 SDValue V2 = IsUnary ? V1 : Ops[1];
5647 KnownUndef = KnownZero = APInt::getZero(Size);
5648
5649 V1 = peekThroughBitcasts(V1);
5650 V2 = peekThroughBitcasts(V2);
5651
5652 assert((VT.getSizeInBits() % Size) == 0 &&
5653 "Illegal split of shuffle value type");
5654 unsigned EltSizeInBits = VT.getSizeInBits() / Size;
5655
5656 // Extract known constant input data.
5657 APInt UndefSrcElts[2];
5658 SmallVector<APInt, 32> SrcEltBits[2];
5659 bool IsSrcConstant[2] = {
5660 getTargetConstantBitsFromNode(V1, EltSizeInBits, UndefSrcElts[0],
5661 SrcEltBits[0], /*AllowWholeUndefs*/ true,
5662 /*AllowPartialUndefs*/ false),
5663 getTargetConstantBitsFromNode(V2, EltSizeInBits, UndefSrcElts[1],
5664 SrcEltBits[1], /*AllowWholeUndefs*/ true,
5665 /*AllowPartialUndefs*/ false)};
5666
5667 for (int i = 0; i < Size; ++i) {
5668 int M = Mask[i];
5669
5670 // Already decoded as SM_SentinelZero / SM_SentinelUndef.
5671 if (M < 0) {
5672 assert(isUndefOrZero(M) && "Unknown shuffle sentinel value!");
5673 if (SM_SentinelUndef == M)
5674 KnownUndef.setBit(i);
5675 if (SM_SentinelZero == M)
5676 KnownZero.setBit(i);
5677 continue;
5678 }
5679
5680 // Determine shuffle input and normalize the mask.
5681 unsigned SrcIdx = M / Size;
5682 SDValue V = M < Size ? V1 : V2;
5683 M %= Size;
5684
5685 // We are referencing an UNDEF input.
5686 if (V.isUndef()) {
5687 KnownUndef.setBit(i);
5688 continue;
5689 }
5690
5691 // SCALAR_TO_VECTOR - only the first element is defined, and the rest UNDEF.
5692 // TODO: We currently only set UNDEF for integer types - floats use the same
5693 // registers as vectors and many of the scalar folded loads rely on the
5694 // SCALAR_TO_VECTOR pattern.
5695 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR &&
5696 (Size % V.getValueType().getVectorNumElements()) == 0) {
5697 int Scale = Size / V.getValueType().getVectorNumElements();
5698 int Idx = M / Scale;
5699 if (Idx != 0 && !VT.isFloatingPoint())
5700 KnownUndef.setBit(i);
5701 else if (Idx == 0 && X86::isZeroNode(V.getOperand(0)))
5702 KnownZero.setBit(i);
5703 continue;
5704 }
5705
5706 // INSERT_SUBVECTOR - to widen vectors we often insert them into UNDEF
5707 // base vectors.
5708 if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
5709 SDValue Vec = V.getOperand(0);
5710 int NumVecElts = Vec.getValueType().getVectorNumElements();
5711 if (Vec.isUndef() && Size == NumVecElts) {
5712 int Idx = V.getConstantOperandVal(2);
5713 int NumSubElts = V.getOperand(1).getValueType().getVectorNumElements();
5714 if (M < Idx || (Idx + NumSubElts) <= M)
5715 KnownUndef.setBit(i);
5716 }
5717 continue;
5718 }
5719
5720 // Attempt to extract from the source's constant bits.
5721 if (IsSrcConstant[SrcIdx]) {
5722 if (UndefSrcElts[SrcIdx][M])
5723 KnownUndef.setBit(i);
5724 else if (SrcEltBits[SrcIdx][M] == 0)
5725 KnownZero.setBit(i);
5726 }
5727 }
5728
5729 assert(VT.getVectorNumElements() == (unsigned)Size &&
5730 "Different mask size from vector size!");
5731 return true;
5732}
5733
5734// Replace target shuffle mask elements with known undef/zero sentinels.
5735static void resolveTargetShuffleFromZeroables(SmallVectorImpl<int> &Mask,
5736 const APInt &KnownUndef,
5737 const APInt &KnownZero,
5738 bool ResolveKnownZeros= true) {
5739 unsigned NumElts = Mask.size();
5740 assert(KnownUndef.getBitWidth() == NumElts &&
5741 KnownZero.getBitWidth() == NumElts && "Shuffle mask size mismatch");
5742
5743 for (unsigned i = 0; i != NumElts; ++i) {
5744 if (KnownUndef[i])
5745 Mask[i] = SM_SentinelUndef;
5746 else if (ResolveKnownZeros && KnownZero[i])
5747 Mask[i] = SM_SentinelZero;
5748 }
5749}
5750
5751// Extract target shuffle mask sentinel elements to known undef/zero bitmasks.
5752static void resolveZeroablesFromTargetShuffle(const SmallVectorImpl<int> &Mask,
5753 APInt &KnownUndef,
5754 APInt &KnownZero) {
5755 unsigned NumElts = Mask.size();
5756 KnownUndef = KnownZero = APInt::getZero(NumElts);
5757
5758 for (unsigned i = 0; i != NumElts; ++i) {
5759 int M = Mask[i];
5760 if (SM_SentinelUndef == M)
5761 KnownUndef.setBit(i);
5762 if (SM_SentinelZero == M)
5763 KnownZero.setBit(i);
5764 }
5765}
5766
5767// Attempt to create a shuffle mask from a VSELECT/BLENDV condition mask.
5768static bool createShuffleMaskFromVSELECT(SmallVectorImpl<int> &Mask,
5769 SDValue Cond, bool IsBLENDV = false) {
5770 EVT CondVT = Cond.getValueType();
5771 unsigned EltSizeInBits = CondVT.getScalarSizeInBits();
5772 unsigned NumElts = CondVT.getVectorNumElements();
5773
5774 APInt UndefElts;
5775 SmallVector<APInt, 32> EltBits;
5776 if (!getTargetConstantBitsFromNode(Cond, EltSizeInBits, UndefElts, EltBits,
5777 /*AllowWholeUndefs*/ true,
5778 /*AllowPartialUndefs*/ false))
5779 return false;
5780
5781 Mask.resize(NumElts, SM_SentinelUndef);
5782
5783 for (int i = 0; i != (int)NumElts; ++i) {
5784 Mask[i] = i;
5785 // Arbitrarily choose from the 2nd operand if the select condition element
5786 // is undef.
5787 // TODO: Can we do better by matching patterns such as even/odd?
5788 if (UndefElts[i] || (!IsBLENDV && EltBits[i].isZero()) ||
5789 (IsBLENDV && EltBits[i].isNonNegative()))
5790 Mask[i] += NumElts;
5791 }
5792
5793 return true;
5794}
5795
5796// Forward declaration (for getFauxShuffleMask recursive check).
5797static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
5798 SmallVectorImpl<SDValue> &Inputs,
5799 SmallVectorImpl<int> &Mask,
5800 const SelectionDAG &DAG, unsigned Depth,
5801 bool ResolveKnownElts);
5802
5803// Attempt to decode ops that could be represented as a shuffle mask.
5804// The decoded shuffle mask may contain a different number of elements to the
5805// destination value type.
5806// TODO: Merge into getTargetShuffleInputs()
5807static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
5808 SmallVectorImpl<int> &Mask,
5809 SmallVectorImpl<SDValue> &Ops,
5810 const SelectionDAG &DAG, unsigned Depth,
5811 bool ResolveKnownElts) {
5812 Mask.clear();
5813 Ops.clear();
5814
5815 MVT VT = N.getSimpleValueType();
5816 unsigned NumElts = VT.getVectorNumElements();
5817 unsigned NumSizeInBits = VT.getSizeInBits();
5818 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
5819 if ((NumBitsPerElt % 8) != 0 || (NumSizeInBits % 8) != 0)
5820 return false;
5821 assert(NumElts == DemandedElts.getBitWidth() && "Unexpected vector size");
5822 unsigned NumSizeInBytes = NumSizeInBits / 8;
5823 unsigned NumBytesPerElt = NumBitsPerElt / 8;
5824
5825 unsigned Opcode = N.getOpcode();
5826 switch (Opcode) {
5827 case ISD::VECTOR_SHUFFLE: {
5828 // Don't treat ISD::VECTOR_SHUFFLE as a target shuffle so decode it here.
5829 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(N)->getMask();
5830 if (isUndefOrInRange(ShuffleMask, 0, 2 * NumElts)) {
5831 Mask.append(ShuffleMask.begin(), ShuffleMask.end());
5832 Ops.push_back(N.getOperand(0));
5833 Ops.push_back(N.getOperand(1));
5834 return true;
5835 }
5836 return false;
5837 }
5838 case ISD::AND:
5839 case X86ISD::ANDNP: {
5840 // Attempt to decode as a per-byte mask.
5841 APInt UndefElts;
5842 SmallVector<APInt, 32> EltBits;
5843 SDValue N0 = N.getOperand(0);
5844 SDValue N1 = N.getOperand(1);
5845 bool IsAndN = (X86ISD::ANDNP == Opcode);
5846 uint64_t ZeroMask = IsAndN ? 255 : 0;
5847 if (!getTargetConstantBitsFromNode(IsAndN ? N0 : N1, 8, UndefElts, EltBits,
5848 /*AllowWholeUndefs*/ false,
5849 /*AllowPartialUndefs*/ false))
5850 return false;
5851 // We can't assume an undef src element gives an undef dst - the other src
5852 // might be zero.
5853 assert(UndefElts.isZero() && "Unexpected UNDEF element in AND/ANDNP mask");
5854 for (int i = 0, e = (int)EltBits.size(); i != e; ++i) {
5855 const APInt &ByteBits = EltBits[i];
5856 if (ByteBits != 0 && ByteBits != 255)
5857 return false;
5858 Mask.push_back(ByteBits == ZeroMask ? SM_SentinelZero : i);
5859 }
5860 Ops.push_back(IsAndN ? N1 : N0);
5861 return true;
5862 }
5863 case ISD::OR: {
5864 // Handle OR(SHUFFLE,SHUFFLE) case where one source is zero and the other
5865 // is a valid shuffle index.
5866 SDValue N0 = peekThroughBitcasts(N.getOperand(0));
5867 SDValue N1 = peekThroughBitcasts(N.getOperand(1));
5868 if (!N0.getValueType().isVector() || !N1.getValueType().isVector())
5869 return false;
5870
5871 SmallVector<int, 64> SrcMask0, SrcMask1;
5872 SmallVector<SDValue, 2> SrcInputs0, SrcInputs1;
5873 APInt Demand0 = APInt::getAllOnes(N0.getValueType().getVectorNumElements());
5874 APInt Demand1 = APInt::getAllOnes(N1.getValueType().getVectorNumElements());
5875 if (!getTargetShuffleInputs(N0, Demand0, SrcInputs0, SrcMask0, DAG,
5876 Depth + 1, true) ||
5877 !getTargetShuffleInputs(N1, Demand1, SrcInputs1, SrcMask1, DAG,
5878 Depth + 1, true))
5879 return false;
5880
5881 size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size());
5882 SmallVector<int, 64> Mask0, Mask1;
5883 narrowShuffleMaskElts(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
5884 narrowShuffleMaskElts(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
5885 for (int i = 0; i != (int)MaskSize; ++i) {
5886 // NOTE: Don't handle SM_SentinelUndef, as we can end up in infinite
5887 // loops converting between OR and BLEND shuffles due to
5888 // canWidenShuffleElements merging away undef elements, meaning we
5889 // fail to recognise the OR as the undef element isn't known zero.
5890 if (Mask0[i] == SM_SentinelZero && Mask1[i] == SM_SentinelZero)
5891 Mask.push_back(SM_SentinelZero);
5892 else if (Mask1[i] == SM_SentinelZero)
5893 Mask.push_back(i);
5894 else if (Mask0[i] == SM_SentinelZero)
5895 Mask.push_back(i + MaskSize);
5896 else
5897 return false;
5898 }
5899 Ops.push_back(N0);
5900 Ops.push_back(N1);
5901 return true;
5902 }
5903 case ISD::INSERT_SUBVECTOR: {
5904 SDValue Src = N.getOperand(0);
5905 SDValue Sub = N.getOperand(1);
5906 EVT SubVT = Sub.getValueType();
5907 unsigned NumSubElts = SubVT.getVectorNumElements();
5908 if (!N->isOnlyUserOf(Sub.getNode()))
5909 return false;
5910 SDValue SubBC = peekThroughBitcasts(Sub);
5911 uint64_t InsertIdx = N.getConstantOperandVal(2);
5912 // Handle INSERT_SUBVECTOR(SRC0, EXTRACT_SUBVECTOR(SRC1)).
5913 if (SubBC.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5914 SubBC.getOperand(0).getValueSizeInBits() == NumSizeInBits) {
5915 uint64_t ExtractIdx = SubBC.getConstantOperandVal(1);
5916 SDValue SubBCSrc = SubBC.getOperand(0);
5917 unsigned NumSubSrcBCElts = SubBCSrc.getValueType().getVectorNumElements();
5918 unsigned MaxElts = std::max(NumElts, NumSubSrcBCElts);
5919 assert((MaxElts % NumElts) == 0 && (MaxElts % NumSubSrcBCElts) == 0 &&
5920 "Subvector valuetype mismatch");
5921 InsertIdx *= (MaxElts / NumElts);
5922 ExtractIdx *= (MaxElts / NumSubSrcBCElts);
5923 NumSubElts *= (MaxElts / NumElts);
5924 bool SrcIsUndef = Src.isUndef();
5925 for (int i = 0; i != (int)MaxElts; ++i)
5926 Mask.push_back(SrcIsUndef ? SM_SentinelUndef : i);
5927 for (int i = 0; i != (int)NumSubElts; ++i)
5928 Mask[InsertIdx + i] = (SrcIsUndef ? 0 : MaxElts) + ExtractIdx + i;
5929 if (!SrcIsUndef)
5930 Ops.push_back(Src);
5931 Ops.push_back(SubBCSrc);
5932 return true;
5933 }
5934 // Handle CONCAT(SUB0, SUB1).
5935 // Limit this to vXi64 512-bit vector cases to make the most of AVX512
5936 // cross lane shuffles.
5937 if (Depth > 0 && InsertIdx == NumSubElts && NumElts == (2 * NumSubElts) &&
5938 NumBitsPerElt == 64 && NumSizeInBits == 512 &&
5939 Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
5940 Src.getOperand(0).isUndef() &&
5941 Src.getOperand(1).getValueType() == SubVT &&
5942 Src.getConstantOperandVal(2) == 0) {
5943 for (int i = 0; i != (int)NumSubElts; ++i)
5944 Mask.push_back(i);
5945 for (int i = 0; i != (int)NumSubElts; ++i)
5946 Mask.push_back(i + NumElts);
5947 Ops.push_back(Src.getOperand(1));
5948 Ops.push_back(Sub);
5949 return true;
5950 }
5951 // Handle INSERT_SUBVECTOR(SRC0, SHUFFLE(SRC1)).
5952 SmallVector<int, 64> SubMask;
5953 SmallVector<SDValue, 2> SubInputs;
5954 SDValue SubSrc = peekThroughOneUseBitcasts(Sub);
5955 EVT SubSrcVT = SubSrc.getValueType();
5956 if (!SubSrcVT.isVector())
5957 return false;
5958
5959 APInt SubDemand = APInt::getAllOnes(SubSrcVT.getVectorNumElements());
5960 if (!getTargetShuffleInputs(SubSrc, SubDemand, SubInputs, SubMask, DAG,
5961 Depth + 1, ResolveKnownElts))
5962 return false;
5963
5964 // Subvector shuffle inputs must not be larger than the subvector.
5965 if (llvm::any_of(SubInputs, [SubVT](SDValue SubInput) {
5966 return SubVT.getFixedSizeInBits() <
5967 SubInput.getValueSizeInBits().getFixedValue();
5968 }))
5969 return false;
5970
5971 if (SubMask.size() != NumSubElts) {
5972 assert(((SubMask.size() % NumSubElts) == 0 ||
5973 (NumSubElts % SubMask.size()) == 0) && "Illegal submask scale");
5974 if ((NumSubElts % SubMask.size()) == 0) {
5975 int Scale = NumSubElts / SubMask.size();
5976 SmallVector<int,64> ScaledSubMask;
5977 narrowShuffleMaskElts(Scale, SubMask, ScaledSubMask);
5978 SubMask = ScaledSubMask;
5979 } else {
5980 int Scale = SubMask.size() / NumSubElts;
5981 NumSubElts = SubMask.size();
5982 NumElts *= Scale;
5983 InsertIdx *= Scale;
5984 }
5985 }
5986 Ops.push_back(Src);
5987 Ops.append(SubInputs.begin(), SubInputs.end());
5988 if (ISD::isBuildVectorAllZeros(Src.getNode()))
5989 Mask.append(NumElts, SM_SentinelZero);
5990 else
5991 for (int i = 0; i != (int)NumElts; ++i)
5992 Mask.push_back(i);
5993 for (int i = 0; i != (int)NumSubElts; ++i) {
5994 int M = SubMask[i];
5995 if (0 <= M) {
5996 int InputIdx = M / NumSubElts;
5997 M = (NumElts * (1 + InputIdx)) + (M % NumSubElts);
5998 }
5999 Mask[i + InsertIdx] = M;
6000 }
6001 return true;
6002 }
6003 case X86ISD::PINSRB:
6004 case X86ISD::PINSRW:
6005 case ISD::SCALAR_TO_VECTOR:
6006 case ISD::INSERT_VECTOR_ELT: {
6007 // Match against a insert_vector_elt/scalar_to_vector of an extract from a
6008 // vector, for matching src/dst vector types.
6009 SDValue Scl = N.getOperand(Opcode == ISD::SCALAR_TO_VECTOR ? 0 : 1);
6010
6011 unsigned DstIdx = 0;
6012 if (Opcode != ISD::SCALAR_TO_VECTOR) {
6013 // Check we have an in-range constant insertion index.
6014 if (!isa<ConstantSDNode>(N.getOperand(2)) ||
6015 N.getConstantOperandAPInt(2).uge(NumElts))
6016 return false;
6017 DstIdx = N.getConstantOperandVal(2);
6018
6019 // Attempt to recognise an INSERT*(VEC, 0, DstIdx) shuffle pattern.
6020 if (X86::isZeroNode(Scl)) {
6021 Ops.push_back(N.getOperand(0));
6022 for (unsigned i = 0; i != NumElts; ++i)
6023 Mask.push_back(i == DstIdx ? SM_SentinelZero : (int)i);
6024 return true;
6025 }
6026 }
6027
6028 // Peek through trunc/aext/zext/bitcast.
6029 // TODO: aext shouldn't require SM_SentinelZero padding.
6030 // TODO: handle shift of scalars.
6031 unsigned MinBitsPerElt = Scl.getScalarValueSizeInBits();
6032 while (Scl.getOpcode() == ISD::TRUNCATE ||
6033 Scl.getOpcode() == ISD::ANY_EXTEND ||
6034 Scl.getOpcode() == ISD::ZERO_EXTEND ||
6035 (Scl.getOpcode() == ISD::BITCAST &&
6036 Scl.getScalarValueSizeInBits() ==
6037 Scl.getOperand(0).getScalarValueSizeInBits())) {
6038 Scl = Scl.getOperand(0);
6039 MinBitsPerElt =
6040 std::min<unsigned>(MinBitsPerElt, Scl.getScalarValueSizeInBits());
6041 }
6042 if ((MinBitsPerElt % 8) != 0)
6043 return false;
6044
6045 // Attempt to find the source vector the scalar was extracted from.
6046 SDValue SrcExtract;
6047 if ((Scl.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
6048 Scl.getOpcode() == X86ISD::PEXTRW ||
6049 Scl.getOpcode() == X86ISD::PEXTRB) &&
6050 Scl.getOperand(0).getValueSizeInBits() == NumSizeInBits) {
6051 SrcExtract = Scl;
6052 }
6053 if (!SrcExtract || !isa<ConstantSDNode>(SrcExtract.getOperand(1)))
6054 return false;
6055
6056 SDValue SrcVec = SrcExtract.getOperand(0);
6057 EVT SrcVT = SrcVec.getValueType();
6058 if (!SrcVT.getScalarType().isByteSized())
6059 return false;
6060 unsigned SrcIdx = SrcExtract.getConstantOperandVal(1);
6061 unsigned SrcByte = SrcIdx * (SrcVT.getScalarSizeInBits() / 8);
6062 unsigned DstByte = DstIdx * NumBytesPerElt;
6063 MinBitsPerElt =
6064 std::min<unsigned>(MinBitsPerElt, SrcVT.getScalarSizeInBits());
6065
6066 // Create 'identity' byte level shuffle mask and then add inserted bytes.
6067 if (Opcode == ISD::SCALAR_TO_VECTOR) {
6068 Ops.push_back(SrcVec);
6069 Mask.append(NumSizeInBytes, SM_SentinelUndef);
6070 } else {
6071 Ops.push_back(SrcVec);
6072 Ops.push_back(N.getOperand(0));
6073 for (int i = 0; i != (int)NumSizeInBytes; ++i)
6074 Mask.push_back(NumSizeInBytes + i);
6075 }
6076
6077 unsigned MinBytesPerElts = MinBitsPerElt / 8;
6078 MinBytesPerElts = std::min(MinBytesPerElts, NumBytesPerElt);
6079 for (unsigned i = 0; i != MinBytesPerElts; ++i)
6080 Mask[DstByte + i] = SrcByte + i;
6081 for (unsigned i = MinBytesPerElts; i < NumBytesPerElt; ++i)
6082 Mask[DstByte + i] = SM_SentinelZero;
6083 return true;
6084 }
6085 case X86ISD::PACKSS:
6086 case X86ISD::PACKUS: {
6087 SDValue N0 = N.getOperand(0);
6088 SDValue N1 = N.getOperand(1);
6089 assert(N0.getValueType().getVectorNumElements() == (NumElts / 2) &&
6090 N1.getValueType().getVectorNumElements() == (NumElts / 2) &&
6091 "Unexpected input value type");
6092
6093 APInt EltsLHS, EltsRHS;
6094 getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS);
6095
6096 // If we know input saturation won't happen (or we don't care for particular
6097 // lanes), we can treat this as a truncation shuffle.
6098 bool Offset0 = false, Offset1 = false;
6099 if (Opcode == X86ISD::PACKSS) {
6100 if ((!(N0.isUndef() || EltsLHS.isZero()) &&
6101 DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) ||
6102 (!(N1.isUndef() || EltsRHS.isZero()) &&
6103 DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt))
6104 return false;
6105 // We can't easily fold ASHR into a shuffle, but if it was feeding a
6106 // PACKSS then it was likely being used for sign-extension for a
6107 // truncation, so just peek through and adjust the mask accordingly.
6108 if (N0.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N0.getNode()) &&
6109 N0.getConstantOperandAPInt(1) == NumBitsPerElt) {
6110 Offset0 = true;
6111 N0 = N0.getOperand(0);
6112 }
6113 if (N1.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N1.getNode()) &&
6114 N1.getConstantOperandAPInt(1) == NumBitsPerElt) {
6115 Offset1 = true;
6116 N1 = N1.getOperand(0);
6117 }
6118 } else {
6119 APInt ZeroMask = APInt::getHighBitsSet(2 * NumBitsPerElt, NumBitsPerElt);
6120 if ((!(N0.isUndef() || EltsLHS.isZero()) &&
6121 !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) ||
6122 (!(N1.isUndef() || EltsRHS.isZero()) &&
6123 !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1)))
6124 return false;
6125 }
6126
6127 bool IsUnary = (N0 == N1);
6128
6129 Ops.push_back(N0);
6130 if (!IsUnary)
6131 Ops.push_back(N1);
6132
6133 createPackShuffleMask(VT, Mask, IsUnary);
6134
6135 if (Offset0 || Offset1) {
6136 for (int &M : Mask)
6137 if ((Offset0 && isInRange(M, 0, NumElts)) ||
6138 (Offset1 && isInRange(M, NumElts, 2 * NumElts)))
6139 ++M;
6140 }
6141 return true;
6142 }
6143 case ISD::VSELECT:
6144 case X86ISD::BLENDV: {
6145 SDValue Cond = N.getOperand(0);
6146 if (createShuffleMaskFromVSELECT(Mask, Cond, Opcode == X86ISD::BLENDV)) {
6147 Ops.push_back(N.getOperand(1));
6148 Ops.push_back(N.getOperand(2));
6149 return true;
6150 }
6151 return false;
6152 }
6153 case X86ISD::VTRUNC: {
6154 SDValue Src = N.getOperand(0);
6155 EVT SrcVT = Src.getValueType();
6156 // Truncated source must be a simple vector.
6157 if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
6158 (SrcVT.getScalarSizeInBits() % 8) != 0)
6159 return false;
6160 unsigned NumSrcElts = SrcVT.getVectorNumElements();
6161 unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits();
6162 unsigned Scale = NumBitsPerSrcElt / NumBitsPerElt;
6163 assert((NumBitsPerSrcElt % NumBitsPerElt) == 0 && "Illegal truncation");
6164 for (unsigned i = 0; i != NumSrcElts; ++i)
6165 Mask.push_back(i * Scale);
6166 Mask.append(NumElts - NumSrcElts, SM_SentinelZero);
6167 Ops.push_back(Src);
6168 return true;
6169 }
6170 case X86ISD::VSHLI:
6171 case X86ISD::VSRLI: {
6172 uint64_t ShiftVal = N.getConstantOperandVal(1);
6173 // Out of range bit shifts are guaranteed to be zero.
6174 if (NumBitsPerElt <= ShiftVal) {
6175 Mask.append(NumElts, SM_SentinelZero);
6176 return true;
6177 }
6178
6179 // We can only decode 'whole byte' bit shifts as shuffles.
6180 if ((ShiftVal % 8) != 0)
6181 break;
6182
6183 uint64_t ByteShift = ShiftVal / 8;
6184 Ops.push_back(N.getOperand(0));
6185
6186 // Clear mask to all zeros and insert the shifted byte indices.
6187 Mask.append(NumSizeInBytes, SM_SentinelZero);
6188
6189 if (X86ISD::VSHLI == Opcode) {
6190 for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt)
6191 for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
6192 Mask[i + j] = i + j - ByteShift;
6193 } else {
6194 for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt)
6195 for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
6196 Mask[i + j - ByteShift] = i + j;
6197 }
6198 return true;
6199 }
6200 case X86ISD::VROTLI:
6201 case X86ISD::VROTRI: {
6202 // We can only decode 'whole byte' bit rotates as shuffles.
6203 uint64_t RotateVal = N.getConstantOperandAPInt(1).urem(NumBitsPerElt);
6204 if ((RotateVal % 8) != 0)
6205 return false;
6206 Ops.push_back(N.getOperand(0));
6207 int Offset = RotateVal / 8;
6208 Offset = (X86ISD::VROTLI == Opcode ? NumBytesPerElt - Offset : Offset);
6209 for (int i = 0; i != (int)NumElts; ++i) {
6210 int BaseIdx = i * NumBytesPerElt;
6211 for (int j = 0; j != (int)NumBytesPerElt; ++j) {
6212 Mask.push_back(BaseIdx + ((Offset + j) % NumBytesPerElt));
6213 }
6214 }
6215 return true;
6216 }
6217 case X86ISD::VBROADCAST: {
6218 SDValue Src = N.getOperand(0);
6219 if (!Src.getSimpleValueType().isVector()) {
6220 if (Src.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6221 !isNullConstant(Src.getOperand(1)) ||
6222 Src.getOperand(0).getValueType().getScalarType() !=
6223 VT.getScalarType())
6224 return false;
6225 Src = Src.getOperand(0);
6226 }
6227 Ops.push_back(Src);
6228 Mask.append(NumElts, 0);
6229 return true;
6230 }
6231 case ISD::SIGN_EXTEND_VECTOR_INREG: {
6232 SDValue Src = N.getOperand(0);
6233 EVT SrcVT = Src.getValueType();
6234 unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits();
6235
6236 // Extended source must be a simple vector.
6237 if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
6238 (NumBitsPerSrcElt % 8) != 0)
6239 return false;
6240
6241 // We can only handle all-signbits extensions.
6242 APInt DemandedSrcElts =
6243 DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
6244 if (DAG.ComputeNumSignBits(Src, DemandedSrcElts) != NumBitsPerSrcElt)
6245 return false;
6246
6247 assert((NumBitsPerElt % NumBitsPerSrcElt) == 0 && "Unexpected extension");
6248 unsigned Scale = NumBitsPerElt / NumBitsPerSrcElt;
6249 for (unsigned I = 0; I != NumElts; ++I)
6250 Mask.append(Scale, I);
6251 Ops.push_back(Src);
6252 return true;
6253 }
6254 case ISD::ZERO_EXTEND:
6255 case ISD::ANY_EXTEND:
6256 case ISD::ZERO_EXTEND_VECTOR_INREG:
6257 case ISD::ANY_EXTEND_VECTOR_INREG: {
6258 SDValue Src = N.getOperand(0);
6259 EVT SrcVT = Src.getValueType();
6260
6261 // Extended source must be a simple vector.
6262 if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
6263 (SrcVT.getScalarSizeInBits() % 8) != 0)
6264 return false;
6265
6266 bool IsAnyExtend =
6267 (ISD::ANY_EXTEND == Opcode || ISD::ANY_EXTEND_VECTOR_INREG == Opcode);
6268 DecodeZeroExtendMask(SrcVT.getScalarSizeInBits(), NumBitsPerElt, NumElts,
6269 IsAnyExtend, Mask);
6270 Ops.push_back(Src);
6271 return true;
6272 }
6273 }
6274
6275 return false;
6276}
6277
6278/// Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
6279static void resolveTargetShuffleInputsAndMask(SmallVectorImpl<SDValue> &Inputs,
6280 SmallVectorImpl<int> &Mask) {
6281 int MaskWidth = Mask.size();
6282 SmallVector<SDValue, 16> UsedInputs;
6283 for (int i = 0, e = Inputs.size(); i < e; ++i) {
6284 int lo = UsedInputs.size() * MaskWidth;
6285 int hi = lo + MaskWidth;
6286
6287 // Strip UNDEF input usage.
6288 if (Inputs[i].isUndef())
6289 for (int &M : Mask)
6290 if ((lo <= M) && (M < hi))
6291 M = SM_SentinelUndef;
6292
6293 // Check for unused inputs.
6294 if (none_of(Mask, [lo, hi](int i) { return (lo <= i) && (i < hi); })) {
6295 for (int &M : Mask)
6296 if (lo <= M)
6297 M -= MaskWidth;
6298 continue;
6299 }
6300
6301 // Check for repeated inputs.
6302 bool IsRepeat = false;
6303 for (int j = 0, ue = UsedInputs.size(); j != ue; ++j) {
6304 if (UsedInputs[j] != Inputs[i])
6305 continue;
6306 for (int &M : Mask)
6307 if (lo <= M)
6308 M = (M < hi) ? ((M - lo) + (j * MaskWidth)) : (M - MaskWidth);
6309 IsRepeat = true;
6310 break;
6311 }
6312 if (IsRepeat)
6313 continue;
6314
6315 UsedInputs.push_back(Inputs[i]);
6316 }
6317 Inputs = UsedInputs;
6318}
6319
6320/// Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs
6321/// and then sets the SM_SentinelUndef and SM_SentinelZero values.
6322/// Returns true if the target shuffle mask was decoded.
6323static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
6324 SmallVectorImpl<SDValue> &Inputs,
6325 SmallVectorImpl<int> &Mask,
6326 APInt &KnownUndef, APInt &KnownZero,
6327 const SelectionDAG &DAG, unsigned Depth,
6328 bool ResolveKnownElts) {
6329 if (Depth >= SelectionDAG::MaxRecursionDepth)
6330 return false; // Limit search depth.
6331
6332 EVT VT = Op.getValueType();
6333 if (!VT.isSimple() || !VT.isVector())
6334 return false;
6335
6336 if (getTargetShuffleAndZeroables(Op, Mask, Inputs, KnownUndef, KnownZero)) {
6337 if (ResolveKnownElts)
6338 resolveTargetShuffleFromZeroables(Mask, KnownUndef, KnownZero);
6339 return true;
6340 }
6341 if (getFauxShuffleMask(Op, DemandedElts, Mask, Inputs, DAG, Depth,
6342 ResolveKnownElts)) {
6343 resolveZeroablesFromTargetShuffle(Mask, KnownUndef, KnownZero);
6344 return true;
6345 }
6346 return false;
6347}
6348
6349static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
6350 SmallVectorImpl<SDValue> &Inputs,
6351 SmallVectorImpl<int> &Mask,
6352 const SelectionDAG &DAG, unsigned Depth,
6353 bool ResolveKnownElts) {
6354 APInt KnownUndef, KnownZero;
6355 return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, KnownUndef,
6356 KnownZero, DAG, Depth, ResolveKnownElts);
6357}
6358
6359static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
6360 SmallVectorImpl<int> &Mask,
6361 const SelectionDAG &DAG, unsigned Depth = 0,
6362 bool ResolveKnownElts = true) {
6363 EVT VT = Op.getValueType();
6364 if (!VT.isSimple() || !VT.isVector())
6365 return false;
6366
6367 unsigned NumElts = Op.getValueType().getVectorNumElements();
6368 APInt DemandedElts = APInt::getAllOnes(NumElts);
6369 return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, DAG, Depth,
6370 ResolveKnownElts);
6371}
6372
6373// Attempt to create a scalar/subvector broadcast from the base MemSDNode.
6374static SDValue getBROADCAST_LOAD(unsigned Opcode, const SDLoc &DL, EVT VT,
6375 EVT MemVT, MemSDNode *Mem, unsigned Offset,
6376 SelectionDAG &DAG) {
6377 assert((Opcode == X86ISD::VBROADCAST_LOAD ||
6378 Opcode == X86ISD::SUBV_BROADCAST_LOAD) &&
6379 "Unknown broadcast load type");
6380
6381 // Ensure this is a simple (non-atomic, non-voltile), temporal read memop.
6382 if (!Mem || !Mem->readMem() || !Mem->isSimple() || Mem->isNonTemporal())
6383 return SDValue();
6384
6385 SDValue Ptr = DAG.getMemBasePlusOffset(Mem->getBasePtr(),
6386 TypeSize::getFixed(Offset), DL);
6387 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
6388 SDValue Ops[] = {Mem->getChain(), Ptr};
6389 SDValue BcstLd = DAG.getMemIntrinsicNode(
6390 Opcode, DL, Tys, Ops, MemVT,
6391 DAG.getMachineFunction().getMachineMemOperand(
6392 Mem->getMemOperand(), Offset, MemVT.getStoreSize()));
6393 DAG.makeEquivalentMemoryOrdering(SDValue(Mem, 1), BcstLd.getValue(1));
6394 return BcstLd;
6395}
6396
6397/// Returns the scalar element that will make up the i'th
6398/// element of the result of the vector shuffle.
6399static SDValue getShuffleScalarElt(SDValue Op, unsigned Index,
6400 SelectionDAG &DAG, unsigned Depth) {
6401 if (Depth >= SelectionDAG::MaxRecursionDepth)
6402 return SDValue(); // Limit search depth.
6403
6404 EVT VT = Op.getValueType();
6405 unsigned Opcode = Op.getOpcode();
6406 unsigned NumElems = VT.getVectorNumElements();
6407
6408 // Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
6409 if (auto *SV = dyn_cast<ShuffleVectorSDNode>(Op)) {
6410 int Elt = SV->getMaskElt(Index);
6411
6412 if (Elt < 0)
6413 return DAG.getUNDEF(VT.getVectorElementType());
6414
6415 SDValue Src = (Elt < (int)NumElems) ? SV->getOperand(0) : SV->getOperand(1);
6416 return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
6417 }
6418
6419 // Recurse into target specific vector shuffles to find scalars.
6420 if (isTargetShuffle(Opcode)) {
6421 MVT ShufVT = VT.getSimpleVT();
6422 MVT ShufSVT = ShufVT.getVectorElementType();
6423 int NumElems = (int)ShufVT.getVectorNumElements();
6424 SmallVector<int, 16> ShuffleMask;
6425 SmallVector<SDValue, 16> ShuffleOps;
6426 if (!getTargetShuffleMask(Op, true, ShuffleOps, ShuffleMask))
6427 return SDValue();
6428
6429 int Elt = ShuffleMask[Index];
6430 if (Elt == SM_SentinelZero)
6431 return ShufSVT.isInteger() ? DAG.getConstant(0, SDLoc(Op), ShufSVT)
6432 : DAG.getConstantFP(+0.0, SDLoc(Op), ShufSVT);
6433 if (Elt == SM_SentinelUndef)
6434 return DAG.getUNDEF(ShufSVT);
6435
6436 assert(0 <= Elt && Elt < (2 * NumElems) && "Shuffle index out of range");
6437 SDValue Src = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1];
6438 return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1);
6439 }
6440
6441 // Recurse into insert_subvector base/sub vector to find scalars.
6442 if (Opcode == ISD::INSERT_SUBVECTOR) {
6443 SDValue Vec = Op.getOperand(0);
6444 SDValue Sub = Op.getOperand(1);
6445 uint64_t SubIdx = Op.getConstantOperandVal(2);
6446 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
6447
6448 if (SubIdx <= Index && Index < (SubIdx + NumSubElts))
6449 return getShuffleScalarElt(Sub, Index - SubIdx, DAG, Depth + 1);
6450 return getShuffleScalarElt(Vec, Index, DAG, Depth + 1);
6451 }
6452
6453 // Recurse into concat_vectors sub vector to find scalars.
6454 if (Opcode == ISD::CONCAT_VECTORS) {
6455 EVT SubVT = Op.getOperand(0).getValueType();
6456 unsigned NumSubElts = SubVT.getVectorNumElements();
6457 uint64_t SubIdx = Index / NumSubElts;
6458 uint64_t SubElt = Index % NumSubElts;
6459 return getShuffleScalarElt(Op.getOperand(SubIdx), SubElt, DAG, Depth + 1);
6460 }
6461
6462 // Recurse into extract_subvector src vector to find scalars.
6463 if (Opcode == ISD::EXTRACT_SUBVECTOR) {
6464 SDValue Src = Op.getOperand(0);
6465 uint64_t SrcIdx = Op.getConstantOperandVal(1);
6466 return getShuffleScalarElt(Src, Index + SrcIdx, DAG, Depth + 1);
6467 }
6468
6469 // We only peek through bitcasts of the same vector width.
6470 if (Opcode == ISD::BITCAST) {
6471 SDValue Src = Op.getOperand(0);
6472 EVT SrcVT = Src.getValueType();
6473 if (SrcVT.isVector() && SrcVT.getVectorNumElements() == NumElems)
6474 return getShuffleScalarElt(Src, Index, DAG, Depth + 1);
6475 return SDValue();
6476 }
6477
6478 // Actual nodes that may contain scalar elements
6479
6480 // For insert_vector_elt - either return the index matching scalar or recurse
6481 // into the base vector.
6482 if (Opcode == ISD::INSERT_VECTOR_ELT &&
6483 isa<ConstantSDNode>(Op.getOperand(2))) {
6484 if (Op.getConstantOperandAPInt(2) == Index)
6485 return Op.getOperand(1);
6486 return getShuffleScalarElt(Op.getOperand(0), Index, DAG, Depth + 1);
6487 }
6488
6489 if (Opcode == ISD::SCALAR_TO_VECTOR)
6490 return (Index == 0) ? Op.getOperand(0)
6491 : DAG.getUNDEF(VT.getVectorElementType());
6492
6493 if (Opcode == ISD::BUILD_VECTOR)
6494 return Op.getOperand(Index);
6495
6496 return SDValue();
6497}
6498
6499// Use PINSRB/PINSRW/PINSRD to create a build vector.
6500static SDValue LowerBuildVectorAsInsert(SDValue Op, const SDLoc &DL,
6501 const APInt &NonZeroMask,
6502 unsigned NumNonZero, unsigned NumZero,
6503 SelectionDAG &DAG,
6504 const X86Subtarget &Subtarget) {
6505 MVT VT = Op.getSimpleValueType();
6506 unsigned NumElts = VT.getVectorNumElements();
6507 assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) ||
6508 ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) &&
6509 "Illegal vector insertion");
6510
6511 SDValue V;
6512 bool First = true;
6513
6514 for (unsigned i = 0; i < NumElts; ++i) {
6515 bool IsNonZero = NonZeroMask[i];
6516 if (!IsNonZero)
6517 continue;
6518
6519 // If the build vector contains zeros or our first insertion is not the
6520 // first index then insert into zero vector to break any register
6521 // dependency else use SCALAR_TO_VECTOR.
6522 if (First) {
6523 First = false;
6524 if (NumZero || 0 != i)
6525 V = getZeroVector(VT, Subtarget, DAG, DL);
6526 else {
6527 assert(0 == i && "Expected insertion into zero-index");
6528 V = DAG.getAnyExtOrTrunc(Op.getOperand(i), DL, MVT::i32);
6529 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, V);
6530 V = DAG.getBitcast(VT, V);
6531 continue;
6532 }
6533 }
6534 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, V, Op.getOperand(i),
6535 DAG.getIntPtrConstant(i, DL));
6536 }
6537
6538 return V;
6539}
6540
6541/// Custom lower build_vector of v16i8.
6542static SDValue LowerBuildVectorv16i8(SDValue Op, const SDLoc &DL,
6543 const APInt &NonZeroMask,
6544 unsigned NumNonZero, unsigned NumZero,
6545 SelectionDAG &DAG,
6546 const X86Subtarget &Subtarget) {
6547 if (NumNonZero > 8 && !Subtarget.hasSSE41())
6548 return SDValue();
6549
6550 // SSE4.1 - use PINSRB to insert each byte directly.
6551 if (Subtarget.hasSSE41())
6552 return LowerBuildVectorAsInsert(Op, DL, NonZeroMask, NumNonZero, NumZero,
6553 DAG, Subtarget);
6554
6555 SDValue V;
6556
6557 // Pre-SSE4.1 - merge byte pairs and insert with PINSRW.
6558 // If both the lowest 16-bits are non-zero, then convert to MOVD.
6559 if (!NonZeroMask.extractBits(2, 0).isZero() &&
6560 !NonZeroMask.extractBits(2, 2).isZero()) {
6561 for (unsigned I = 0; I != 4; ++I) {
6562 if (!NonZeroMask[I])
6563 continue;
6564 SDValue Elt = DAG.getZExtOrTrunc(Op.getOperand(I), DL, MVT::i32);
6565 if (I != 0)
6566 Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt,
6567 DAG.getConstant(I * 8, DL, MVT::i8));
6568 V = V ? DAG.getNode(ISD::OR, DL, MVT::i32, V, Elt) : Elt;
6569 }
6570 assert(V && "Failed to fold v16i8 vector to zero");
6571 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, V);
6572 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v4i32, V);
6573 V = DAG.getBitcast(MVT::v8i16, V);
6574 }
6575 for (unsigned i = V ? 4 : 0; i < 16; i += 2) {
6576 bool ThisIsNonZero = NonZeroMask[i];
6577 bool NextIsNonZero = NonZeroMask[i + 1];
6578 if (!ThisIsNonZero && !NextIsNonZero)
6579 continue;
6580
6581 SDValue Elt;
6582 if (ThisIsNonZero) {
6583 if (NumZero || NextIsNonZero)
6584 Elt = DAG.getZExtOrTrunc(Op.getOperand(i), DL, MVT::i32);
6585 else
6586 Elt = DAG.getAnyExtOrTrunc(Op.getOperand(i), DL, MVT::i32);
6587 }
6588
6589 if (NextIsNonZero) {
6590 SDValue NextElt = Op.getOperand(i + 1);
6591 if (i == 0 && NumZero)
6592 NextElt = DAG.getZExtOrTrunc(NextElt, DL, MVT::i32);
6593 else
6594 NextElt = DAG.getAnyExtOrTrunc(NextElt, DL, MVT::i32);
6595 NextElt = DAG.getNode(ISD::SHL, DL, MVT::i32, NextElt,
6596 DAG.getConstant(8, DL, MVT::i8));
6597 if (ThisIsNonZero)
6598 Elt = DAG.getNode(ISD::OR, DL, MVT::i32, NextElt, Elt);
6599 else
6600 Elt = NextElt;
6601 }
6602
6603 // If our first insertion is not the first index or zeros are needed, then
6604 // insert into zero vector. Otherwise, use SCALAR_TO_VECTOR (leaves high
6605 // elements undefined).
6606 if (!V) {
6607 if (i != 0 || NumZero)
6608 V = getZeroVector(MVT::v8i16, Subtarget, DAG, DL);
6609 else {
6610 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, Elt);
6611 V = DAG.getBitcast(MVT::v8i16, V);
6612 continue;
6613 }
6614 }
6615 Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt);
6616 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16, V, Elt,
6617 DAG.getIntPtrConstant(i / 2, DL));
6618 }
6619
6620 return DAG.getBitcast(MVT::v16i8, V);
6621}
6622
6623/// Custom lower build_vector of v8i16.
6624static SDValue LowerBuildVectorv8i16(SDValue Op, const SDLoc &DL,
6625 const APInt &NonZeroMask,
6626 unsigned NumNonZero, unsigned NumZero,
6627 SelectionDAG &DAG,
6628 const X86Subtarget &Subtarget) {
6629 if (NumNonZero > 4 && !Subtarget.hasSSE41())
6630 return SDValue();
6631
6632 // Use PINSRW to insert each byte directly.
6633 return LowerBuildVectorAsInsert(Op, DL, NonZeroMask, NumNonZero, NumZero, DAG,
6634 Subtarget);
6635}
6636
6637/// Custom lower build_vector of v4i32 or v4f32.
6638static SDValue LowerBuildVectorv4x32(SDValue Op, const SDLoc &DL,
6639 SelectionDAG &DAG,
6640 const X86Subtarget &Subtarget) {
6641 // If this is a splat of a pair of elements, use MOVDDUP (unless the target
6642 // has XOP; in that case defer lowering to potentially use VPERMIL2PS).
6643 // Because we're creating a less complicated build vector here, we may enable
6644 // further folding of the MOVDDUP via shuffle transforms.
6645 if (Subtarget.hasSSE3() && !Subtarget.hasXOP() &&
6646 Op.getOperand(0) == Op.getOperand(2) &&
6647 Op.getOperand(1) == Op.getOperand(3) &&
6648 Op.getOperand(0) != Op.getOperand(1)) {
6649 MVT VT = Op.getSimpleValueType();
6650 MVT EltVT = VT.getVectorElementType();
6651 // Create a new build vector with the first 2 elements followed by undef
6652 // padding, bitcast to v2f64, duplicate, and bitcast back.
6653 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
6654 DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
6655 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
6656 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
6657 return DAG.getBitcast(VT, Dup);
6658 }
6659
6660 // Find all zeroable elements.
6661 std::bitset<4> Zeroable, Undefs;
6662 for (int i = 0; i < 4; ++i) {
6663 SDValue Elt = Op.getOperand(i);
6664 Undefs[i] = Elt.isUndef();
6665 Zeroable[i] = (Elt.isUndef() || X86::isZeroNode(Elt));
6666 }
6667 assert(Zeroable.size() - Zeroable.count() > 1 &&
6668 "We expect at least two non-zero elements!");
6669
6670 // We only know how to deal with build_vector nodes where elements are either
6671 // zeroable or extract_vector_elt with constant index.
6672 SDValue FirstNonZero;
6673 unsigned FirstNonZeroIdx;
6674 for (unsigned i = 0; i < 4; ++i) {
6675 if (Zeroable[i])
6676 continue;
6677 SDValue Elt = Op.getOperand(i);
6678 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6679 !isa<ConstantSDNode>(Elt.getOperand(1)))
6680 return SDValue();
6681 // Make sure that this node is extracting from a 128-bit vector.
6682 MVT VT = Elt.getOperand(0).getSimpleValueType();
6683 if (!VT.is128BitVector())
6684 return SDValue();
6685 if (!FirstNonZero.getNode()) {
6686 FirstNonZero = Elt;
6687 FirstNonZeroIdx = i;
6688 }
6689 }
6690
6691 assert(FirstNonZero.getNode() && "Unexpected build vector of all zeros!");
6692 SDValue V1 = FirstNonZero.getOperand(0);
6693 MVT VT = V1.getSimpleValueType();
6694
6695 // See if this build_vector can be lowered as a blend with zero.
6696 SDValue Elt;
6697 unsigned EltMaskIdx, EltIdx;
6698 int Mask[4];
6699 for (EltIdx = 0; EltIdx < 4; ++EltIdx) {
6700 if (Zeroable[EltIdx]) {
6701 // The zero vector will be on the right hand side.
6702 Mask[EltIdx] = EltIdx+4;
6703 continue;
6704 }
6705
6706 Elt = Op->getOperand(EltIdx);
6707 // By construction, Elt is a EXTRACT_VECTOR_ELT with constant index.
6708 EltMaskIdx = Elt.getConstantOperandVal(1);
6709 if (Elt.getOperand(0) != V1 || EltMaskIdx != EltIdx)
6710 break;
6711 Mask[EltIdx] = EltIdx;
6712 }
6713
6714 if (EltIdx == 4) {
6715 // Let the shuffle legalizer deal with blend operations.
6716 SDValue VZeroOrUndef = (Zeroable == Undefs)
6717 ? DAG.getUNDEF(VT)
6718 : getZeroVector(VT, Subtarget, DAG, DL);
6719 if (V1.getSimpleValueType() != VT)
6720 V1 = DAG.getBitcast(VT, V1);
6721 return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZeroOrUndef, Mask);
6722 }
6723
6724 // See if we can lower this build_vector to a INSERTPS.
6725 if (!Subtarget.hasSSE41())
6726 return SDValue();
6727
6728 SDValue V2 = Elt.getOperand(0);
6729 if (Elt == FirstNonZero && EltIdx == FirstNonZeroIdx)
6730 V1 = SDValue();
6731
6732 bool CanFold = true;
6733 for (unsigned i = EltIdx + 1; i < 4 && CanFold; ++i) {
6734 if (Zeroable[i])
6735 continue;
6736
6737 SDValue Current = Op->getOperand(i);
6738 SDValue SrcVector = Current->getOperand(0);
6739 if (!V1.getNode())
6740 V1 = SrcVector;
6741 CanFold = (SrcVector == V1) && (Current.getConstantOperandAPInt(1) == i);
6742 }
6743
6744 if (!CanFold)
6745 return SDValue();
6746
6747 assert(V1.getNode() && "Expected at least two non-zero elements!");
6748 if (V1.getSimpleValueType() != MVT::v4f32)
6749 V1 = DAG.getBitcast(MVT::v4f32, V1);
6750 if (V2.getSimpleValueType() != MVT::v4f32)
6751 V2 = DAG.getBitcast(MVT::v4f32, V2);
6752
6753 // Ok, we can emit an INSERTPS instruction.
6754 unsigned ZMask = Zeroable.to_ulong();
6755
6756 unsigned InsertPSMask = EltMaskIdx << 6 | EltIdx << 4 | ZMask;
6757 assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
6758 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
6759 DAG.getIntPtrConstant(InsertPSMask, DL, true));
6760 return DAG.getBitcast(VT, Result);
6761}
6762
6763/// Return a vector logical shift node.
6764static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,
6765 SelectionDAG &DAG, const TargetLowering &TLI,
6766 const SDLoc &dl) {
6767 assert(VT.is128BitVector() && "Unknown type for VShift");
6768 MVT ShVT = MVT::v16i8;
6769 unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
6770 SrcOp = DAG.getBitcast(ShVT, SrcOp);
6771 assert(NumBits % 8 == 0 && "Only support byte sized shifts");
6772 SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);
6773 return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
6774}
6775
6776static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
6777 SelectionDAG &DAG) {
6778
6779 // Check if the scalar load can be widened into a vector load. And if
6780 // the address is "base + cst" see if the cst can be "absorbed" into
6781 // the shuffle mask.
6782 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) {
6783 SDValue Ptr = LD->getBasePtr();
6784 if (!ISD::isNormalLoad(LD) || !LD->isSimple())
6785 return SDValue();
6786 EVT PVT = LD->getValueType(0);
6787 if (PVT != MVT::i32 && PVT != MVT::f32)
6788 return SDValue();
6789
6790 int FI = -1;
6791 int64_t Offset = 0;
6792 if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) {
6793 FI = FINode->getIndex();
6794 Offset = 0;
6795 } else if (DAG.isBaseWithConstantOffset(Ptr) &&
6796 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
6797 FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6798 Offset = Ptr.getConstantOperandVal(1);
6799 Ptr = Ptr.getOperand(0);
6800 } else {
6801 return SDValue();
6802 }
6803
6804 // FIXME: 256-bit vector instructions don't require a strict alignment,
6805 // improve this code to support it better.
6806 Align RequiredAlign(VT.getSizeInBits() / 8);
6807 SDValue Chain = LD->getChain();
6808 // Make sure the stack object alignment is at least 16 or 32.
6809 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
6810 MaybeAlign InferredAlign = DAG.InferPtrAlign(Ptr);
6811 if (!InferredAlign || *InferredAlign < RequiredAlign) {
6812 if (MFI.isFixedObjectIndex(FI)) {
6813 // Can't change the alignment. FIXME: It's possible to compute
6814 // the exact stack offset and reference FI + adjust offset instead.
6815 // If someone *really* cares about this. That's the way to implement it.
6816 return SDValue();
6817 } else {
6818 MFI.setObjectAlignment(FI, RequiredAlign);
6819 }
6820 }
6821
6822 // (Offset % 16 or 32) must be multiple of 4. Then address is then
6823 // Ptr + (Offset & ~15).
6824 if (Offset < 0)
6825 return SDValue();
6826 if ((Offset % RequiredAlign.value()) & 3)
6827 return SDValue();
6828 int64_t StartOffset = Offset & ~int64_t(RequiredAlign.value() - 1);
6829 if (StartOffset) {
6830 SDLoc DL(Ptr);
6831 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
6832 DAG.getConstant(StartOffset, DL, Ptr.getValueType()));
6833 }
6834
6835 int EltNo = (Offset - StartOffset) >> 2;
6836 unsigned NumElems = VT.getVectorNumElements();
6837
6838 EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems);
6839 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
6840 LD->getPointerInfo().getWithOffset(StartOffset));
6841
6842 SmallVector<int, 8> Mask(NumElems, EltNo);
6843
6844 return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), Mask);
6845 }
6846
6847 return SDValue();
6848}
6849
6850// Recurse to find a LoadSDNode source and the accumulated ByteOffest.
6851static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) {
6852 if (ISD::isNON_EXTLoad(Elt.getNode())) {
6853 auto *BaseLd = cast<LoadSDNode>(Elt);
6854 if (!BaseLd->isSimple())
6855 return false;
6856 Ld = BaseLd;
6857 ByteOffset = 0;
6858 return true;
6859 }
6860
6861 switch (Elt.getOpcode()) {
6862 case ISD::BITCAST:
6863 case ISD::TRUNCATE:
6864 case ISD::SCALAR_TO_VECTOR:
6865 return findEltLoadSrc(Elt.getOperand(0), Ld, ByteOffset);
6866 case ISD::SRL:
6867 if (auto *AmtC = dyn_cast<ConstantSDNode>(Elt.getOperand(1))) {
6868 uint64_t Amt = AmtC->getZExtValue();
6869 if ((Amt % 8) == 0 && findEltLoadSrc(Elt.getOperand(0), Ld, ByteOffset)) {
6870 ByteOffset += Amt / 8;
6871 return true;
6872 }
6873 }
6874 break;
6875 case ISD::EXTRACT_VECTOR_ELT:
6876 if (auto *IdxC = dyn_cast<ConstantSDNode>(Elt.getOperand(1))) {
6877 SDValue Src = Elt.getOperand(0);
6878 unsigned SrcSizeInBits = Src.getScalarValueSizeInBits();
6879 unsigned DstSizeInBits = Elt.getScalarValueSizeInBits();
6880 if (DstSizeInBits == SrcSizeInBits && (SrcSizeInBits % 8) == 0 &&
6881 findEltLoadSrc(Src, Ld, ByteOffset)) {
6882 uint64_t Idx = IdxC->getZExtValue();
6883 ByteOffset += Idx * (SrcSizeInBits / 8);
6884 return true;
6885 }
6886 }
6887 break;
6888 }
6889
6890 return false;
6891}
6892
6893/// Given the initializing elements 'Elts' of a vector of type 'VT', see if the
6894/// elements can be replaced by a single large load which has the same value as
6895/// a build_vector or insert_subvector whose loaded operands are 'Elts'.
6896///
6897/// Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a
6898static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
6899 const SDLoc &DL, SelectionDAG &DAG,
6900 const X86Subtarget &Subtarget,
6901 bool IsAfterLegalize) {
6902 if ((VT.getScalarSizeInBits() % 8) != 0)
6903 return SDValue();
6904
6905 unsigned NumElems = Elts.size();
6906
6907 int LastLoadedElt = -1;
6908 APInt LoadMask = APInt::getZero(NumElems);
6909 APInt ZeroMask = APInt::getZero(NumElems);
6910 APInt UndefMask = APInt::getZero(NumElems);
6911
6912 SmallVector<LoadSDNode*, 8> Loads(NumElems, nullptr);
6913 SmallVector<int64_t, 8> ByteOffsets(NumElems, 0);
6914
6915 // For each element in the initializer, see if we've found a load, zero or an
6916 // undef.
6917 for (unsigned i = 0; i < NumElems; ++i) {
6918 SDValue Elt = peekThroughBitcasts(Elts[i]);
6919 if (!Elt.getNode())
6920 return SDValue();
6921 if (Elt.isUndef()) {
6922 UndefMask.setBit(i);
6923 continue;
6924 }
6925 if (X86::isZeroNode(Elt) || ISD::isBuildVectorAllZeros(Elt.getNode())) {
6926 ZeroMask.setBit(i);
6927 continue;
6928 }
6929
6930 // Each loaded element must be the correct fractional portion of the
6931 // requested vector load.
6932 unsigned EltSizeInBits = Elt.getValueSizeInBits();
6933 if ((NumElems * EltSizeInBits) != VT.getSizeInBits())
6934 return SDValue();
6935
6936 if (!findEltLoadSrc(Elt, Loads[i], ByteOffsets[i]) || ByteOffsets[i] < 0)
6937 return SDValue();
6938 unsigned LoadSizeInBits = Loads[i]->getValueSizeInBits(0);
6939 if (((ByteOffsets[i] * 8) + EltSizeInBits) > LoadSizeInBits)
6940 return SDValue();
6941
6942 LoadMask.setBit(i);
6943 LastLoadedElt = i;
6944 }
6945 assert((ZeroMask.popcount() + UndefMask.popcount() + LoadMask.popcount()) ==
6946 NumElems &&
6947 "Incomplete element masks");
6948
6949 // Handle Special Cases - all undef or undef/zero.
6950 if (UndefMask.popcount() == NumElems)
6951 return DAG.getUNDEF(VT);
6952 if ((ZeroMask.popcount() + UndefMask.popcount()) == NumElems)
6953 return VT.isInteger() ? DAG.getConstant(0, DL, VT)
6954 : DAG.getConstantFP(0.0, DL, VT);
6955
6956 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6957 int FirstLoadedElt = LoadMask.countr_zero();
6958 SDValue EltBase = peekThroughBitcasts(Elts[FirstLoadedElt]);
6959 EVT EltBaseVT = EltBase.getValueType();
6960 assert(EltBaseVT.getSizeInBits() == EltBaseVT.getStoreSizeInBits() &&
6961 "Register/Memory size mismatch");
6962 LoadSDNode *LDBase = Loads[FirstLoadedElt];
6963 assert(LDBase && "Did not find base load for merging consecutive loads");
6964 unsigned BaseSizeInBits = EltBaseVT.getStoreSizeInBits();
6965 unsigned BaseSizeInBytes = BaseSizeInBits / 8;
6966 int NumLoadedElts = (1 + LastLoadedElt - FirstLoadedElt);
6967 int LoadSizeInBits = NumLoadedElts * BaseSizeInBits;
6968 assert((BaseSizeInBits % 8) == 0 && "Sub-byte element loads detected");
6969
6970 // TODO: Support offsetting the base load.
6971 if (ByteOffsets[FirstLoadedElt] != 0)
6972 return SDValue();
6973
6974 // Check to see if the element's load is consecutive to the base load
6975 // or offset from a previous (already checked) load.
6976 auto CheckConsecutiveLoad = [&](LoadSDNode *Base, int EltIdx) {
6977 LoadSDNode *Ld = Loads[EltIdx];
6978 int64_t ByteOffset = ByteOffsets[EltIdx];
6979 if (ByteOffset && (ByteOffset % BaseSizeInBytes) == 0) {
6980 int64_t BaseIdx = EltIdx - (ByteOffset / BaseSizeInBytes);
6981 return (0 <= BaseIdx && BaseIdx < (int)NumElems && LoadMask[BaseIdx] &&
6982 Loads[BaseIdx] == Ld && ByteOffsets[BaseIdx] == 0);
6983 }
6984 return DAG.areNonVolatileConsecutiveLoads(Ld, Base, BaseSizeInBytes,
6985 EltIdx - FirstLoadedElt);
6986 };
6987
6988 // Consecutive loads can contain UNDEFS but not ZERO elements.
6989 // Consecutive loads with UNDEFs and ZEROs elements require a
6990 // an additional shuffle stage to clear the ZERO elements.
6991 bool IsConsecutiveLoad = true;
6992 bool IsConsecutiveLoadWithZeros = true;
6993 for (int i = FirstLoadedElt + 1; i <= LastLoadedElt; ++i) {
6994 if (LoadMask[i]) {
6995 if (!CheckConsecutiveLoad(LDBase, i)) {
6996 IsConsecutiveLoad = false;
6997 IsConsecutiveLoadWithZeros = false;
6998 break;
6999 }
7000 } else if (ZeroMask[i]) {
7001 IsConsecutiveLoad = false;
7002 }
7003 }
7004
7005 auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {
7006 auto MMOFlags = LDBase->getMemOperand()->getFlags();
7007 assert(LDBase->isSimple() &&
7008 "Cannot merge volatile or atomic loads.");
7009 SDValue NewLd =
7010 DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
7011 LDBase->getPointerInfo(), LDBase->getOriginalAlign(),
7012 MMOFlags);
7013 for (auto *LD : Loads)
7014 if (LD)
7015 DAG.makeEquivalentMemoryOrdering(LD, NewLd);
7016 return NewLd;
7017 };
7018
7019 // Check if the base load is entirely dereferenceable.
7020 bool IsDereferenceable = LDBase->getPointerInfo().isDereferenceable(
7021 VT.getSizeInBits() / 8, *DAG.getContext(), DAG.getDataLayout());
7022
7023 // LOAD - all consecutive load/undefs (must start/end with a load or be
7024 // entirely dereferenceable). If we have found an entire vector of loads and
7025 // undefs, then return a large load of the entire vector width starting at the
7026 // base pointer. If the vector contains zeros, then attempt to shuffle those
7027 // elements.
7028 if (FirstLoadedElt == 0 &&
7029 (NumLoadedElts == (int)NumElems || IsDereferenceable) &&
7030 (IsConsecutiveLoad || IsConsecutiveLoadWithZeros)) {
7031 if (IsAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))
7032 return SDValue();
7033
7034 // Don't create 256-bit non-temporal aligned loads without AVX2 as these
7035 // will lower to regular temporal loads and use the cache.
7036 if (LDBase->isNonTemporal() && LDBase->getAlign() >= Align(32) &&
7037 VT.is256BitVector() && !Subtarget.hasInt256())
7038 return SDValue();
7039
7040 if (NumElems == 1)
7041 return DAG.getBitcast(VT, Elts[FirstLoadedElt]);
7042
7043 if (!ZeroMask)
7044 return CreateLoad(VT, LDBase);
7045
7046 // IsConsecutiveLoadWithZeros - we need to create a shuffle of the loaded
7047 // vector and a zero vector to clear out the zero elements.
7048 if (!IsAfterLegalize && VT.isVector()) {
7049 unsigned NumMaskElts = VT.getVectorNumElements();
7050 if ((NumMaskElts % NumElems) == 0) {
7051 unsigned Scale = NumMaskElts / NumElems;
7052 SmallVector<int, 4> ClearMask(NumMaskElts, -1);
7053 for (unsigned i = 0; i < NumElems; ++i) {
7054 if (UndefMask[i])
7055 continue;
7056 int Offset = ZeroMask[i] ? NumMaskElts : 0;
7057 for (unsigned j = 0; j != Scale; ++j)
7058 ClearMask[(i * Scale) + j] = (i * Scale) + j + Offset;
7059 }
7060 SDValue V = CreateLoad(VT, LDBase);
7061 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
7062 : DAG.getConstantFP(0.0, DL, VT);
7063 return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);
7064 }
7065 }
7066 }
7067
7068 // If the upper half of a ymm/zmm load is undef then just load the lower half.
7069 if (VT.is256BitVector() || VT.is512BitVector()) {
7070 unsigned HalfNumElems = NumElems / 2;
7071 if (UndefMask.extractBits(HalfNumElems, HalfNumElems).isAllOnes()) {
7072 EVT HalfVT =
7073 EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems);
7074 SDValue HalfLD =
7075 EltsFromConsecutiveLoads(HalfVT, Elts.drop_back(HalfNumElems), DL,
7076 DAG, Subtarget, IsAfterLegalize);
7077 if (HalfLD)
7078 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
7079 HalfLD, DAG.getIntPtrConstant(0, DL));
7080 }
7081 }
7082
7083 // VZEXT_LOAD - consecutive 32/64-bit load/undefs followed by zeros/undefs.
7084 if (IsConsecutiveLoad && FirstLoadedElt == 0 &&
7085 ((LoadSizeInBits == 16 && Subtarget.hasFP16()) || LoadSizeInBits == 32 ||
7086 LoadSizeInBits == 64) &&
7087 ((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))) {
7088 MVT VecSVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(LoadSizeInBits)
7089 : MVT::getIntegerVT(LoadSizeInBits);
7090 MVT VecVT = MVT::getVectorVT(VecSVT, VT.getSizeInBits() / LoadSizeInBits);
7091 // Allow v4f32 on SSE1 only targets.
7092 // FIXME: Add more isel patterns so we can just use VT directly.
7093 if (!Subtarget.hasSSE2() && VT == MVT::v4f32)
7094 VecVT = MVT::v4f32;
7095 if (TLI.isTypeLegal(VecVT)) {
7096 SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
7097 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
7098 SDValue ResNode = DAG.getMemIntrinsicNode(
7099 X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT, LDBase->getPointerInfo(),
7100 LDBase->getOriginalAlign(), MachineMemOperand::MOLoad);
7101 for (auto *LD : Loads)
7102 if (LD)
7103 DAG.makeEquivalentMemoryOrdering(LD, ResNode);
7104 return DAG.getBitcast(VT, ResNode);
7105 }
7106 }
7107
7108 // BROADCAST - match the smallest possible repetition pattern, load that
7109 // scalar/subvector element and then broadcast to the entire vector.
7110 if (ZeroMask.isZero() && isPowerOf2_32(NumElems) && Subtarget.hasAVX() &&
7111 (VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) {
7112 for (unsigned SubElems = 1; SubElems < NumElems; SubElems *= 2) {
7113 unsigned RepeatSize = SubElems * BaseSizeInBits;
7114 unsigned ScalarSize = std::min(RepeatSize, 64u);
7115 if (!Subtarget.hasAVX2() && ScalarSize < 32)
7116 continue;
7117
7118 // Don't attempt a 1:N subvector broadcast - it should be caught by
7119 // combineConcatVectorOps, else will cause infinite loops.
7120 if (RepeatSize > ScalarSize && SubElems == 1)
7121 continue;
7122
7123 bool Match = true;
7124 SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT));
7125 for (unsigned i = 0; i != NumElems && Match; ++i) {
7126 if (!LoadMask[i])
7127 continue;
7128 SDValue Elt = peekThroughBitcasts(Elts[i]);
7129 if (RepeatedLoads[i % SubElems].isUndef())
7130 RepeatedLoads[i % SubElems] = Elt;
7131 else
7132 Match &= (RepeatedLoads[i % SubElems] == Elt);
7133 }
7134
7135 // We must have loads at both ends of the repetition.
7136 Match &= !RepeatedLoads.front().isUndef();
7137 Match &= !RepeatedLoads.back().isUndef();
7138 if (!Match)
7139 continue;
7140
7141 EVT RepeatVT =
7142 VT.isInteger() && (RepeatSize != 64 || TLI.isTypeLegal(MVT::i64))
7143 ? EVT::getIntegerVT(*DAG.getContext(), ScalarSize)
7144 : EVT::getFloatingPointVT(ScalarSize);
7145 if (RepeatSize > ScalarSize)
7146 RepeatVT = EVT::getVectorVT(*DAG.getContext(), RepeatVT,
7147 RepeatSize / ScalarSize);
7148 EVT BroadcastVT =
7149 EVT::getVectorVT(*DAG.getContext(), RepeatVT.getScalarType(),
7150 VT.getSizeInBits() / ScalarSize);
7151 if (TLI.isTypeLegal(BroadcastVT)) {
7152 if (SDValue RepeatLoad = EltsFromConsecutiveLoads(
7153 RepeatVT, RepeatedLoads, DL, DAG, Subtarget, IsAfterLegalize)) {
7154 SDValue Broadcast = RepeatLoad;
7155 if (RepeatSize > ScalarSize) {
7156 while (Broadcast.getValueSizeInBits() < VT.getSizeInBits())
7157 Broadcast = concatSubVectors(Broadcast, Broadcast, DAG, DL);
7158 } else {
7159 if (!Subtarget.hasAVX2() &&
7160 !X86::mayFoldLoadIntoBroadcastFromMem(
7161 RepeatLoad, RepeatVT.getScalarType().getSimpleVT(),
7162 Subtarget,
7163 /*AssumeSingleUse=*/true))
7164 return SDValue();
7165 Broadcast =
7166 DAG.getNode(X86ISD::VBROADCAST, DL, BroadcastVT, RepeatLoad);
7167 }
7168 return DAG.getBitcast(VT, Broadcast);
7169 }
7170 }
7171 }
7172 }
7173
7174 return SDValue();
7175}
7176
7177// Combine a vector ops (shuffles etc.) that is equal to build_vector load1,
7178// load2, load3, load4, <0, 1, 2, 3> into a vector load if the load addresses
7179// are consecutive, non-overlapping, and in the right order.
7180static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL,
7181 SelectionDAG &DAG,
7182 const X86Subtarget &Subtarget,
7183 bool IsAfterLegalize) {
7184 SmallVector<SDValue, 64> Elts;
7185 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
7186 if (SDValue Elt = getShuffleScalarElt(Op, i, DAG, 0)) {
7187 Elts.push_back(Elt);
7188 continue;
7189 }
7190 return SDValue();
7191 }
7192 assert(Elts.size() == VT.getVectorNumElements());
7193 return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget,
7194 IsAfterLegalize);
7195}
7196
7197static Constant *getConstantVector(MVT VT, ArrayRef<APInt> Bits,
7198 const APInt &Undefs, LLVMContext &C) {
7199 unsigned ScalarSize = VT.getScalarSizeInBits();
7200 Type *Ty = EVT(VT.getScalarType()).getTypeForEVT(C);
7201
7202 auto getConstantScalar = [&](const APInt &Val) -> Constant * {
7203 if (VT.isFloatingPoint()) {
7204 if (ScalarSize == 16)
7205 return ConstantFP::get(C, APFloat(APFloat::IEEEhalf(), Val));
7206 if (ScalarSize == 32)
7207 return ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val));
7208 assert(ScalarSize == 64 && "Unsupported floating point scalar size");
7209 return ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val));
7210 }
7211 return Constant::getIntegerValue(Ty, Val);
7212 };
7213
7214 SmallVector<Constant *, 32> ConstantVec;
7215 for (unsigned I = 0, E = Bits.size(); I != E; ++I)
7216 ConstantVec.push_back(Undefs[I] ? UndefValue::get(Ty)
7217 : getConstantScalar(Bits[I]));
7218
7219 return ConstantVector::get(ArrayRef<Constant *>(ConstantVec));
7220}
7221
7222static Constant *getConstantVector(MVT VT, const APInt &SplatValue,
7223 unsigned SplatBitSize, LLVMContext &C) {
7224 unsigned ScalarSize = VT.getScalarSizeInBits();
7225
7226 auto getConstantScalar = [&](const APInt &Val) -> Constant * {
7227 if (VT.isFloatingPoint()) {
7228 if (ScalarSize == 16)
7229 return ConstantFP::get(C, APFloat(APFloat::IEEEhalf(), Val));
7230 if (ScalarSize == 32)
7231 return ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val));
7232 assert(ScalarSize == 64 && "Unsupported floating point scalar size");
7233 return ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val));
7234 }
7235 return Constant::getIntegerValue(Type::getIntNTy(C, ScalarSize), Val);
7236 };
7237
7238 if (ScalarSize == SplatBitSize)
7239 return getConstantScalar(SplatValue);
7240
7241 unsigned NumElm = SplatBitSize / ScalarSize;
7242 SmallVector<Constant *, 32> ConstantVec;
7243 for (unsigned I = 0; I != NumElm; ++I) {
7244 APInt Val = SplatValue.extractBits(ScalarSize, ScalarSize * I);
7245 ConstantVec.push_back(getConstantScalar(Val));
7246 }
7247 return ConstantVector::get(ArrayRef<Constant *>(ConstantVec));
7248}
7249
7250static bool isFoldableUseOfShuffle(SDNode *N) {
7251 for (auto *U : N->uses()) {
7252 unsigned Opc = U->getOpcode();
7253 // VPERMV/VPERMV3 shuffles can never fold their index operands.
7254 if (Opc == X86ISD::VPERMV && U->getOperand(0).getNode() == N)
7255 return false;
7256 if (Opc == X86ISD::VPERMV3 && U->getOperand(1).getNode() == N)
7257 return false;
7258 if (isTargetShuffle(Opc))
7259 return true;
7260 if (Opc == ISD::BITCAST) // Ignore bitcasts
7261 return isFoldableUseOfShuffle(U);
7262 if (N->hasOneUse()) {
7263 // TODO, there may be some general way to know if a SDNode can
7264 // be folded. We now only know whether an MI is foldable.
7265 if (Opc == X86ISD::VPDPBUSD && U->getOperand(2).getNode() != N)
7266 return false;
7267 return true;
7268 }
7269 }
7270 return false;
7271}
7272
7273/// Attempt to use the vbroadcast instruction to generate a splat value
7274/// from a splat BUILD_VECTOR which uses:
7275/// a. A single scalar load, or a constant.
7276/// b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).
7277///
7278/// The VBROADCAST node is returned when a pattern is found,
7279/// or SDValue() otherwise.
7280static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
7281 const SDLoc &dl,
7282 const X86Subtarget &Subtarget,
7283 SelectionDAG &DAG) {
7284 // VBROADCAST requires AVX.
7285 // TODO: Splats could be generated for non-AVX CPUs using SSE
7286 // instructions, but there's less potential gain for only 128-bit vectors.
7287 if (!Subtarget.hasAVX())
7288 return SDValue();
7289
7290 MVT VT = BVOp->getSimpleValueType(0);
7291 unsigned NumElts = VT.getVectorNumElements();
7292 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7293 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
7294 "Unsupported vector type for broadcast.");
7295
7296 // See if the build vector is a repeating sequence of scalars (inc. splat).
7297 SDValue Ld;
7298 BitVector UndefElements;
7299 SmallVector<SDValue, 16> Sequence;
7300 if (BVOp->getRepeatedSequence(Sequence, &UndefElements)) {
7301 assert((NumElts % Sequence.size()) == 0 && "Sequence doesn't fit.");
7302 if (Sequence.size() == 1)
7303 Ld = Sequence[0];
7304 }
7305
7306 // Attempt to use VBROADCASTM
7307 // From this pattern:
7308 // a. t0 = (zext_i64 (bitcast_i8 v2i1 X))
7309 // b. t1 = (build_vector t0 t0)
7310 //
7311 // Create (VBROADCASTM v2i1 X)
7312 if (!Sequence.empty() && Subtarget.hasCDI()) {
7313 // If not a splat, are the upper sequence values zeroable?
7314 unsigned SeqLen = Sequence.size();
7315 bool UpperZeroOrUndef =
7316 SeqLen == 1 ||
7317 llvm::all_of(ArrayRef(Sequence).drop_front(),
7318 [](SDValue V) { return !V || isNullConstantOrUndef(V); });
7319 SDValue Op0 = Sequence[0];
7320 if (UpperZeroOrUndef && ((Op0.getOpcode() == ISD::BITCAST) ||
7321 (Op0.getOpcode() == ISD::ZERO_EXTEND &&
7322 Op0.getOperand(0).getOpcode() == ISD::BITCAST))) {
7323 SDValue BOperand = Op0.getOpcode() == ISD::BITCAST
7324 ? Op0.getOperand(0)
7325 : Op0.getOperand(0).getOperand(0);
7326 MVT MaskVT = BOperand.getSimpleValueType();
7327 MVT EltType = MVT::getIntegerVT(VT.getScalarSizeInBits() * SeqLen);
7328 if ((EltType == MVT::i64 && MaskVT == MVT::v8i1) || // for broadcastmb2q
7329 (EltType == MVT::i32 && MaskVT == MVT::v16i1)) { // for broadcastmw2d
7330 MVT BcstVT = MVT::getVectorVT(EltType, NumElts / SeqLen);
7331 if (!VT.is512BitVector() && !Subtarget.hasVLX()) {
7332 unsigned Scale = 512 / VT.getSizeInBits();
7333 BcstVT = MVT::getVectorVT(EltType, Scale * (NumElts / SeqLen));
7334 }
7335 SDValue Bcst = DAG.getNode(X86ISD::VBROADCASTM, dl, BcstVT, BOperand);
7336 if (BcstVT.getSizeInBits() != VT.getSizeInBits())
7337 Bcst = extractSubVector(Bcst, 0, DAG, dl, VT.getSizeInBits());
7338 return DAG.getBitcast(VT, Bcst);
7339 }
7340 }
7341 }
7342
7343 unsigned NumUndefElts = UndefElements.count();
7344 if (!Ld || (NumElts - NumUndefElts) <= 1) {
7345 APInt SplatValue, Undef;
7346 unsigned SplatBitSize;
7347 bool HasUndef;
7348 // Check if this is a repeated constant pattern suitable for broadcasting.
7349 if (BVOp->isConstantSplat(SplatValue, Undef, SplatBitSize, HasUndef) &&
7350 SplatBitSize > VT.getScalarSizeInBits() &&
7351 SplatBitSize < VT.getSizeInBits()) {
7352 // Avoid replacing with broadcast when it's a use of a shuffle
7353 // instruction to preserve the present custom lowering of shuffles.
7354 if (isFoldableUseOfShuffle(BVOp))
7355 return SDValue();
7356 // replace BUILD_VECTOR with broadcast of the repeated constants.
7357 LLVMContext *Ctx = DAG.getContext();
7358 MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
7359 if (SplatBitSize == 32 || SplatBitSize == 64 ||
7360 (SplatBitSize < 32 && Subtarget.hasAVX2())) {
7361 // Load the constant scalar/subvector and broadcast it.
7362 MVT CVT = MVT::getIntegerVT(SplatBitSize);
7363 Constant *C = getConstantVector(VT, SplatValue, SplatBitSize, *Ctx);
7364 SDValue CP = DAG.getConstantPool(C, PVT);
7365 unsigned Repeat = VT.getSizeInBits() / SplatBitSize;
7366
7367 Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign();
7368 SDVTList Tys = DAG.getVTList(MVT::getVectorVT(CVT, Repeat), MVT::Other);
7369 SDValue Ops[] = {DAG.getEntryNode(), CP};
7370 MachinePointerInfo MPI =
7371 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7372 SDValue Brdcst =
7373 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT,
7374 MPI, Alignment, MachineMemOperand::MOLoad);
7375 return DAG.getBitcast(VT, Brdcst);
7376 }
7377 if (SplatBitSize > 64) {
7378 // Load the vector of constants and broadcast it.
7379 Constant *VecC = getConstantVector(VT, SplatValue, SplatBitSize, *Ctx);
7380 SDValue VCP = DAG.getConstantPool(VecC, PVT);
7381 unsigned NumElm = SplatBitSize / VT.getScalarSizeInBits();
7382 MVT VVT = MVT::getVectorVT(VT.getScalarType(), NumElm);
7383 Align Alignment = cast<ConstantPoolSDNode>(VCP)->getAlign();
7384 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7385 SDValue Ops[] = {DAG.getEntryNode(), VCP};
7386 MachinePointerInfo MPI =
7387 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7388 return DAG.getMemIntrinsicNode(X86ISD::SUBV_BROADCAST_LOAD, dl, Tys,
7389 Ops, VVT, MPI, Alignment,
7390 MachineMemOperand::MOLoad);
7391 }
7392 }
7393
7394 // If we are moving a scalar into a vector (Ld must be set and all elements
7395 // but 1 are undef) and that operation is not obviously supported by
7396 // vmovd/vmovq/vmovss/vmovsd, then keep trying to form a broadcast.
7397 // That's better than general shuffling and may eliminate a load to GPR and
7398 // move from scalar to vector register.
7399 if (!Ld || NumElts - NumUndefElts != 1)
7400 return SDValue();
7401 unsigned ScalarSize = Ld.getValueSizeInBits();
7402 if (!(UndefElements[0] || (ScalarSize != 32 && ScalarSize != 64)))
7403 return SDValue();
7404 }
7405
7406 bool ConstSplatVal =
7407 (Ld.getOpcode() == ISD::Constant || Ld.getOpcode() == ISD::ConstantFP);
7408 bool IsLoad = ISD::isNormalLoad(Ld.getNode());
7409
7410 // TODO: Handle broadcasts of non-constant sequences.
7411
7412 // Make sure that all of the users of a non-constant load are from the
7413 // BUILD_VECTOR node.
7414 // FIXME: Is the use count needed for non-constant, non-load case?
7415 if (!ConstSplatVal && !IsLoad && !BVOp->isOnlyUserOf(Ld.getNode()))
7416 return SDValue();
7417
7418 unsigned ScalarSize = Ld.getValueSizeInBits();
7419 bool IsGE256 = (VT.getSizeInBits() >= 256);
7420
7421 // When optimizing for size, generate up to 5 extra bytes for a broadcast
7422 // instruction to save 8 or more bytes of constant pool data.
7423 // TODO: If multiple splats are generated to load the same constant,
7424 // it may be detrimental to overall size. There needs to be a way to detect
7425 // that condition to know if this is truly a size win.
7426 bool OptForSize = DAG.shouldOptForSize();
7427
7428 // Handle broadcasting a single constant scalar from the constant pool
7429 // into a vector.
7430 // On Sandybridge (no AVX2), it is still better to load a constant vector
7431 // from the constant pool and not to broadcast it from a scalar.
7432 // But override that restriction when optimizing for size.
7433 // TODO: Check if splatting is recommended for other AVX-capable CPUs.
7434 if (ConstSplatVal && (Subtarget.hasAVX2() || OptForSize)) {
7435 EVT CVT = Ld.getValueType();
7436 assert(!CVT.isVector() && "Must not broadcast a vector type");
7437
7438 // Splat f16, f32, i32, v4f64, v4i64 in all cases with AVX2.
7439 // For size optimization, also splat v2f64 and v2i64, and for size opt
7440 // with AVX2, also splat i8 and i16.
7441 // With pattern matching, the VBROADCAST node may become a VMOVDDUP.
7442 if (ScalarSize == 32 ||
7443 (ScalarSize == 64 && (IsGE256 || Subtarget.hasVLX())) ||
7444 (CVT == MVT::f16 && Subtarget.hasAVX2()) ||
7445 (OptForSize && (ScalarSize == 64 || Subtarget.hasAVX2()))) {
7446 const Constant *C = nullptr;
7447 if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Ld))
7448 C = CI->getConstantIntValue();
7449 else if (ConstantFPSDNode *CF = dyn_cast<ConstantFPSDNode>(Ld))
7450 C = CF->getConstantFPValue();
7451
7452 assert(C && "Invalid constant type");
7453
7454 SDValue CP =
7455 DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout()));
7456 Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign();
7457
7458 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7459 SDValue Ops[] = {DAG.getEntryNode(), CP};
7460 MachinePointerInfo MPI =
7461 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
7462 return DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops, CVT,
7463 MPI, Alignment, MachineMemOperand::MOLoad);
7464 }
7465 }
7466
7467 // Handle AVX2 in-register broadcasts.
7468 if (!IsLoad && Subtarget.hasInt256() &&
7469 (ScalarSize == 32 || (IsGE256 && ScalarSize == 64)))
7470 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7471
7472 // The scalar source must be a normal load.
7473 if (!IsLoad)
7474 return SDValue();
7475
7476 // Make sure the non-chain result is only used by this build vector.
7477 if (!Ld->hasNUsesOfValue(NumElts - NumUndefElts, 0))
7478 return SDValue();
7479
7480 if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) ||
7481 (Subtarget.hasVLX() && ScalarSize == 64)) {
7482 auto *LN = cast<LoadSDNode>(Ld);
7483 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7484 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
7485 SDValue BCast =
7486 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
7487 LN->getMemoryVT(), LN->getMemOperand());
7488 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7489 return BCast;
7490 }
7491
7492 // The integer check is needed for the 64-bit into 128-bit so it doesn't match
7493 // double since there is no vbroadcastsd xmm
7494 if (Subtarget.hasInt256() && Ld.getValueType().isInteger() &&
7495 (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64)) {
7496 auto *LN = cast<LoadSDNode>(Ld);
7497 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
7498 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
7499 SDValue BCast =
7500 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
7501 LN->getMemoryVT(), LN->getMemOperand());
7502 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
7503 return BCast;
7504 }
7505
7506 if (ScalarSize == 16 && Subtarget.hasFP16() && IsGE256)
7507 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7508
7509 // Unsupported broadcast.
7510 return SDValue();
7511}
7512
7513/// For an EXTRACT_VECTOR_ELT with a constant index return the real
7514/// underlying vector and index.
7515///
7516/// Modifies \p ExtractedFromVec to the real vector and returns the real
7517/// index.
7518static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec,
7519 SDValue ExtIdx) {
7520 int Idx = ExtIdx->getAsZExtVal();
7521 if (!isa<ShuffleVectorSDNode>(ExtractedFromVec))
7522 return Idx;
7523
7524 // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already
7525 // lowered this:
7526 // (extract_vector_elt (v8f32 %1), Constant<6>)
7527 // to:
7528 // (extract_vector_elt (vector_shuffle<2,u,u,u>
7529 // (extract_subvector (v8f32 %0), Constant<4>),
7530 // undef)
7531 // Constant<0>)
7532 // In this case the vector is the extract_subvector expression and the index
7533 // is 2, as specified by the shuffle.
7534 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(ExtractedFromVec);
7535 SDValue ShuffleVec = SVOp->getOperand(0);
7536 MVT ShuffleVecVT = ShuffleVec.getSimpleValueType();
7537 assert(ShuffleVecVT.getVectorElementType() ==
7538 ExtractedFromVec.getSimpleValueType().getVectorElementType());
7539
7540 int ShuffleIdx = SVOp->getMaskElt(Idx);
7541 if (isUndefOrInRange(ShuffleIdx, 0, ShuffleVecVT.getVectorNumElements())) {
7542 ExtractedFromVec = ShuffleVec;
7543 return ShuffleIdx;
7544 }
7545 return Idx;
7546}
7547
7548static SDValue buildFromShuffleMostly(SDValue Op, const SDLoc &DL,
7549 SelectionDAG &DAG) {
7550 MVT VT = Op.getSimpleValueType();
7551
7552 // Skip if insert_vec_elt is not supported.
7553 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7554 if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))
7555 return SDValue();
7556
7557 unsigned NumElems = Op.getNumOperands();
7558 SDValue VecIn1;
7559 SDValue VecIn2;
7560 SmallVector<unsigned, 4> InsertIndices;
7561 SmallVector<int, 8> Mask(NumElems, -1);
7562
7563 for (unsigned i = 0; i != NumElems; ++i) {
7564 unsigned Opc = Op.getOperand(i).getOpcode();
7565
7566 if (Opc == ISD::UNDEF)
7567 continue;
7568
7569 if (Opc != ISD::EXTRACT_VECTOR_ELT) {
7570 // Quit if more than 1 elements need inserting.
7571 if (InsertIndices.size() > 1)
7572 return SDValue();
7573
7574 InsertIndices.push_back(i);
7575 continue;
7576 }
7577
7578 SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
7579 SDValue ExtIdx = Op.getOperand(i).getOperand(1);
7580
7581 // Quit if non-constant index.
7582 if (!isa<ConstantSDNode>(ExtIdx))
7583 return SDValue();
7584 int Idx = getUnderlyingExtractedFromVec(ExtractedFromVec, ExtIdx);
7585
7586 // Quit if extracted from vector of different type.
7587 if (ExtractedFromVec.getValueType() != VT)
7588 return SDValue();
7589
7590 if (!VecIn1.getNode())
7591 VecIn1 = ExtractedFromVec;
7592 else if (VecIn1 != ExtractedFromVec) {
7593 if (!VecIn2.getNode())
7594 VecIn2 = ExtractedFromVec;
7595 else if (VecIn2 != ExtractedFromVec)
7596 // Quit if more than 2 vectors to shuffle
7597 return SDValue();
7598 }
7599
7600 if (ExtractedFromVec == VecIn1)
7601 Mask[i] = Idx;
7602 else if (ExtractedFromVec == VecIn2)
7603 Mask[i] = Idx + NumElems;
7604 }
7605
7606 if (!VecIn1.getNode())
7607 return SDValue();
7608
7609 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
7610 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);
7611
7612 for (unsigned Idx : InsertIndices)
7613 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
7614 DAG.getIntPtrConstant(Idx, DL));
7615
7616 return NV;
7617}
7618
7619// Lower BUILD_VECTOR operation for v8bf16, v16bf16 and v32bf16 types.
7620static SDValue LowerBUILD_VECTORvXbf16(SDValue Op, SelectionDAG &DAG,
7621 const X86Subtarget &Subtarget) {
7622 MVT VT = Op.getSimpleValueType();
7623 MVT IVT =
7624 VT.changeVectorElementType(Subtarget.hasFP16() ? MVT::f16 : MVT::i16);
7625 SmallVector<SDValue, 16> NewOps;
7626 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I)
7627 NewOps.push_back(DAG.getBitcast(Subtarget.hasFP16() ? MVT::f16 : MVT::i16,
7628 Op.getOperand(I)));
7629 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps);
7630 return DAG.getBitcast(VT, Res);
7631}
7632
7633// Lower BUILD_VECTOR operation for v8i1 and v16i1 types.
7634static SDValue LowerBUILD_VECTORvXi1(SDValue Op, const SDLoc &dl,
7635 SelectionDAG &DAG,
7636 const X86Subtarget &Subtarget) {
7637
7638 MVT VT = Op.getSimpleValueType();
7639 assert((VT.getVectorElementType() == MVT::i1) &&
7640 "Unexpected type in LowerBUILD_VECTORvXi1!");
7641 if (ISD::isBuildVectorAllZeros(Op.getNode()) ||
7642 ISD::isBuildVectorAllOnes(Op.getNode()))
7643 return Op;
7644
7645 uint64_t Immediate = 0;
7646 SmallVector<unsigned, 16> NonConstIdx;
7647 bool IsSplat = true;
7648 bool HasConstElts = false;
7649 int SplatIdx = -1;
7650 for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
7651 SDValue In = Op.getOperand(idx);
7652 if (In.isUndef())
7653 continue;
7654 if (auto *InC = dyn_cast<ConstantSDNode>(In)) {
7655 Immediate |= (InC->getZExtValue() & 0x1) << idx;
7656 HasConstElts = true;
7657 } else {
7658 NonConstIdx.push_back(idx);
7659 }
7660 if (SplatIdx < 0)
7661 SplatIdx = idx;
7662 else if (In != Op.getOperand(SplatIdx))
7663 IsSplat = false;
7664 }
7665
7666 // for splat use " (select i1 splat_elt, all-ones, all-zeroes)"
7667 if (IsSplat) {
7668 // The build_vector allows the scalar element to be larger than the vector
7669 // element type. We need to mask it to use as a condition unless we know
7670 // the upper bits are zero.
7671 // FIXME: Use computeKnownBits instead of checking specific opcode?
7672 SDValue Cond = Op.getOperand(SplatIdx);
7673 assert(Cond.getValueType() == MVT::i8 && "Unexpected VT!");
7674 if (Cond.getOpcode() != ISD::SETCC)
7675 Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond,
7676 DAG.getConstant(1, dl, MVT::i8));
7677
7678 // Perform the select in the scalar domain so we can use cmov.
7679 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
7680 SDValue Select = DAG.getSelect(dl, MVT::i32, Cond,
7681 DAG.getAllOnesConstant(dl, MVT::i32),
7682 DAG.getConstant(0, dl, MVT::i32));
7683 Select = DAG.getBitcast(MVT::v32i1, Select);
7684 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Select, Select);
7685 } else {
7686 MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
7687 SDValue Select = DAG.getSelect(dl, ImmVT, Cond,
7688 DAG.getAllOnesConstant(dl, ImmVT),
7689 DAG.getConstant(0, dl, ImmVT));
7690 MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;
7691 Select = DAG.getBitcast(VecVT, Select);
7692 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Select,
7693 DAG.getIntPtrConstant(0, dl));
7694 }
7695 }
7696
7697 // insert elements one by one
7698 SDValue DstVec;
7699 if (HasConstElts) {
7700 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
7701 SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);
7702 SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);
7703 ImmL = DAG.getBitcast(MVT::v32i1, ImmL);
7704 ImmH = DAG.getBitcast(MVT::v32i1, ImmH);
7705 DstVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, ImmL, ImmH);
7706 } else {
7707 MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
7708 SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT);
7709 MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;
7710 DstVec = DAG.getBitcast(VecVT, Imm);
7711 DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec,
7712 DAG.getIntPtrConstant(0, dl));
7713 }
7714 } else
7715 DstVec = DAG.getUNDEF(VT);
7716
7717 for (unsigned InsertIdx : NonConstIdx) {
7718 DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
7719 Op.getOperand(InsertIdx),
7720 DAG.getIntPtrConstant(InsertIdx, dl));
7721 }
7722 return DstVec;
7723}
7724
7725LLVM_ATTRIBUTE_UNUSED static bool isHorizOp(unsigned Opcode) {
7726 switch (Opcode) {
7727 case X86ISD::PACKSS:
7728 case X86ISD::PACKUS:
7729 case X86ISD::FHADD:
7730 case X86ISD::FHSUB:
7731 case X86ISD::HADD:
7732 case X86ISD::HSUB:
7733 return true;
7734 }
7735 return false;
7736}
7737
7738/// This is a helper function of LowerToHorizontalOp().
7739/// This function checks that the build_vector \p N in input implements a
7740/// 128-bit partial horizontal operation on a 256-bit vector, but that operation
7741/// may not match the layout of an x86 256-bit horizontal instruction.
7742/// In other words, if this returns true, then some extraction/insertion will
7743/// be required to produce a valid horizontal instruction.
7744///
7745/// Parameter \p Opcode defines the kind of horizontal operation to match.
7746/// For example, if \p Opcode is equal to ISD::ADD, then this function
7747/// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode
7748/// is equal to ISD::SUB, then this function checks if this is a horizontal
7749/// arithmetic sub.
7750///
7751/// This function only analyzes elements of \p N whose indices are
7752/// in range [BaseIdx, LastIdx).
7753///
7754/// TODO: This function was originally used to match both real and fake partial
7755/// horizontal operations, but the index-matching logic is incorrect for that.
7756/// See the corrected implementation in isHopBuildVector(). Can we reduce this
7757/// code because it is only used for partial h-op matching now?
7758static bool isHorizontalBinOpPart(const BuildVectorSDNode *N, unsigned Opcode,
7759 const SDLoc &DL, SelectionDAG &DAG,
7760 unsigned BaseIdx, unsigned LastIdx,
7761 SDValue &V0, SDValue &V1) {
7762 EVT VT = N->getValueType(0);
7763 assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops");
7764 assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!");
7765 assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&
7766 "Invalid Vector in input!");
7767
7768 bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD);
7769 bool CanFold = true;
7770 unsigned ExpectedVExtractIdx = BaseIdx;
7771 unsigned NumElts = LastIdx - BaseIdx;
7772 V0 = DAG.getUNDEF(VT);
7773 V1 = DAG.getUNDEF(VT);
7774
7775 // Check if N implements a horizontal binop.
7776 for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) {
7777 SDValue Op = N->getOperand(i + BaseIdx);
7778
7779 // Skip UNDEFs.
7780 if (Op->isUndef()) {
7781 // Update the expected vector extract index.
7782 if (i * 2 == NumElts)
7783 ExpectedVExtractIdx = BaseIdx;
7784 ExpectedVExtractIdx += 2;
7785 continue;
7786 }
7787
7788 CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
7789
7790 if (!CanFold)
7791 break;
7792
7793 SDValue Op0 = Op.getOperand(0);
7794 SDValue Op1 = Op.getOperand(1);
7795
7796 // Try to match the following pattern:
7797 // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1))
7798 CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7799 Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7800 Op0.getOperand(0) == Op1.getOperand(0) &&
7801 isa<ConstantSDNode>(Op0.getOperand(1)) &&
7802 isa<ConstantSDNode>(Op1.getOperand(1)));
7803 if (!CanFold)
7804 break;
7805
7806 unsigned I0 = Op0.getConstantOperandVal(1);
7807 unsigned I1 = Op1.getConstantOperandVal(1);
7808
7809 if (i * 2 < NumElts) {
7810 if (V0.isUndef()) {
7811 V0 = Op0.getOperand(0);
7812 if (V0.getValueType() != VT)
7813 return false;
7814 }
7815 } else {
7816 if (V1.isUndef()) {
7817 V1 = Op0.getOperand(0);
7818 if (V1.getValueType() != VT)
7819 return false;
7820 }
7821 if (i * 2 == NumElts)
7822 ExpectedVExtractIdx = BaseIdx;
7823 }
7824
7825 SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
7826 if (I0 == ExpectedVExtractIdx)
7827 CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected;
7828 else if (IsCommutable && I1 == ExpectedVExtractIdx) {
7829 // Try to match the following dag sequence:
7830 // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I))
7831 CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected;
7832 } else
7833 CanFold = false;
7834
7835 ExpectedVExtractIdx += 2;
7836 }
7837
7838 return CanFold;
7839}
7840
7841/// Emit a sequence of two 128-bit horizontal add/sub followed by
7842/// a concat_vector.
7843///
7844/// This is a helper function of LowerToHorizontalOp().
7845/// This function expects two 256-bit vectors called V0 and V1.
7846/// At first, each vector is split into two separate 128-bit vectors.
7847/// Then, the resulting 128-bit vectors are used to implement two
7848/// horizontal binary operations.
7849///
7850/// The kind of horizontal binary operation is defined by \p X86Opcode.
7851///
7852/// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to
7853/// the two new horizontal binop.
7854/// When Mode is set, the first horizontal binop dag node would take as input
7855/// the lower 128-bit of V0 and the upper 128-bit of V0. The second
7856/// horizontal binop dag node would take as input the lower 128-bit of V1
7857/// and the upper 128-bit of V1.
7858/// Example:
7859/// HADD V0_LO, V0_HI
7860/// HADD V1_LO, V1_HI
7861///
7862/// Otherwise, the first horizontal binop dag node takes as input the lower
7863/// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop
7864/// dag node takes the upper 128-bit of V0 and the upper 128-bit of V1.
7865/// Example:
7866/// HADD V0_LO, V1_LO
7867/// HADD V0_HI, V1_HI
7868///
7869/// If \p isUndefLO is set, then the algorithm propagates UNDEF to the lower
7870/// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to
7871/// the upper 128-bits of the result.
7872static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
7873 const SDLoc &DL, SelectionDAG &DAG,
7874 unsigned X86Opcode, bool Mode,
7875 bool isUndefLO, bool isUndefHI) {
7876 MVT VT = V0.getSimpleValueType();
7877 assert(VT.is256BitVector() && VT == V1.getSimpleValueType() &&
7878 "Invalid nodes in input!");
7879
7880 unsigned NumElts = VT.getVectorNumElements();
7881 SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
7882 SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
7883 SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
7884 SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
7885 MVT NewVT = V0_LO.getSimpleValueType();
7886
7887 SDValue LO = DAG.getUNDEF(NewVT);
7888 SDValue HI = DAG.getUNDEF(NewVT);
7889
7890 if (Mode) {
7891 // Don't emit a horizontal binop if the result is expected to be UNDEF.
7892 if (!isUndefLO && !V0->isUndef())
7893 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
7894 if (!isUndefHI && !V1->isUndef())
7895 HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
7896 } else {
7897 // Don't emit a horizontal binop if the result is expected to be UNDEF.
7898 if (!isUndefLO && (!V0_LO->isUndef() || !V1_LO->isUndef()))
7899 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
7900
7901 if (!isUndefHI && (!V0_HI->isUndef() || !V1_HI->isUndef()))
7902 HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
7903 }
7904
7905 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
7906}
7907
7908/// Returns true iff \p BV builds a vector with the result equivalent to
7909/// the result of ADDSUB/SUBADD operation.
7910/// If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1
7911/// (SUBADD = Opnd0 -+ Opnd1) operation are written to the parameters
7912/// \p Opnd0 and \p Opnd1.
7913static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
7914 const X86Subtarget &Subtarget, SelectionDAG &DAG,
7915 SDValue &Opnd0, SDValue &Opnd1,
7916 unsigned &NumExtracts,
7917 bool &IsSubAdd) {
7918
7919 MVT VT = BV->getSimpleValueType(0);
7920 if (!Subtarget.hasSSE3() || !VT.isFloatingPoint())
7921 return false;
7922
7923 unsigned NumElts = VT.getVectorNumElements();
7924 SDValue InVec0 = DAG.getUNDEF(VT);
7925 SDValue InVec1 = DAG.getUNDEF(VT);
7926
7927 NumExtracts = 0;
7928
7929 // Odd-numbered elements in the input build vector are obtained from
7930 // adding/subtracting two integer/float elements.
7931 // Even-numbered elements in the input build vector are obtained from
7932 // subtracting/adding two integer/float elements.
7933 unsigned Opc[2] = {0, 0};
7934 for (unsigned i = 0, e = NumElts; i != e; ++i) {
7935 SDValue Op = BV->getOperand(i);
7936
7937 // Skip 'undef' values.
7938 unsigned Opcode = Op.getOpcode();
7939 if (Opcode == ISD::UNDEF)
7940 continue;
7941
7942 // Early exit if we found an unexpected opcode.
7943 if (Opcode != ISD::FADD && Opcode != ISD::FSUB)
7944 return false;
7945
7946 SDValue Op0 = Op.getOperand(0);
7947 SDValue Op1 = Op.getOperand(1);
7948
7949 // Try to match the following pattern:
7950 // (BINOP (extract_vector_elt A, i), (extract_vector_elt B, i))
7951 // Early exit if we cannot match that sequence.
7952 if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7953 Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7954 !isa<ConstantSDNode>(Op0.getOperand(1)) ||
7955 Op0.getOperand(1) != Op1.getOperand(1))
7956 return false;
7957
7958 unsigned I0 = Op0.getConstantOperandVal(1);
7959 if (I0 != i)
7960 return false;
7961
7962 // We found a valid add/sub node, make sure its the same opcode as previous
7963 // elements for this parity.
7964 if (Opc[i % 2] != 0 && Opc[i % 2] != Opcode)
7965 return false;
7966 Opc[i % 2] = Opcode;
7967
7968 // Update InVec0 and InVec1.
7969 if (InVec0.isUndef()) {
7970 InVec0 = Op0.getOperand(0);
7971 if (InVec0.getSimpleValueType() != VT)
7972 return false;
7973 }
7974 if (InVec1.isUndef()) {
7975 InVec1 = Op1.getOperand(0);
7976 if (InVec1.getSimpleValueType() != VT)
7977 return false;
7978 }
7979
7980 // Make sure that operands in input to each add/sub node always
7981 // come from a same pair of vectors.
7982 if (InVec0 != Op0.getOperand(0)) {
7983 if (Opcode == ISD::FSUB)
7984 return false;
7985
7986 // FADD is commutable. Try to commute the operands
7987 // and then test again.
7988 std::swap(Op0, Op1);
7989 if (InVec0 != Op0.getOperand(0))
7990 return false;
7991 }
7992
7993 if (InVec1 != Op1.getOperand(0))
7994 return false;
7995
7996 // Increment the number of extractions done.
7997 ++NumExtracts;
7998 }
7999
8000 // Ensure we have found an opcode for both parities and that they are
8001 // different. Don't try to fold this build_vector into an ADDSUB/SUBADD if the
8002 // inputs are undef.
8003 if (!Opc[0] || !Opc[1] || Opc[0] == Opc[1] ||
8004 InVec0.isUndef() || InVec1.isUndef())
8005 return false;
8006
8007 IsSubAdd = Opc[0] == ISD::FADD;
8008
8009 Opnd0 = InVec0;
8010 Opnd1 = InVec1;
8011 return true;
8012}
8013
8014/// Returns true if is possible to fold MUL and an idiom that has already been
8015/// recognized as ADDSUB/SUBADD(\p Opnd0, \p Opnd1) into
8016/// FMADDSUB/FMSUBADD(x, y, \p Opnd1). If (and only if) true is returned, the
8017/// operands of FMADDSUB/FMSUBADD are written to parameters \p Opnd0, \p Opnd1, \p Opnd2.
8018///
8019/// Prior to calling this function it should be known that there is some
8020/// SDNode that potentially can be replaced with an X86ISD::ADDSUB operation
8021/// using \p Opnd0 and \p Opnd1 as operands. Also, this method is called
8022/// before replacement of such SDNode with ADDSUB operation. Thus the number
8023/// of \p Opnd0 uses is expected to be equal to 2.
8024/// For example, this function may be called for the following IR:
8025/// %AB = fmul fast <2 x double> %A, %B
8026/// %Sub = fsub fast <2 x double> %AB, %C
8027/// %Add = fadd fast <2 x double> %AB, %C
8028/// %Addsub = shufflevector <2 x double> %Sub, <2 x double> %Add,
8029/// <2 x i32> <i32 0, i32 3>
8030/// There is a def for %Addsub here, which potentially can be replaced by
8031/// X86ISD::ADDSUB operation:
8032/// %Addsub = X86ISD::ADDSUB %AB, %C
8033/// and such ADDSUB can further be replaced with FMADDSUB:
8034/// %Addsub = FMADDSUB %A, %B, %C.
8035///
8036/// The main reason why this method is called before the replacement of the
8037/// recognized ADDSUB idiom with ADDSUB operation is that such replacement
8038/// is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit
8039/// FMADDSUB is.
8040static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget,
8041 SelectionDAG &DAG,
8042 SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
8043 unsigned ExpectedUses) {
8044 if (Opnd0.getOpcode() != ISD::FMUL ||
8045 !Opnd0->hasNUsesOfValue(ExpectedUses, 0) || !Subtarget.hasAnyFMA())
8046 return false;
8047
8048 // FIXME: These checks must match the similar ones in
8049 // DAGCombiner::visitFADDForFMACombine. It would be good to have one
8050 // function that would answer if it is Ok to fuse MUL + ADD to FMADD
8051 // or MUL + ADDSUB to FMADDSUB.
8052 const TargetOptions &Options = DAG.getTarget().Options;
8053 bool AllowFusion =
8054 (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath);
8055 if (!AllowFusion)
8056 return false;
8057
8058 Opnd2 = Opnd1;
8059 Opnd1 = Opnd0.getOperand(1);
8060 Opnd0 = Opnd0.getOperand(0);
8061
8062 return true;
8063}
8064
8065/// Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or
8066/// 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or
8067/// X86ISD::FMSUBADD node.
8068static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
8069 const SDLoc &DL,
8070 const X86Subtarget &Subtarget,
8071 SelectionDAG &DAG) {
8072 SDValue Opnd0, Opnd1;
8073 unsigned NumExtracts;
8074 bool IsSubAdd;
8075 if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts,
8076 IsSubAdd))
8077 return SDValue();
8078
8079 MVT VT = BV->getSimpleValueType(0);
8080
8081 // Try to generate X86ISD::FMADDSUB node here.
8082 SDValue Opnd2;
8083 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts)) {
8084 unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
8085 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
8086 }
8087
8088 // We only support ADDSUB.
8089 if (IsSubAdd)
8090 return SDValue();
8091
8092 // There are no known X86 targets with 512-bit ADDSUB instructions!
8093 // Convert to blend(fsub,fadd).
8094 if (VT.is512BitVector()) {
8095 SmallVector<int> Mask;
8096 for (int I = 0, E = VT.getVectorNumElements(); I != E; I += 2) {
8097 Mask.push_back(I);
8098 Mask.push_back(I + E + 1);
8099 }
8100 SDValue Sub = DAG.getNode(ISD::FSUB, DL, VT, Opnd0, Opnd1);
8101 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, Opnd0, Opnd1);
8102 return DAG.getVectorShuffle(VT, DL, Sub, Add, Mask);
8103 }
8104
8105 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
8106}
8107
8108static bool isHopBuildVector(const BuildVectorSDNode *BV, SelectionDAG &DAG,
8109 unsigned &HOpcode, SDValue &V0, SDValue &V1) {
8110 // Initialize outputs to known values.
8111 MVT VT = BV->getSimpleValueType(0);
8112 HOpcode = ISD::DELETED_NODE;
8113 V0 = DAG.getUNDEF(VT);
8114 V1 = DAG.getUNDEF(VT);
8115
8116 // x86 256-bit horizontal ops are defined in a non-obvious way. Each 128-bit
8117 // half of the result is calculated independently from the 128-bit halves of
8118 // the inputs, so that makes the index-checking logic below more complicated.
8119 unsigned NumElts = VT.getVectorNumElements();
8120 unsigned GenericOpcode = ISD::DELETED_NODE;
8121 unsigned Num128BitChunks = VT.is256BitVector() ? 2 : 1;
8122 unsigned NumEltsIn128Bits = NumElts / Num128BitChunks;
8123 unsigned NumEltsIn64Bits = NumEltsIn128Bits / 2;
8124 for (unsigned i = 0; i != Num128BitChunks; ++i) {
8125 for (unsigned j = 0; j != NumEltsIn128Bits; ++j) {
8126 // Ignore undef elements.
8127 SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j);
8128 if (Op.isUndef())
8129 continue;
8130
8131 // If there's an opcode mismatch, we're done.
8132 if (HOpcode != ISD::DELETED_NODE && Op.getOpcode() != GenericOpcode)
8133 return false;
8134
8135 // Initialize horizontal opcode.
8136 if (HOpcode == ISD::DELETED_NODE) {
8137 GenericOpcode = Op.getOpcode();
8138 switch (GenericOpcode) {
8139 // clang-format off
8140 case ISD::ADD: HOpcode = X86ISD::HADD; break;
8141 case ISD::SUB: HOpcode = X86ISD::HSUB; break;
8142 case ISD::FADD: HOpcode = X86ISD::FHADD; break;
8143 case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
8144 default: return false;
8145 // clang-format on
8146 }
8147 }
8148
8149 SDValue Op0 = Op.getOperand(0);
8150 SDValue Op1 = Op.getOperand(1);
8151 if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
8152 Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
8153 Op0.getOperand(0) != Op1.getOperand(0) ||
8154 !isa<ConstantSDNode>(Op0.getOperand(1)) ||
8155 !isa<ConstantSDNode>(Op1.getOperand(1)) || !Op.hasOneUse())
8156 return false;
8157
8158 // The source vector is chosen based on which 64-bit half of the
8159 // destination vector is being calculated.
8160 if (j < NumEltsIn64Bits) {
8161 if (V0.isUndef())
8162 V0 = Op0.getOperand(0);
8163 } else {
8164 if (V1.isUndef())
8165 V1 = Op0.getOperand(0);
8166 }
8167
8168 SDValue SourceVec = (j < NumEltsIn64Bits) ? V0 : V1;
8169 if (SourceVec != Op0.getOperand(0))
8170 return false;
8171
8172 // op (extract_vector_elt A, I), (extract_vector_elt A, I+1)
8173 unsigned ExtIndex0 = Op0.getConstantOperandVal(1);
8174 unsigned ExtIndex1 = Op1.getConstantOperandVal(1);
8175 unsigned ExpectedIndex = i * NumEltsIn128Bits +
8176 (j % NumEltsIn64Bits) * 2;
8177 if (ExpectedIndex == ExtIndex0 && ExtIndex1 == ExtIndex0 + 1)
8178 continue;
8179
8180 // If this is not a commutative op, this does not match.
8181 if (GenericOpcode != ISD::ADD && GenericOpcode != ISD::FADD)
8182 return false;
8183
8184 // Addition is commutative, so try swapping the extract indexes.
8185 // op (extract_vector_elt A, I+1), (extract_vector_elt A, I)
8186 if (ExpectedIndex == ExtIndex1 && ExtIndex0 == ExtIndex1 + 1)
8187 continue;
8188
8189 // Extract indexes do not match horizontal requirement.
8190 return false;
8191 }
8192 }
8193 // We matched. Opcode and operands are returned by reference as arguments.
8194 return true;
8195}
8196
8197static SDValue getHopForBuildVector(const BuildVectorSDNode *BV,
8198 const SDLoc &DL, SelectionDAG &DAG,
8199 unsigned HOpcode, SDValue V0, SDValue V1) {
8200 // If either input vector is not the same size as the build vector,
8201 // extract/insert the low bits to the correct size.
8202 // This is free (examples: zmm --> xmm, xmm --> ymm).
8203 MVT VT = BV->getSimpleValueType(0);
8204 unsigned Width = VT.getSizeInBits();
8205 if (V0.getValueSizeInBits() > Width)
8206 V0 = extractSubVector(V0, 0, DAG, DL, Width);
8207 else if (V0.getValueSizeInBits() < Width)
8208 V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, DL, Width);
8209
8210 if (V1.getValueSizeInBits() > Width)
8211 V1 = extractSubVector(V1, 0, DAG, DL, Width);
8212 else if (V1.getValueSizeInBits() < Width)
8213 V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, DL, Width);
8214
8215 unsigned NumElts = VT.getVectorNumElements();
8216 APInt DemandedElts = APInt::getAllOnes(NumElts);
8217 for (unsigned i = 0; i != NumElts; ++i)
8218 if (BV->getOperand(i).isUndef())
8219 DemandedElts.clearBit(i);
8220
8221 // If we don't need the upper xmm, then perform as a xmm hop.
8222 unsigned HalfNumElts = NumElts / 2;
8223 if (VT.is256BitVector() && DemandedElts.lshr(HalfNumElts) == 0) {
8224 MVT HalfVT = VT.getHalfNumVectorElementsVT();
8225 V0 = extractSubVector(V0, 0, DAG, DL, 128);
8226 V1 = extractSubVector(V1, 0, DAG, DL, 128);
8227 SDValue Half = DAG.getNode(HOpcode, DL, HalfVT, V0, V1);
8228 return insertSubVector(DAG.getUNDEF(VT), Half, 0, DAG, DL, 256);
8229 }
8230
8231 return DAG.getNode(HOpcode, DL, VT, V0, V1);
8232}
8233
8234/// Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
8235static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV, const SDLoc &DL,
8236 const X86Subtarget &Subtarget,
8237 SelectionDAG &DAG) {
8238 // We need at least 2 non-undef elements to make this worthwhile by default.
8239 unsigned NumNonUndefs =
8240 count_if(BV->op_values(), [](SDValue V) { return !V.isUndef(); });
8241 if (NumNonUndefs < 2)
8242 return SDValue();
8243
8244 // There are 4 sets of horizontal math operations distinguished by type:
8245 // int/FP at 128-bit/256-bit. Each type was introduced with a different
8246 // subtarget feature. Try to match those "native" patterns first.
8247 MVT VT = BV->getSimpleValueType(0);
8248 if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) ||
8249 ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) ||
8250 ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) ||
8251 ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())) {
8252 unsigned HOpcode;
8253 SDValue V0, V1;
8254 if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
8255 return getHopForBuildVector(BV, DL, DAG, HOpcode, V0, V1);
8256 }
8257
8258 // Try harder to match 256-bit ops by using extract/concat.
8259 if (!Subtarget.hasAVX() || !VT.is256BitVector())
8260 return SDValue();
8261
8262 // Count the number of UNDEF operands in the build_vector in input.
8263 unsigned NumElts = VT.getVectorNumElements();
8264 unsigned Half = NumElts / 2;
8265 unsigned NumUndefsLO = 0;
8266 unsigned NumUndefsHI = 0;
8267 for (unsigned i = 0, e = Half; i != e; ++i)
8268 if (BV->getOperand(i)->isUndef())
8269 NumUndefsLO++;
8270
8271 for (unsigned i = Half, e = NumElts; i != e; ++i)
8272 if (BV->getOperand(i)->isUndef())
8273 NumUndefsHI++;
8274
8275 SDValue InVec0, InVec1;
8276 if (VT == MVT::v8i32 || VT == MVT::v16i16) {
8277 SDValue InVec2, InVec3;
8278 unsigned X86Opcode;
8279 bool CanFold = true;
8280
8281 if (isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, 0, Half, InVec0, InVec1) &&
8282 isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, Half, NumElts, InVec2,
8283 InVec3) &&
8284 ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
8285 ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
8286 X86Opcode = X86ISD::HADD;
8287 else if (isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, 0, Half, InVec0,
8288 InVec1) &&
8289 isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, Half, NumElts, InVec2,
8290 InVec3) &&
8291 ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
8292 ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
8293 X86Opcode = X86ISD::HSUB;
8294 else
8295 CanFold = false;
8296
8297 if (CanFold) {
8298 // Do not try to expand this build_vector into a pair of horizontal
8299 // add/sub if we can emit a pair of scalar add/sub.
8300 if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
8301 return SDValue();
8302
8303 // Convert this build_vector into a pair of horizontal binops followed by
8304 // a concat vector. We must adjust the outputs from the partial horizontal
8305 // matching calls above to account for undefined vector halves.
8306 SDValue V0 = InVec0.isUndef() ? InVec2 : InVec0;
8307 SDValue V1 = InVec1.isUndef() ? InVec3 : InVec1;
8308 assert((!V0.isUndef() || !V1.isUndef()) && "Horizontal-op of undefs?");
8309 bool isUndefLO = NumUndefsLO == Half;
8310 bool isUndefHI = NumUndefsHI == Half;
8311 return ExpandHorizontalBinOp(V0, V1, DL, DAG, X86Opcode, false, isUndefLO,
8312 isUndefHI);
8313 }
8314 }
8315
8316 if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
8317 VT == MVT::v16i16) {
8318 unsigned X86Opcode;
8319 if (isHorizontalBinOpPart(BV, ISD::ADD, DL, DAG, 0, NumElts, InVec0,
8320 InVec1))
8321 X86Opcode = X86ISD::HADD;
8322 else if (isHorizontalBinOpPart(BV, ISD::SUB, DL, DAG, 0, NumElts, InVec0,
8323 InVec1))
8324 X86Opcode = X86ISD::HSUB;
8325 else if (isHorizontalBinOpPart(BV, ISD::FADD, DL, DAG, 0, NumElts, InVec0,
8326 InVec1))
8327 X86Opcode = X86ISD::FHADD;
8328 else if (isHorizontalBinOpPart(BV, ISD::FSUB, DL, DAG, 0, NumElts, InVec0,
8329 InVec1))
8330 X86Opcode = X86ISD::FHSUB;
8331 else
8332 return SDValue();
8333
8334 // Don't try to expand this build_vector into a pair of horizontal add/sub
8335 // if we can simply emit a pair of scalar add/sub.
8336 if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
8337 return SDValue();
8338
8339 // Convert this build_vector into two horizontal add/sub followed by
8340 // a concat vector.
8341 bool isUndefLO = NumUndefsLO == Half;
8342 bool isUndefHI = NumUndefsHI == Half;
8343 return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
8344 isUndefLO, isUndefHI);
8345 }
8346
8347 return SDValue();
8348}
8349
8350static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
8351 SelectionDAG &DAG);
8352
8353/// If a BUILD_VECTOR's source elements all apply the same bit operation and
8354/// one of their operands is constant, lower to a pair of BUILD_VECTOR and
8355/// just apply the bit to the vectors.
8356/// NOTE: Its not in our interest to start make a general purpose vectorizer
8357/// from this, but enough scalar bit operations are created from the later
8358/// legalization + scalarization stages to need basic support.
8359static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op, const SDLoc &DL,
8360 const X86Subtarget &Subtarget,
8361 SelectionDAG &DAG) {
8362 MVT VT = Op->getSimpleValueType(0);
8363 unsigned NumElems = VT.getVectorNumElements();
8364 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8365
8366 // Check that all elements have the same opcode.
8367 // TODO: Should we allow UNDEFS and if so how many?
8368 unsigned Opcode = Op->getOperand(0).getOpcode();
8369 for (unsigned i = 1; i < NumElems; ++i)
8370 if (Opcode != Op->getOperand(i).getOpcode())
8371 return SDValue();
8372
8373 // TODO: We may be able to add support for other Ops (ADD/SUB + shifts).
8374 bool IsShift = false;
8375 switch (Opcode) {
8376 default:
8377 return SDValue();
8378 case ISD::SHL:
8379 case ISD::SRL:
8380 case ISD::SRA:
8381 IsShift = true;
8382 break;
8383 case ISD::AND:
8384 case ISD::XOR:
8385 case ISD::OR:
8386 // Don't do this if the buildvector is a splat - we'd replace one
8387 // constant with an entire vector.
8388 if (Op->getSplatValue())
8389 return SDValue();
8390 if (!TLI.isOperationLegalOrPromote(Opcode, VT))
8391 return SDValue();
8392 break;
8393 }
8394
8395 SmallVector<SDValue, 4> LHSElts, RHSElts;
8396 for (SDValue Elt : Op->ops()) {
8397 SDValue LHS = Elt.getOperand(0);
8398 SDValue RHS = Elt.getOperand(1);
8399
8400 // We expect the canonicalized RHS operand to be the constant.
8401 if (!isa<ConstantSDNode>(RHS))
8402 return SDValue();
8403
8404 // Extend shift amounts.
8405 if (RHS.getValueSizeInBits() != VT.getScalarSizeInBits()) {
8406 if (!IsShift)
8407 return SDValue();
8408 RHS = DAG.getZExtOrTrunc(RHS, DL, VT.getScalarType());
8409 }
8410
8411 LHSElts.push_back(LHS);
8412 RHSElts.push_back(RHS);
8413 }
8414
8415 // Limit to shifts by uniform immediates.
8416 // TODO: Only accept vXi8/vXi64 special cases?
8417 // TODO: Permit non-uniform XOP/AVX2/MULLO cases?
8418 if (IsShift && any_of(RHSElts, [&](SDValue V) { return RHSElts[0] != V; }))
8419 return SDValue();
8420
8421 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
8422 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
8423 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
8424
8425 if (!IsShift)
8426 return Res;
8427
8428 // Immediately lower the shift to ensure the constant build vector doesn't
8429 // get converted to a constant pool before the shift is lowered.
8430 return LowerShift(Res, Subtarget, DAG);
8431}
8432
8433/// Create a vector constant without a load. SSE/AVX provide the bare minimum
8434/// functionality to do this, so it's all zeros, all ones, or some derivation
8435/// that is cheap to calculate.
8436static SDValue materializeVectorConstant(SDValue Op, const SDLoc &DL,
8437 SelectionDAG &DAG,
8438 const X86Subtarget &Subtarget) {
8439 MVT VT = Op.getSimpleValueType();
8440
8441 // Vectors containing all zeros can be matched by pxor and xorps.
8442 if (ISD::isBuildVectorAllZeros(Op.getNode()))
8443 return Op;
8444
8445 // Vectors containing all ones can be matched by pcmpeqd on 128-bit width
8446 // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use
8447 // vpcmpeqd on 256-bit vectors.
8448 if (Subtarget.hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) {
8449 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32)
8450 return Op;
8451
8452 return getOnesVector(VT, DAG, DL);
8453 }
8454
8455 return SDValue();
8456}
8457
8458/// Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute
8459/// from a vector of source values and a vector of extraction indices.
8460/// The vectors might be manipulated to match the type of the permute op.
8461static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
8462 const SDLoc &DL, SelectionDAG &DAG,
8463 const X86Subtarget &Subtarget) {
8464 MVT ShuffleVT = VT;
8465 EVT IndicesVT = EVT(VT).changeVectorElementTypeToInteger();
8466 unsigned NumElts = VT.getVectorNumElements();
8467 unsigned SizeInBits = VT.getSizeInBits();
8468
8469 // Adjust IndicesVec to match VT size.
8470 assert(IndicesVec.getValueType().getVectorNumElements() >= NumElts &&
8471 "Illegal variable permute mask size");
8472 if (IndicesVec.getValueType().getVectorNumElements() > NumElts) {
8473 // Narrow/widen the indices vector to the correct size.
8474 if (IndicesVec.getValueSizeInBits() > SizeInBits)
8475 IndicesVec = extractSubVector(IndicesVec, 0, DAG, SDLoc(IndicesVec),
8476 NumElts * VT.getScalarSizeInBits());
8477 else if (IndicesVec.getValueSizeInBits() < SizeInBits)
8478 IndicesVec = widenSubVector(IndicesVec, false, Subtarget, DAG,
8479 SDLoc(IndicesVec), SizeInBits);
8480 // Zero-extend the index elements within the vector.
8481 if (IndicesVec.getValueType().getVectorNumElements() > NumElts)
8482 IndicesVec = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(IndicesVec),
8483 IndicesVT, IndicesVec);
8484 }
8485 IndicesVec = DAG.getZExtOrTrunc(IndicesVec, SDLoc(IndicesVec), IndicesVT);
8486
8487 // Handle SrcVec that don't match VT type.
8488 if (SrcVec.getValueSizeInBits() != SizeInBits) {
8489 if ((SrcVec.getValueSizeInBits() % SizeInBits) == 0) {
8490 // Handle larger SrcVec by treating it as a larger permute.
8491 unsigned Scale = SrcVec.getValueSizeInBits() / SizeInBits;
8492 VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts);
8493 IndicesVT = EVT(VT).changeVectorElementTypeToInteger();
8494 IndicesVec = widenSubVector(IndicesVT.getSimpleVT(), IndicesVec, false,
8495 Subtarget, DAG, SDLoc(IndicesVec));
8496 SDValue NewSrcVec =
8497 createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
8498 if (NewSrcVec)
8499 return extractSubVector(NewSrcVec, 0, DAG, DL, SizeInBits);
8500 return SDValue();
8501 } else if (SrcVec.getValueSizeInBits() < SizeInBits) {
8502 // Widen smaller SrcVec to match VT.
8503 SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec));
8504 } else
8505 return SDValue();
8506 }
8507
8508 auto ScaleIndices = [&DAG](SDValue Idx, uint64_t Scale) {
8509 assert(isPowerOf2_64(Scale) && "Illegal variable permute shuffle scale");
8510 EVT SrcVT = Idx.getValueType();
8511 unsigned NumDstBits = SrcVT.getScalarSizeInBits() / Scale;
8512 uint64_t IndexScale = 0;
8513 uint64_t IndexOffset = 0;
8514
8515 // If we're scaling a smaller permute op, then we need to repeat the
8516 // indices, scaling and offsetting them as well.
8517 // e.g. v4i32 -> v16i8 (Scale = 4)
8518 // IndexScale = v4i32 Splat(4 << 24 | 4 << 16 | 4 << 8 | 4)
8519 // IndexOffset = v4i32 Splat(3 << 24 | 2 << 16 | 1 << 8 | 0)
8520 for (uint64_t i = 0; i != Scale; ++i) {
8521 IndexScale |= Scale << (i * NumDstBits);
8522 IndexOffset |= i << (i * NumDstBits);
8523 }
8524
8525 Idx = DAG.getNode(ISD::MUL, SDLoc(Idx), SrcVT, Idx,
8526 DAG.getConstant(IndexScale, SDLoc(Idx), SrcVT));
8527 Idx = DAG.getNode(ISD::ADD, SDLoc(Idx), SrcVT, Idx,
8528 DAG.getConstant(IndexOffset, SDLoc(Idx), SrcVT));
8529 return Idx;
8530 };
8531
8532 unsigned Opcode = 0;
8533 switch (VT.SimpleTy) {
8534 default:
8535 break;
8536 case MVT::v16i8:
8537 if (Subtarget.hasSSSE3())
8538 Opcode = X86ISD::PSHUFB;
8539 break;
8540 case MVT::v8i16:
8541 if (Subtarget.hasVLX() && Subtarget.hasBWI())
8542 Opcode = X86ISD::VPERMV;
8543 else if (Subtarget.hasSSSE3()) {
8544 Opcode = X86ISD::PSHUFB;
8545 ShuffleVT = MVT::v16i8;
8546 }
8547 break;
8548 case MVT::v4f32:
8549 case MVT::v4i32:
8550 if (Subtarget.hasAVX()) {
8551 Opcode = X86ISD::VPERMILPV;
8552 ShuffleVT = MVT::v4f32;
8553 } else if (Subtarget.hasSSSE3()) {
8554 Opcode = X86ISD::PSHUFB;
8555 ShuffleVT = MVT::v16i8;
8556 }
8557 break;
8558 case MVT::v2f64:
8559 case MVT::v2i64:
8560 if (Subtarget.hasAVX()) {
8561 // VPERMILPD selects using bit#1 of the index vector, so scale IndicesVec.
8562 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
8563 Opcode = X86ISD::VPERMILPV;
8564 ShuffleVT = MVT::v2f64;
8565 } else if (Subtarget.hasSSE41()) {
8566 // SSE41 can compare v2i64 - select between indices 0 and 1.
8567 return DAG.getSelectCC(
8568 DL, IndicesVec,
8569 getZeroVector(IndicesVT.getSimpleVT(), Subtarget, DAG, DL),
8570 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}),
8571 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}),
8572 ISD::CondCode::SETEQ);
8573 }
8574 break;
8575 case MVT::v32i8:
8576 if (Subtarget.hasVLX() && Subtarget.hasVBMI())
8577 Opcode = X86ISD::VPERMV;
8578 else if (Subtarget.hasXOP()) {
8579 SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL);
8580 SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL);
8581 SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL);
8582 SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL);
8583 return DAG.getNode(
8584 ISD::CONCAT_VECTORS, DL, VT,
8585 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx),
8586 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx));
8587 } else if (Subtarget.hasAVX()) {
8588 SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL);
8589 SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL);
8590 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);
8591 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);
8592 auto PSHUFBBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
8593 ArrayRef<SDValue> Ops) {
8594 // Permute Lo and Hi and then select based on index range.
8595 // This works as SHUFB uses bits[3:0] to permute elements and we don't
8596 // care about the bit[7] as its just an index vector.
8597 SDValue Idx = Ops[2];
8598 EVT VT = Idx.getValueType();
8599 return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT),
8600 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx),
8601 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx),
8602 ISD::CondCode::SETGT);
8603 };
8604 SDValue Ops[] = {LoLo, HiHi, IndicesVec};
8605 return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops,
8606 PSHUFBBuilder);
8607 }
8608 break;
8609 case MVT::v16i16:
8610 if (Subtarget.hasVLX() && Subtarget.hasBWI())
8611 Opcode = X86ISD::VPERMV;
8612 else if (Subtarget.hasAVX()) {
8613 // Scale to v32i8 and perform as v32i8.
8614 IndicesVec = ScaleIndices(IndicesVec, 2);
8615 return DAG.getBitcast(
8616 VT, createVariablePermute(
8617 MVT::v32i8, DAG.getBitcast(MVT::v32i8, SrcVec),
8618 DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget));
8619 }
8620 break;
8621 case MVT::v8f32:
8622 case MVT::v8i32:
8623 if (Subtarget.hasAVX2())
8624 Opcode = X86ISD::VPERMV;
8625 else if (Subtarget.hasAVX()) {
8626 SrcVec = DAG.getBitcast(MVT::v8f32, SrcVec);
8627 SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8628 {0, 1, 2, 3, 0, 1, 2, 3});
8629 SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
8630 {4, 5, 6, 7, 4, 5, 6, 7});
8631 if (Subtarget.hasXOP())
8632 return DAG.getBitcast(
8633 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi,
8634 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
8635 // Permute Lo and Hi and then select based on index range.
8636 // This works as VPERMILPS only uses index bits[0:1] to permute elements.
8637 SDValue Res = DAG.getSelectCC(
8638 DL, IndicesVec, DAG.getConstant(3, DL, MVT::v8i32),
8639 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec),
8640 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec),
8641 ISD::CondCode::SETGT);
8642 return DAG.getBitcast(VT, Res);
8643 }
8644 break;
8645 case MVT::v4i64:
8646 case MVT::v4f64:
8647 if (Subtarget.hasAVX512()) {
8648 if (!Subtarget.hasVLX()) {
8649 MVT WidenSrcVT = MVT::getVectorVT(VT.getScalarType(), 8);
8650 SrcVec = widenSubVector(WidenSrcVT, SrcVec, false, Subtarget, DAG,
8651 SDLoc(SrcVec));
8652 IndicesVec = widenSubVector(MVT::v8i64, IndicesVec, false, Subtarget,
8653 DAG, SDLoc(IndicesVec));
8654 SDValue Res = createVariablePermute(WidenSrcVT, SrcVec, IndicesVec, DL,
8655 DAG, Subtarget);
8656 return extract256BitVector(Res, 0, DAG, DL);
8657 }
8658 Opcode = X86ISD::VPERMV;
8659 } else if (Subtarget.hasAVX()) {
8660 SrcVec = DAG.getBitcast(MVT::v4f64, SrcVec);
8661 SDValue LoLo =
8662 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {0, 1, 0, 1});
8663 SDValue HiHi =
8664 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {2, 3, 2, 3});
8665 // VPERMIL2PD selects with bit#1 of the index vector, so scale IndicesVec.
8666 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
8667 if (Subtarget.hasXOP())
8668 return DAG.getBitcast(
8669 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi,
8670 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
8671 // Permute Lo and Hi and then select based on index range.
8672 // This works as VPERMILPD only uses index bit[1] to permute elements.
8673 SDValue Res = DAG.getSelectCC(
8674 DL, IndicesVec, DAG.getConstant(2, DL, MVT::v4i64),
8675 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec),
8676 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec),
8677 ISD::CondCode::SETGT);
8678 return DAG.getBitcast(VT, Res);
8679 }
8680 break;
8681 case MVT::v64i8:
8682 if (Subtarget.hasVBMI())
8683 Opcode = X86ISD::VPERMV;
8684 break;
8685 case MVT::v32i16:
8686 if (Subtarget.hasBWI())
8687 Opcode = X86ISD::VPERMV;
8688 break;
8689 case MVT::v16f32:
8690 case MVT::v16i32:
8691 case MVT::v8f64:
8692 case MVT::v8i64:
8693 if (Subtarget.hasAVX512())
8694 Opcode = X86ISD::VPERMV;
8695 break;
8696 }
8697 if (!Opcode)
8698 return SDValue();
8699
8700 assert((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) &&
8701 (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 &&
8702 "Illegal variable permute shuffle type");
8703
8704 uint64_t Scale = VT.getScalarSizeInBits() / ShuffleVT.getScalarSizeInBits();
8705 if (Scale > 1)
8706 IndicesVec = ScaleIndices(IndicesVec, Scale);
8707
8708 EVT ShuffleIdxVT = EVT(ShuffleVT).changeVectorElementTypeToInteger();
8709 IndicesVec = DAG.getBitcast(ShuffleIdxVT, IndicesVec);
8710
8711 SrcVec = DAG.getBitcast(ShuffleVT, SrcVec);
8712 SDValue Res = Opcode == X86ISD::VPERMV
8713 ? DAG.getNode(Opcode, DL, ShuffleVT, IndicesVec, SrcVec)
8714 : DAG.getNode(Opcode, DL, ShuffleVT, SrcVec, IndicesVec);
8715 return DAG.getBitcast(VT, Res);
8716}
8717
8718// Tries to lower a BUILD_VECTOR composed of extract-extract chains that can be
8719// reasoned to be a permutation of a vector by indices in a non-constant vector.
8720// (build_vector (extract_elt V, (extract_elt I, 0)),
8721// (extract_elt V, (extract_elt I, 1)),
8722// ...
8723// ->
8724// (vpermv I, V)
8725//
8726// TODO: Handle undefs
8727// TODO: Utilize pshufb and zero mask blending to support more efficient
8728// construction of vectors with constant-0 elements.
8729static SDValue
8730LowerBUILD_VECTORAsVariablePermute(SDValue V, const SDLoc &DL,
8731 SelectionDAG &DAG,
8732 const X86Subtarget &Subtarget) {
8733 SDValue SrcVec, IndicesVec;
8734 // Check for a match of the permute source vector and permute index elements.
8735 // This is done by checking that the i-th build_vector operand is of the form:
8736 // (extract_elt SrcVec, (extract_elt IndicesVec, i)).
8737 for (unsigned Idx = 0, E = V.getNumOperands(); Idx != E; ++Idx) {
8738 SDValue Op = V.getOperand(Idx);
8739 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
8740 return SDValue();
8741
8742 // If this is the first extract encountered in V, set the source vector,
8743 // otherwise verify the extract is from the previously defined source
8744 // vector.
8745 if (!SrcVec)
8746 SrcVec = Op.getOperand(0);
8747 else if (SrcVec != Op.getOperand(0))
8748 return SDValue();
8749 SDValue ExtractedIndex = Op->getOperand(1);
8750 // Peek through extends.
8751 if (ExtractedIndex.getOpcode() == ISD::ZERO_EXTEND ||
8752 ExtractedIndex.getOpcode() == ISD::SIGN_EXTEND)
8753 ExtractedIndex = ExtractedIndex.getOperand(0);
8754 if (ExtractedIndex.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
8755 return SDValue();
8756
8757 // If this is the first extract from the index vector candidate, set the
8758 // indices vector, otherwise verify the extract is from the previously
8759 // defined indices vector.
8760 if (!IndicesVec)
8761 IndicesVec = ExtractedIndex.getOperand(0);
8762 else if (IndicesVec != ExtractedIndex.getOperand(0))
8763 return SDValue();
8764
8765 auto *PermIdx = dyn_cast<ConstantSDNode>(ExtractedIndex.getOperand(1));
8766 if (!PermIdx || PermIdx->getAPIntValue() != Idx)
8767 return SDValue();
8768 }
8769
8770 MVT VT = V.getSimpleValueType();
8771 return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
8772}
8773
8774SDValue
8775X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
8776 SDLoc dl(Op);
8777
8778 MVT VT = Op.getSimpleValueType();
8779 MVT EltVT = VT.getVectorElementType();
8780 MVT OpEltVT = Op.getOperand(0).getSimpleValueType();
8781 unsigned NumElems = Op.getNumOperands();
8782
8783 // Generate vectors for predicate vectors.
8784 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512())
8785 return LowerBUILD_VECTORvXi1(Op, dl, DAG, Subtarget);
8786
8787 if (VT.getVectorElementType() == MVT::bf16 &&
8788 (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16()))
8789 return LowerBUILD_VECTORvXbf16(Op, DAG, Subtarget);
8790
8791 if (SDValue VectorCst = materializeVectorConstant(Op, dl, DAG, Subtarget))
8792 return VectorCst;
8793
8794 unsigned EVTBits = EltVT.getSizeInBits();
8795 APInt UndefMask = APInt::getZero(NumElems);
8796 APInt FrozenUndefMask = APInt::getZero(NumElems);
8797 APInt ZeroMask = APInt::getZero(NumElems);
8798 APInt NonZeroMask = APInt::getZero(NumElems);
8799 bool IsAllConstants = true;
8800 bool OneUseFrozenUndefs = true;
8801 SmallSet<SDValue, 8> Values;
8802 unsigned NumConstants = NumElems;
8803 for (unsigned i = 0; i < NumElems; ++i) {
8804 SDValue Elt = Op.getOperand(i);
8805 if (Elt.isUndef()) {
8806 UndefMask.setBit(i);
8807 continue;
8808 }
8809 if (ISD::isFreezeUndef(Elt.getNode())) {
8810 OneUseFrozenUndefs = OneUseFrozenUndefs && Elt->hasOneUse();
8811 FrozenUndefMask.setBit(i);
8812 continue;
8813 }
8814 Values.insert(Elt);
8815 if (!isIntOrFPConstant(Elt)) {
8816 IsAllConstants = false;
8817 NumConstants--;
8818 }
8819 if (X86::isZeroNode(Elt)) {
8820 ZeroMask.setBit(i);
8821 } else {
8822 NonZeroMask.setBit(i);
8823 }
8824 }
8825
8826 // All undef vector. Return an UNDEF.
8827 if (UndefMask.isAllOnes())
8828 return DAG.getUNDEF(VT);
8829
8830 // All undef/freeze(undef) vector. Return a FREEZE UNDEF.
8831 if (OneUseFrozenUndefs && (UndefMask | FrozenUndefMask).isAllOnes())
8832 return DAG.getFreeze(DAG.getUNDEF(VT));
8833
8834 // All undef/freeze(undef)/zero vector. Return a zero vector.
8835 if ((UndefMask | FrozenUndefMask | ZeroMask).isAllOnes())
8836 return getZeroVector(VT, Subtarget, DAG, dl);
8837
8838 // If we have multiple FREEZE-UNDEF operands, we are likely going to end up
8839 // lowering into a suboptimal insertion sequence. Instead, thaw the UNDEF in
8840 // our source BUILD_VECTOR, create another FREEZE-UNDEF splat BUILD_VECTOR,
8841 // and blend the FREEZE-UNDEF operands back in.
8842 // FIXME: is this worthwhile even for a single FREEZE-UNDEF operand?
8843 if (unsigned NumFrozenUndefElts = FrozenUndefMask.popcount();
8844 NumFrozenUndefElts >= 2 && NumFrozenUndefElts < NumElems) {
8845 SmallVector<int, 16> BlendMask(NumElems, -1);
8846 SmallVector<SDValue, 16> Elts(NumElems, DAG.getUNDEF(OpEltVT));
8847 for (unsigned i = 0; i < NumElems; ++i) {
8848 if (UndefMask[i]) {
8849 BlendMask[i] = -1;
8850 continue;
8851 }
8852 BlendMask[i] = i;
8853 if (!FrozenUndefMask[i])
8854 Elts[i] = Op.getOperand(i);
8855 else
8856 BlendMask[i] += NumElems;
8857 }
8858 SDValue EltsBV = DAG.getBuildVector(VT, dl, Elts);
8859 SDValue FrozenUndefElt = DAG.getFreeze(DAG.getUNDEF(OpEltVT));
8860 SDValue FrozenUndefBV = DAG.getSplatBuildVector(VT, dl, FrozenUndefElt);
8861 return DAG.getVectorShuffle(VT, dl, EltsBV, FrozenUndefBV, BlendMask);
8862 }
8863
8864 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode());
8865
8866 // If the upper elts of a ymm/zmm are undef/freeze(undef)/zero then we might
8867 // be better off lowering to a smaller build vector and padding with
8868 // undef/zero.
8869 if ((VT.is256BitVector() || VT.is512BitVector()) &&
8870 !isFoldableUseOfShuffle(BV)) {
8871 unsigned UpperElems = NumElems / 2;
8872 APInt UndefOrZeroMask = FrozenUndefMask | UndefMask | ZeroMask;
8873 unsigned NumUpperUndefsOrZeros = UndefOrZeroMask.countl_one();
8874 if (NumUpperUndefsOrZeros >= UpperElems) {
8875 if (VT.is512BitVector() &&
8876 NumUpperUndefsOrZeros >= (NumElems - (NumElems / 4)))
8877 UpperElems = NumElems - (NumElems / 4);
8878 // If freeze(undef) is in any upper elements, force to zero.
8879 bool UndefUpper = UndefMask.countl_one() >= UpperElems;
8880 MVT LowerVT = MVT::getVectorVT(EltVT, NumElems - UpperElems);
8881 SDValue NewBV =
8882 DAG.getBuildVector(LowerVT, dl, Op->ops().drop_back(UpperElems));
8883 return widenSubVector(VT, NewBV, !UndefUpper, Subtarget, DAG, dl);
8884 }
8885 }
8886
8887 if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, dl, Subtarget, DAG))
8888 return AddSub;
8889 if (SDValue HorizontalOp = LowerToHorizontalOp(BV, dl, Subtarget, DAG))
8890 return HorizontalOp;
8891 if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, dl, Subtarget, DAG))
8892 return Broadcast;
8893 if (SDValue BitOp = lowerBuildVectorToBitOp(BV, dl, Subtarget, DAG))
8894 return BitOp;
8895
8896 unsigned NumZero = ZeroMask.popcount();
8897 unsigned NumNonZero = NonZeroMask.popcount();
8898
8899 // If we are inserting one variable into a vector of non-zero constants, try
8900 // to avoid loading each constant element as a scalar. Load the constants as a
8901 // vector and then insert the variable scalar element. If insertion is not
8902 // supported, fall back to a shuffle to get the scalar blended with the
8903 // constants. Insertion into a zero vector is handled as a special-case
8904 // somewhere below here.
8905 if (NumConstants == NumElems - 1 && NumNonZero != 1 &&
8906 FrozenUndefMask.isZero() &&
8907 (isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT) ||
8908 isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) {
8909 // Create an all-constant vector. The variable element in the old
8910 // build vector is replaced by undef in the constant vector. Save the
8911 // variable scalar element and its index for use in the insertelement.
8912 LLVMContext &Context = *DAG.getContext();
8913 Type *EltType = Op.getValueType().getScalarType().getTypeForEVT(Context);
8914 SmallVector<Constant *, 16> ConstVecOps(NumElems, UndefValue::get(EltType));
8915 SDValue VarElt;
8916 SDValue InsIndex;
8917 for (unsigned i = 0; i != NumElems; ++i) {
8918 SDValue Elt = Op.getOperand(i);
8919 if (auto *C = dyn_cast<ConstantSDNode>(Elt))
8920 ConstVecOps[i] = ConstantInt::get(Context, C->getAPIntValue());
8921 else if (auto *C = dyn_cast<ConstantFPSDNode>(Elt))
8922 ConstVecOps[i] = ConstantFP::get(Context, C->getValueAPF());
8923 else if (!Elt.isUndef()) {
8924 assert(!VarElt.getNode() && !InsIndex.getNode() &&
8925 "Expected one variable element in this vector");
8926 VarElt = Elt;
8927 InsIndex = DAG.getVectorIdxConstant(i, dl);
8928 }
8929 }
8930 Constant *CV = ConstantVector::get(ConstVecOps);
8931 SDValue DAGConstVec = DAG.getConstantPool(CV, VT);
8932
8933 // The constants we just created may not be legal (eg, floating point). We
8934 // must lower the vector right here because we can not guarantee that we'll
8935 // legalize it before loading it. This is also why we could not just create
8936 // a new build vector here. If the build vector contains illegal constants,
8937 // it could get split back up into a series of insert elements.
8938 // TODO: Improve this by using shorter loads with broadcast/VZEXT_LOAD.
8939 SDValue LegalDAGConstVec = LowerConstantPool(DAGConstVec, DAG);
8940 MachineFunction &MF = DAG.getMachineFunction();
8941 MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(MF);
8942 SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI);
8943 unsigned InsertC = InsIndex->getAsZExtVal();
8944 unsigned NumEltsInLow128Bits = 128 / VT.getScalarSizeInBits();
8945 if (InsertC < NumEltsInLow128Bits)
8946 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex);
8947
8948 // There's no good way to insert into the high elements of a >128-bit
8949 // vector, so use shuffles to avoid an extract/insert sequence.
8950 assert(VT.getSizeInBits() > 128 && "Invalid insertion index?");
8951 assert(Subtarget.hasAVX() && "Must have AVX with >16-byte vector");
8952 SmallVector<int, 8> ShuffleMask;
8953 unsigned NumElts = VT.getVectorNumElements();
8954 for (unsigned i = 0; i != NumElts; ++i)
8955 ShuffleMask.push_back(i == InsertC ? NumElts : i);
8956 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt);
8957 return DAG.getVectorShuffle(VT, dl, Ld, S2V, ShuffleMask);
8958 }
8959
8960 // Special case for single non-zero, non-undef, element.
8961 if (NumNonZero == 1) {
8962 unsigned Idx = NonZeroMask.countr_zero();
8963 SDValue Item = Op.getOperand(Idx);
8964
8965 // If we have a constant or non-constant insertion into the low element of
8966 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into
8967 // the rest of the elements. This will be matched as movd/movq/movss/movsd
8968 // depending on what the source datatype is.
8969 if (Idx == 0) {
8970 if (NumZero == 0)
8971 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
8972
8973 if (EltVT == MVT::i32 || EltVT == MVT::f16 || EltVT == MVT::f32 ||
8974 EltVT == MVT::f64 || (EltVT == MVT::i64 && Subtarget.is64Bit()) ||
8975 (EltVT == MVT::i16 && Subtarget.hasFP16())) {
8976 assert((VT.is128BitVector() || VT.is256BitVector() ||
8977 VT.is512BitVector()) &&
8978 "Expected an SSE value type!");
8979 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
8980 // Turn it into a MOVL (i.e. movsh, movss, movsd, movw or movd) to a
8981 // zero vector.
8982 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
8983 }
8984
8985 // We can't directly insert an i8 or i16 into a vector, so zero extend
8986 // it to i32 first.
8987 if (EltVT == MVT::i16 || EltVT == MVT::i8) {
8988 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
8989 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
8990 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, Item);
8991 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
8992 return DAG.getBitcast(VT, Item);
8993 }
8994 }
8995
8996 // Is it a vector logical left shift?
8997 if (NumElems == 2 && Idx == 1 &&
8998 X86::isZeroNode(Op.getOperand(0)) &&
8999 !X86::isZeroNode(Op.getOperand(1))) {
9000 unsigned NumBits = VT.getSizeInBits();
9001 return getVShift(true, VT,
9002 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
9003 VT, Op.getOperand(1)),
9004 NumBits/2, DAG, *this, dl);
9005 }
9006
9007 if (IsAllConstants) // Otherwise, it's better to do a constpool load.
9008 return SDValue();
9009
9010 // Otherwise, if this is a vector with i32 or f32 elements, and the element
9011 // is a non-constant being inserted into an element other than the low one,
9012 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka
9013 // movd/movss) to move this into the low element, then shuffle it into
9014 // place.
9015 if (EVTBits == 32) {
9016 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
9017 return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG);
9018 }
9019 }
9020
9021 // Splat is obviously ok. Let legalizer expand it to a shuffle.
9022 if (Values.size() == 1) {
9023 if (EVTBits == 32) {
9024 // Instead of a shuffle like this:
9025 // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0>
9026 // Check if it's possible to issue this instead.
9027 // shuffle (vload ptr)), undef, <1, 1, 1, 1>
9028 unsigned Idx = NonZeroMask.countr_zero();
9029 SDValue Item = Op.getOperand(Idx);
9030 if (Op.getNode()->isOnlyUserOf(Item.getNode()))
9031 return LowerAsSplatVectorLoad(Item, VT, dl, DAG);
9032 }
9033 return SDValue();
9034 }
9035
9036 // A vector full of immediates; various special cases are already
9037 // handled, so this is best done with a single constant-pool load.
9038 if (IsAllConstants)
9039 return SDValue();
9040
9041 if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, dl, DAG, Subtarget))
9042 return V;
9043
9044 // See if we can use a vector load to get all of the elements.
9045 {
9046 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
9047 if (SDValue LD =
9048 EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false))
9049 return LD;
9050 }
9051
9052 // If this is a splat of pairs of 32-bit elements, we can use a narrower
9053 // build_vector and broadcast it.
9054 // TODO: We could probably generalize this more.
9055 if (Subtarget.hasAVX2() && EVTBits == 32 && Values.size() == 2) {
9056 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
9057 DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
9058 auto CanSplat = [](SDValue Op, unsigned NumElems, ArrayRef<SDValue> Ops) {
9059 // Make sure all the even/odd operands match.
9060 for (unsigned i = 2; i != NumElems; ++i)
9061 if (Ops[i % 2] != Op.getOperand(i))
9062 return false;
9063 return true;
9064 };
9065 if (CanSplat(Op, NumElems, Ops)) {
9066 MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64;
9067 MVT NarrowVT = MVT::getVectorVT(EltVT, 4);
9068 // Create a new build vector and cast to v2i64/v2f64.
9069 SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),
9070 DAG.getBuildVector(NarrowVT, dl, Ops));
9071 // Broadcast from v2i64/v2f64 and cast to final VT.
9072 MVT BcastVT = MVT::getVectorVT(WideEltVT, NumElems / 2);
9073 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,
9074 NewBV));
9075 }
9076 }
9077
9078 // For AVX-length vectors, build the individual 128-bit pieces and use
9079 // shuffles to put them in place.
9080 if (VT.getSizeInBits() > 128) {
9081 MVT HVT = MVT::getVectorVT(EltVT, NumElems / 2);
9082
9083 // Build both the lower and upper subvector.
9084 SDValue Lower =
9085 DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2));
9086 SDValue Upper = DAG.getBuildVector(
9087 HVT, dl, Op->ops().slice(NumElems / 2, NumElems /2));
9088
9089 // Recreate the wider vector with the lower and upper part.
9090 return concatSubVectors(Lower, Upper, DAG, dl);
9091 }
9092
9093 // Let legalizer expand 2-wide build_vectors.
9094 if (EVTBits == 64) {
9095 if (NumNonZero == 1) {
9096 // One half is zero or undef.
9097 unsigned Idx = NonZeroMask.countr_zero();
9098 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
9099 Op.getOperand(Idx));
9100 return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
9101 }
9102 return SDValue();
9103 }
9104
9105 // If element VT is < 32 bits, convert it to inserts into a zero vector.
9106 if (EVTBits == 8 && NumElems == 16)
9107 if (SDValue V = LowerBuildVectorv16i8(Op, dl, NonZeroMask, NumNonZero,
9108 NumZero, DAG, Subtarget))
9109 return V;
9110
9111 if (EltVT == MVT::i16 && NumElems == 8)
9112 if (SDValue V = LowerBuildVectorv8i16(Op, dl, NonZeroMask, NumNonZero,
9113 NumZero, DAG, Subtarget))
9114 return V;
9115
9116 // If element VT is == 32 bits and has 4 elems, try to generate an INSERTPS
9117 if (EVTBits == 32 && NumElems == 4)
9118 if (SDValue V = LowerBuildVectorv4x32(Op, dl, DAG, Subtarget))
9119 return V;
9120
9121 // If element VT is == 32 bits, turn it into a number of shuffles.
9122 if (NumElems == 4 && NumZero > 0) {
9123 SmallVector<SDValue, 8> Ops(NumElems);
9124 for (unsigned i = 0; i < 4; ++i) {
9125 bool isZero = !NonZeroMask[i];
9126 if (isZero)
9127 Ops[i] = getZeroVector(VT, Subtarget, DAG, dl);
9128 else
9129 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
9130 }
9131
9132 for (unsigned i = 0; i < 2; ++i) {
9133 switch (NonZeroMask.extractBitsAsZExtValue(2, i * 2)) {
9134 default: llvm_unreachable("Unexpected NonZero count");
9135 case 0:
9136 Ops[i] = Ops[i*2]; // Must be a zero vector.
9137 break;
9138 case 1:
9139 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]);
9140 break;
9141 case 2:
9142 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
9143 break;
9144 case 3:
9145 Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
9146 break;
9147 }
9148 }
9149
9150 bool Reverse1 = NonZeroMask.extractBitsAsZExtValue(2, 0) == 2;
9151 bool Reverse2 = NonZeroMask.extractBitsAsZExtValue(2, 2) == 2;
9152 int MaskVec[] = {
9153 Reverse1 ? 1 : 0,
9154 Reverse1 ? 0 : 1,
9155 static_cast<int>(Reverse2 ? NumElems+1 : NumElems),
9156 static_cast<int>(Reverse2 ? NumElems : NumElems+1)
9157 };
9158 return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec);
9159 }
9160
9161 assert(Values.size() > 1 && "Expected non-undef and non-splat vector");
9162
9163 // Check for a build vector from mostly shuffle plus few inserting.
9164 if (SDValue Sh = buildFromShuffleMostly(Op, dl, DAG))
9165 return Sh;
9166
9167 // For SSE 4.1, use insertps to put the high elements into the low element.
9168 if (Subtarget.hasSSE41() && EltVT != MVT::f16) {
9169 SDValue Result;
9170 if (!Op.getOperand(0).isUndef())
9171 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
9172 else
9173 Result = DAG.getUNDEF(VT);
9174
9175 for (unsigned i = 1; i < NumElems; ++i) {
9176 if (Op.getOperand(i).isUndef()) continue;
9177 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
9178 Op.getOperand(i), DAG.getIntPtrConstant(i, dl));
9179 }
9180 return Result;
9181 }
9182
9183 // Otherwise, expand into a number of unpckl*, start by extending each of
9184 // our (non-undef) elements to the full vector width with the element in the
9185 // bottom slot of the vector (which generates no code for SSE).
9186 SmallVector<SDValue, 8> Ops(NumElems);
9187 for (unsigned i = 0; i < NumElems; ++i) {
9188 if (!Op.getOperand(i).isUndef())
9189 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
9190 else
9191 Ops[i] = DAG.getUNDEF(VT);
9192 }
9193
9194 // Next, we iteratively mix elements, e.g. for v4f32:
9195 // Step 1: unpcklps 0, 1 ==> X: <?, ?, 1, 0>
9196 // : unpcklps 2, 3 ==> Y: <?, ?, 3, 2>
9197 // Step 2: unpcklpd X, Y ==> <3, 2, 1, 0>
9198 for (unsigned Scale = 1; Scale < NumElems; Scale *= 2) {
9199 // Generate scaled UNPCKL shuffle mask.
9200 SmallVector<int, 16> Mask;
9201 for(unsigned i = 0; i != Scale; ++i)
9202 Mask.push_back(i);
9203 for (unsigned i = 0; i != Scale; ++i)
9204 Mask.push_back(NumElems+i);
9205 Mask.append(NumElems - Mask.size(), SM_SentinelUndef);
9206
9207 for (unsigned i = 0, e = NumElems / (2 * Scale); i != e; ++i)
9208 Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask);
9209 }
9210 return Ops[0];
9211}
9212
9213// 256-bit AVX can use the vinsertf128 instruction
9214// to create 256-bit vectors from two other 128-bit ones.
9215// TODO: Detect subvector broadcast here instead of DAG combine?
9216static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
9217 const X86Subtarget &Subtarget) {
9218 SDLoc dl(Op);
9219 MVT ResVT = Op.getSimpleValueType();
9220
9221 assert((ResVT.is256BitVector() ||
9222 ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide");
9223
9224 unsigned NumOperands = Op.getNumOperands();
9225 unsigned NumFreezeUndef = 0;
9226 unsigned NumZero = 0;
9227 unsigned NumNonZero = 0;
9228 unsigned NonZeros = 0;
9229 for (unsigned i = 0; i != NumOperands; ++i) {
9230 SDValue SubVec = Op.getOperand(i);
9231 if (SubVec.isUndef())
9232 continue;
9233 if (ISD::isFreezeUndef(SubVec.getNode())) {
9234 // If the freeze(undef) has multiple uses then we must fold to zero.
9235 if (SubVec.hasOneUse())
9236 ++NumFreezeUndef;
9237 else
9238 ++NumZero;
9239 }
9240 else if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
9241 ++NumZero;
9242 else {
9243 assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
9244 NonZeros |= 1 << i;
9245 ++NumNonZero;
9246 }
9247 }
9248
9249 // If we have more than 2 non-zeros, build each half separately.
9250 if (NumNonZero > 2) {
9251 MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
9252 ArrayRef<SDUse> Ops = Op->ops();
9253 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9254 Ops.slice(0, NumOperands/2));
9255 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9256 Ops.slice(NumOperands/2));
9257 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
9258 }
9259
9260 // Otherwise, build it up through insert_subvectors.
9261 SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl)
9262 : (NumFreezeUndef ? DAG.getFreeze(DAG.getUNDEF(ResVT))
9263 : DAG.getUNDEF(ResVT));
9264
9265 MVT SubVT = Op.getOperand(0).getSimpleValueType();
9266 unsigned NumSubElems = SubVT.getVectorNumElements();
9267 for (unsigned i = 0; i != NumOperands; ++i) {
9268 if ((NonZeros & (1 << i)) == 0)
9269 continue;
9270
9271 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec,
9272 Op.getOperand(i),
9273 DAG.getIntPtrConstant(i * NumSubElems, dl));
9274 }
9275
9276 return Vec;
9277}
9278
9279// Returns true if the given node is a type promotion (by concatenating i1
9280// zeros) of the result of a node that already zeros all upper bits of
9281// k-register.
9282// TODO: Merge this with LowerAVXCONCAT_VECTORS?
9283static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
9284 const X86Subtarget &Subtarget,
9285 SelectionDAG & DAG) {
9286 SDLoc dl(Op);
9287 MVT ResVT = Op.getSimpleValueType();
9288 unsigned NumOperands = Op.getNumOperands();
9289
9290 assert(NumOperands > 1 && isPowerOf2_32(NumOperands) &&
9291 "Unexpected number of operands in CONCAT_VECTORS");
9292
9293 uint64_t Zeros = 0;
9294 uint64_t NonZeros = 0;
9295 for (unsigned i = 0; i != NumOperands; ++i) {
9296 SDValue SubVec = Op.getOperand(i);
9297 if (SubVec.isUndef())
9298 continue;
9299 assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
9300 if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
9301 Zeros |= (uint64_t)1 << i;
9302 else
9303 NonZeros |= (uint64_t)1 << i;
9304 }
9305
9306 unsigned NumElems = ResVT.getVectorNumElements();
9307
9308 // If we are inserting non-zero vector and there are zeros in LSBs and undef
9309 // in the MSBs we need to emit a KSHIFTL. The generic lowering to
9310 // insert_subvector will give us two kshifts.
9311 if (isPowerOf2_64(NonZeros) && Zeros != 0 && NonZeros > Zeros &&
9312 Log2_64(NonZeros) != NumOperands - 1) {
9313 unsigned Idx = Log2_64(NonZeros);
9314 SDValue SubVec = Op.getOperand(Idx);
9315 unsigned SubVecNumElts = SubVec.getSimpleValueType().getVectorNumElements();
9316 MVT ShiftVT = widenMaskVectorType(ResVT, Subtarget);
9317 Op = widenSubVector(ShiftVT, SubVec, false, Subtarget, DAG, dl);
9318 Op = DAG.getNode(X86ISD::KSHIFTL, dl, ShiftVT, Op,
9319 DAG.getTargetConstant(Idx * SubVecNumElts, dl, MVT::i8));
9320 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, Op,
9321 DAG.getIntPtrConstant(0, dl));
9322 }
9323
9324 // If there are zero or one non-zeros we can handle this very simply.
9325 if (NonZeros == 0 || isPowerOf2_64(NonZeros)) {
9326 SDValue Vec = Zeros ? DAG.getConstant(0, dl, ResVT) : DAG.getUNDEF(ResVT);
9327 if (!NonZeros)
9328 return Vec;
9329 unsigned Idx = Log2_64(NonZeros);
9330 SDValue SubVec = Op.getOperand(Idx);
9331 unsigned SubVecNumElts = SubVec.getSimpleValueType().getVectorNumElements();
9332 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, SubVec,
9333 DAG.getIntPtrConstant(Idx * SubVecNumElts, dl));
9334 }
9335
9336 if (NumOperands > 2) {
9337 MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
9338 ArrayRef<SDUse> Ops = Op->ops();
9339 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9340 Ops.slice(0, NumOperands/2));
9341 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
9342 Ops.slice(NumOperands/2));
9343 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
9344 }
9345
9346 assert(llvm::popcount(NonZeros) == 2 && "Simple cases not handled?");
9347
9348 if (ResVT.getVectorNumElements() >= 16)
9349 return Op; // The operation is legal with KUNPCK
9350
9351 SDValue Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT,
9352 DAG.getUNDEF(ResVT), Op.getOperand(0),
9353 DAG.getIntPtrConstant(0, dl));
9354 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Op.getOperand(1),
9355 DAG.getIntPtrConstant(NumElems/2, dl));
9356}
9357
9358static SDValue LowerCONCAT_VECTORS(SDValue Op,
9359 const X86Subtarget &Subtarget,
9360 SelectionDAG &DAG) {
9361 MVT VT = Op.getSimpleValueType();
9362 if (VT.getVectorElementType() == MVT::i1)
9363 return LowerCONCAT_VECTORSvXi1(Op, Subtarget, DAG);
9364
9365 assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||
9366 (VT.is512BitVector() && (Op.getNumOperands() == 2 ||
9367 Op.getNumOperands() == 4)));
9368
9369 // AVX can use the vinsertf128 instruction to create 256-bit vectors
9370 // from two other 128-bit ones.
9371
9372 // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors
9373 return LowerAVXCONCAT_VECTORS(Op, DAG, Subtarget);
9374}
9375
9376//===----------------------------------------------------------------------===//
9377// Vector shuffle lowering
9378//
9379// This is an experimental code path for lowering vector shuffles on x86. It is
9380// designed to handle arbitrary vector shuffles and blends, gracefully
9381// degrading performance as necessary. It works hard to recognize idiomatic
9382// shuffles and lower them to optimal instruction patterns without leaving
9383// a framework that allows reasonably efficient handling of all vector shuffle
9384// patterns.
9385//===----------------------------------------------------------------------===//
9386
9387/// Tiny helper function to identify a no-op mask.
9388///
9389/// This is a somewhat boring predicate function. It checks whether the mask
9390/// array input, which is assumed to be a single-input shuffle mask of the kind
9391/// used by the X86 shuffle instructions (not a fully general
9392/// ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an
9393/// in-place shuffle are 'no-op's.
9394static bool isNoopShuffleMask(ArrayRef<int> Mask) {
9395 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
9396 assert(Mask[i] >= -1 && "Out of bound mask element!");
9397 if (Mask[i] >= 0 && Mask[i] != i)
9398 return false;
9399 }
9400 return true;
9401}
9402
9403/// Test whether there are elements crossing LaneSizeInBits lanes in this
9404/// shuffle mask.
9405///
9406/// X86 divides up its shuffles into in-lane and cross-lane shuffle operations
9407/// and we routinely test for these.
9408static bool isLaneCrossingShuffleMask(unsigned LaneSizeInBits,
9409 unsigned ScalarSizeInBits,
9410 ArrayRef<int> Mask) {
9411 assert(LaneSizeInBits && ScalarSizeInBits &&
9412 (LaneSizeInBits % ScalarSizeInBits) == 0 &&
9413 "Illegal shuffle lane size");
9414 int LaneSize = LaneSizeInBits / ScalarSizeInBits;
9415 int Size = Mask.size();
9416 for (int i = 0; i < Size; ++i)
9417 if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
9418 return true;
9419 return false;
9420}
9421
9422/// Test whether there are elements crossing 128-bit lanes in this
9423/// shuffle mask.
9424static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) {
9425 return isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), Mask);
9426}
9427
9428/// Test whether elements in each LaneSizeInBits lane in this shuffle mask come
9429/// from multiple lanes - this is different to isLaneCrossingShuffleMask to
9430/// better support 'repeated mask + lane permute' style shuffles.
9431static bool isMultiLaneShuffleMask(unsigned LaneSizeInBits,
9432 unsigned ScalarSizeInBits,
9433 ArrayRef<int> Mask) {
9434 assert(LaneSizeInBits && ScalarSizeInBits &&
9435 (LaneSizeInBits % ScalarSizeInBits) == 0 &&
9436 "Illegal shuffle lane size");
9437 int NumElts = Mask.size();
9438 int NumEltsPerLane = LaneSizeInBits / ScalarSizeInBits;
9439 int NumLanes = NumElts / NumEltsPerLane;
9440 if (NumLanes > 1) {
9441 for (int i = 0; i != NumLanes; ++i) {
9442 int SrcLane = -1;
9443 for (int j = 0; j != NumEltsPerLane; ++j) {
9444 int M = Mask[(i * NumEltsPerLane) + j];
9445 if (M < 0)
9446 continue;
9447 int Lane = (M % NumElts) / NumEltsPerLane;
9448 if (SrcLane >= 0 && SrcLane != Lane)
9449 return true;
9450 SrcLane = Lane;
9451 }
9452 }
9453 }
9454 return false;
9455}
9456
9457/// Test whether a shuffle mask is equivalent within each sub-lane.
9458///
9459/// This checks a shuffle mask to see if it is performing the same
9460/// lane-relative shuffle in each sub-lane. This trivially implies
9461/// that it is also not lane-crossing. It may however involve a blend from the
9462/// same lane of a second vector.
9463///
9464/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
9465/// non-trivial to compute in the face of undef lanes. The representation is
9466/// suitable for use with existing 128-bit shuffles as entries from the second
9467/// vector have been remapped to [LaneSize, 2*LaneSize).
9468static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
9469 ArrayRef<int> Mask,
9470 SmallVectorImpl<int> &RepeatedMask) {
9471 auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
9472 RepeatedMask.assign(LaneSize, -1);
9473 int Size = Mask.size();
9474 for (int i = 0; i < Size; ++i) {
9475 assert(Mask[i] == SM_SentinelUndef || Mask[i] >= 0);
9476 if (Mask[i] < 0)
9477 continue;
9478 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
9479 // This entry crosses lanes, so there is no way to model this shuffle.
9480 return false;
9481
9482 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
9483 // Adjust second vector indices to start at LaneSize instead of Size.
9484 int LocalM = Mask[i] < Size ? Mask[i] % LaneSize
9485 : Mask[i] % LaneSize + LaneSize;
9486 if (RepeatedMask[i % LaneSize] < 0)
9487 // This is the first non-undef entry in this slot of a 128-bit lane.
9488 RepeatedMask[i % LaneSize] = LocalM;
9489 else if (RepeatedMask[i % LaneSize] != LocalM)
9490 // Found a mismatch with the repeated mask.
9491 return false;
9492 }
9493 return true;
9494}
9495
9496/// Test whether a shuffle mask is equivalent within each 128-bit lane.
9497static bool
9498is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
9499 SmallVectorImpl<int> &RepeatedMask) {
9500 return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
9501}
9502
9503static bool
9504is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {
9505 SmallVector<int, 32> RepeatedMask;
9506 return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
9507}
9508
9509/// Test whether a shuffle mask is equivalent within each 256-bit lane.
9510static bool
9511is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
9512 SmallVectorImpl<int> &RepeatedMask) {
9513 return isRepeatedShuffleMask(256, VT, Mask, RepeatedMask);
9514}
9515
9516/// Test whether a target shuffle mask is equivalent within each sub-lane.
9517/// Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
9518static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits,
9519 unsigned EltSizeInBits,
9520 ArrayRef<int> Mask,
9521 SmallVectorImpl<int> &RepeatedMask) {
9522 int LaneSize = LaneSizeInBits / EltSizeInBits;
9523 RepeatedMask.assign(LaneSize, SM_SentinelUndef);
9524 int Size = Mask.size();
9525 for (int i = 0; i < Size; ++i) {
9526 assert(isUndefOrZero(Mask[i]) || (Mask[i] >= 0));
9527 if (Mask[i] == SM_SentinelUndef)
9528 continue;
9529 if (Mask[i] == SM_SentinelZero) {
9530 if (!isUndefOrZero(RepeatedMask[i % LaneSize]))
9531 return false;
9532 RepeatedMask[i % LaneSize] = SM_SentinelZero;
9533 continue;
9534 }
9535 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
9536 // This entry crosses lanes, so there is no way to model this shuffle.
9537 return false;
9538
9539 // Handle the in-lane shuffles by detecting if and when they repeat. Adjust
9540 // later vector indices to start at multiples of LaneSize instead of Size.
9541 int LaneM = Mask[i] / Size;
9542 int LocalM = (Mask[i] % LaneSize) + (LaneM * LaneSize);
9543 if (RepeatedMask[i % LaneSize] == SM_SentinelUndef)
9544 // This is the first non-undef entry in this slot of a 128-bit lane.
9545 RepeatedMask[i % LaneSize] = LocalM;
9546 else if (RepeatedMask[i % LaneSize] != LocalM)
9547 // Found a mismatch with the repeated mask.
9548 return false;
9549 }
9550 return true;
9551}
9552
9553/// Test whether a target shuffle mask is equivalent within each sub-lane.
9554/// Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
9555static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT,
9556 ArrayRef<int> Mask,
9557 SmallVectorImpl<int> &RepeatedMask) {
9558 return isRepeatedTargetShuffleMask(LaneSizeInBits, VT.getScalarSizeInBits(),
9559 Mask, RepeatedMask);
9560}
9561
9562/// Checks whether the vector elements referenced by two shuffle masks are
9563/// equivalent.
9564static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp,
9565 int Idx, int ExpectedIdx) {
9566 assert(0 <= Idx && Idx < MaskSize && 0 <= ExpectedIdx &&
9567 ExpectedIdx < MaskSize && "Out of range element index");
9568 if (!Op || !ExpectedOp || Op.getOpcode() != ExpectedOp.getOpcode())
9569 return false;
9570
9571 switch (Op.getOpcode()) {
9572 case ISD::BUILD_VECTOR:
9573 // If the values are build vectors, we can look through them to find
9574 // equivalent inputs that make the shuffles equivalent.
9575 // TODO: Handle MaskSize != Op.getNumOperands()?
9576 if (MaskSize == (int)Op.getNumOperands() &&
9577 MaskSize == (int)ExpectedOp.getNumOperands())
9578 return Op.getOperand(Idx) == ExpectedOp.getOperand(ExpectedIdx);
9579 break;
9580 case X86ISD::VBROADCAST:
9581 case X86ISD::VBROADCAST_LOAD:
9582 // TODO: Handle MaskSize != Op.getValueType().getVectorNumElements()?
9583 return (Op == ExpectedOp &&
9584 (int)Op.getValueType().getVectorNumElements() == MaskSize);
9585 case X86ISD::HADD:
9586 case X86ISD::HSUB:
9587 case X86ISD::FHADD:
9588 case X86ISD::FHSUB:
9589 case X86ISD::PACKSS:
9590 case X86ISD::PACKUS:
9591 // HOP(X,X) can refer to the elt from the lower/upper half of a lane.
9592 // TODO: Handle MaskSize != NumElts?
9593 // TODO: Handle HOP(X,Y) vs HOP(Y,X) equivalence cases.
9594 if (Op == ExpectedOp && Op.getOperand(0) == Op.getOperand(1)) {
9595 MVT VT = Op.getSimpleValueType();
9596 int NumElts = VT.getVectorNumElements();
9597 if (MaskSize == NumElts) {
9598 int NumLanes = VT.getSizeInBits() / 128;
9599 int NumEltsPerLane = NumElts / NumLanes;
9600 int NumHalfEltsPerLane = NumEltsPerLane / 2;
9601 bool SameLane =
9602 (Idx / NumEltsPerLane) == (ExpectedIdx / NumEltsPerLane);
9603 bool SameElt =
9604 (Idx % NumHalfEltsPerLane) == (ExpectedIdx % NumHalfEltsPerLane);
9605 return SameLane && SameElt;
9606 }
9607 }
9608 break;
9609 }
9610
9611 return false;
9612}
9613
9614/// Checks whether a shuffle mask is equivalent to an explicit list of
9615/// arguments.
9616///
9617/// This is a fast way to test a shuffle mask against a fixed pattern:
9618///
9619/// if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }
9620///
9621/// It returns true if the mask is exactly as wide as the argument list, and
9622/// each element of the mask is either -1 (signifying undef) or the value given
9623/// in the argument.
9624static bool isShuffleEquivalent(ArrayRef<int> Mask, ArrayRef<int> ExpectedMask,
9625 SDValue V1 = SDValue(),
9626 SDValue V2 = SDValue()) {
9627 int Size = Mask.size();
9628 if (Size != (int)ExpectedMask.size())
9629 return false;
9630
9631 for (int i = 0; i < Size; ++i) {
9632 assert(Mask[i] >= -1 && "Out of bound mask element!");
9633 int MaskIdx = Mask[i];
9634 int ExpectedIdx = ExpectedMask[i];
9635 if (0 <= MaskIdx && MaskIdx != ExpectedIdx) {
9636 SDValue MaskV = MaskIdx < Size ? V1 : V2;
9637 SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9638 MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size);
9639 ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size);
9640 if (!IsElementEquivalent(Size, MaskV, ExpectedV, MaskIdx, ExpectedIdx))
9641 return false;
9642 }
9643 }
9644 return true;
9645}
9646
9647/// Checks whether a target shuffle mask is equivalent to an explicit pattern.
9648///
9649/// The masks must be exactly the same width.
9650///
9651/// If an element in Mask matches SM_SentinelUndef (-1) then the corresponding
9652/// value in ExpectedMask is always accepted. Otherwise the indices must match.
9653///
9654/// SM_SentinelZero is accepted as a valid negative index but must match in
9655/// both, or via a known bits test.
9656static bool isTargetShuffleEquivalent(MVT VT, ArrayRef<int> Mask,
9657 ArrayRef<int> ExpectedMask,
9658 const SelectionDAG &DAG,
9659 SDValue V1 = SDValue(),
9660 SDValue V2 = SDValue()) {
9661 int Size = Mask.size();
9662 if (Size != (int)ExpectedMask.size())
9663 return false;
9664 assert(llvm::all_of(ExpectedMask,
9665 [Size](int M) { return isInRange(M, 0, 2 * Size); }) &&
9666 "Illegal target shuffle mask");
9667
9668 // Check for out-of-range target shuffle mask indices.
9669 if (!isUndefOrZeroOrInRange(Mask, 0, 2 * Size))
9670 return false;
9671
9672 // Don't use V1/V2 if they're not the same size as the shuffle mask type.
9673 if (V1 && (V1.getValueSizeInBits() != VT.getSizeInBits() ||
9674 !V1.getValueType().isVector()))
9675 V1 = SDValue();
9676 if (V2 && (V2.getValueSizeInBits() != VT.getSizeInBits() ||
9677 !V2.getValueType().isVector()))
9678 V2 = SDValue();
9679
9680 APInt ZeroV1 = APInt::getZero(Size);
9681 APInt ZeroV2 = APInt::getZero(Size);
9682
9683 for (int i = 0; i < Size; ++i) {
9684 int MaskIdx = Mask[i];
9685 int ExpectedIdx = ExpectedMask[i];
9686 if (MaskIdx == SM_SentinelUndef || MaskIdx == ExpectedIdx)
9687 continue;
9688 if (MaskIdx == SM_SentinelZero) {
9689 // If we need this expected index to be a zero element, then update the
9690 // relevant zero mask and perform the known bits at the end to minimize
9691 // repeated computes.
9692 SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9693 if (ExpectedV &&
9694 Size == (int)ExpectedV.getValueType().getVectorNumElements()) {
9695 int BitIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size);
9696 APInt &ZeroMask = ExpectedIdx < Size ? ZeroV1 : ZeroV2;
9697 ZeroMask.setBit(BitIdx);
9698 continue;
9699 }
9700 }
9701 if (MaskIdx >= 0) {
9702 SDValue MaskV = MaskIdx < Size ? V1 : V2;
9703 SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2;
9704 MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size);
9705 ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size);
9706 if (IsElementEquivalent(Size, MaskV, ExpectedV, MaskIdx, ExpectedIdx))
9707 continue;
9708 }
9709 return false;
9710 }
9711 return (ZeroV1.isZero() || DAG.MaskedVectorIsZero(V1, ZeroV1)) &&
9712 (ZeroV2.isZero() || DAG.MaskedVectorIsZero(V2, ZeroV2));
9713}
9714
9715// Check if the shuffle mask is suitable for the AVX vpunpcklwd or vpunpckhwd
9716// instructions.
9717static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT,
9718 const SelectionDAG &DAG) {
9719 if (VT != MVT::v8i32 && VT != MVT::v8f32)
9720 return false;
9721
9722 SmallVector<int, 8> Unpcklwd;
9723 createUnpackShuffleMask(MVT::v8i16, Unpcklwd, /* Lo = */ true,
9724 /* Unary = */ false);
9725 SmallVector<int, 8> Unpckhwd;
9726 createUnpackShuffleMask(MVT::v8i16, Unpckhwd, /* Lo = */ false,
9727 /* Unary = */ false);
9728 bool IsUnpackwdMask = (isTargetShuffleEquivalent(VT, Mask, Unpcklwd, DAG) ||
9729 isTargetShuffleEquivalent(VT, Mask, Unpckhwd, DAG));
9730 return IsUnpackwdMask;
9731}
9732
9733static bool is128BitUnpackShuffleMask(ArrayRef<int> Mask,
9734 const SelectionDAG &DAG) {
9735 // Create 128-bit vector type based on mask size.
9736 MVT EltVT = MVT::getIntegerVT(128 / Mask.size());
9737 MVT VT = MVT::getVectorVT(EltVT, Mask.size());
9738
9739 // We can't assume a canonical shuffle mask, so try the commuted version too.
9740 SmallVector<int, 4> CommutedMask(Mask);
9741 ShuffleVectorSDNode::commuteMask(CommutedMask);
9742
9743 // Match any of unary/binary or low/high.
9744 for (unsigned i = 0; i != 4; ++i) {
9745 SmallVector<int, 16> UnpackMask;
9746 createUnpackShuffleMask(VT, UnpackMask, (i >> 1) % 2, i % 2);
9747 if (isTargetShuffleEquivalent(VT, Mask, UnpackMask, DAG) ||
9748 isTargetShuffleEquivalent(VT, CommutedMask, UnpackMask, DAG))
9749 return true;
9750 }
9751 return false;
9752}
9753
9754/// Return true if a shuffle mask chooses elements identically in its top and
9755/// bottom halves. For example, any splat mask has the same top and bottom
9756/// halves. If an element is undefined in only one half of the mask, the halves
9757/// are not considered identical.
9758static bool hasIdenticalHalvesShuffleMask(ArrayRef<int> Mask) {
9759 assert(Mask.size() % 2 == 0 && "Expecting even number of elements in mask");
9760 unsigned HalfSize = Mask.size() / 2;
9761 for (unsigned i = 0; i != HalfSize; ++i) {
9762 if (Mask[i] != Mask[i + HalfSize])
9763 return false;
9764 }
9765 return true;
9766}
9767
9768/// Get a 4-lane 8-bit shuffle immediate for a mask.
9769///
9770/// This helper function produces an 8-bit shuffle immediate corresponding to
9771/// the ubiquitous shuffle encoding scheme used in x86 instructions for
9772/// shuffling 4 lanes. It can be used with most of the PSHUF instructions for
9773/// example.
9774///
9775/// NB: We rely heavily on "undef" masks preserving the input lane.
9776static unsigned getV4X86ShuffleImm(ArrayRef<int> Mask) {
9777 assert(Mask.size() == 4 && "Only 4-lane shuffle masks");
9778 assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!");
9779 assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!");
9780 assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!");
9781 assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!");
9782
9783 // If the mask only uses one non-undef element, then fully 'splat' it to
9784 // improve later broadcast matching.
9785 int FirstIndex = find_if(Mask, [](int M) { return M >= 0; }) - Mask.begin();
9786 assert(0 <= FirstIndex && FirstIndex < 4 && "All undef shuffle mask");
9787
9788 int FirstElt = Mask[FirstIndex];
9789 if (all_of(Mask, [FirstElt](int M) { return M < 0 || M == FirstElt; }))
9790 return (FirstElt << 6) | (FirstElt << 4) | (FirstElt << 2) | FirstElt;
9791
9792 unsigned Imm = 0;
9793 Imm |= (Mask[0] < 0 ? 0 : Mask[0]) << 0;
9794 Imm |= (Mask[1] < 0 ? 1 : Mask[1]) << 2;
9795 Imm |= (Mask[2] < 0 ? 2 : Mask[2]) << 4;
9796 Imm |= (Mask[3] < 0 ? 3 : Mask[3]) << 6;
9797 return Imm;
9798}
9799
9800static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, const SDLoc &DL,
9801 SelectionDAG &DAG) {
9802 return DAG.getTargetConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8);
9803}
9804
9805// The Shuffle result is as follow:
9806// 0*a[0]0*a[1]...0*a[n] , n >=0 where a[] elements in a ascending order.
9807// Each Zeroable's element correspond to a particular Mask's element.
9808// As described in computeZeroableShuffleElements function.
9809//
9810// The function looks for a sub-mask that the nonzero elements are in
9811// increasing order. If such sub-mask exist. The function returns true.
9812static bool isNonZeroElementsInOrder(const APInt &Zeroable,
9813 ArrayRef<int> Mask, const EVT &VectorType,
9814 bool &IsZeroSideLeft) {
9815 int NextElement = -1;
9816 // Check if the Mask's nonzero elements are in increasing order.
9817 for (int i = 0, e = Mask.size(); i < e; i++) {
9818 // Checks if the mask's zeros elements are built from only zeros.
9819 assert(Mask[i] >= -1 && "Out of bound mask element!");
9820 if (Mask[i] < 0)
9821 return false;
9822 if (Zeroable[i])
9823 continue;
9824 // Find the lowest non zero element
9825 if (NextElement < 0) {
9826 NextElement = Mask[i] != 0 ? VectorType.getVectorNumElements() : 0;
9827 IsZeroSideLeft = NextElement != 0;
9828 }
9829 // Exit if the mask's non zero elements are not in increasing order.
9830 if (NextElement != Mask[i])
9831 return false;
9832 NextElement++;
9833 }
9834 return true;
9835}
9836
9837/// Try to lower a shuffle with a single PSHUFB of V1 or V2.
9838static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
9839 ArrayRef<int> Mask, SDValue V1,
9840 SDValue V2, const APInt &Zeroable,
9841 const X86Subtarget &Subtarget,
9842 SelectionDAG &DAG) {
9843 int Size = Mask.size();
9844 int LaneSize = 128 / VT.getScalarSizeInBits();
9845 const int NumBytes = VT.getSizeInBits() / 8;
9846 const int NumEltBytes = VT.getScalarSizeInBits() / 8;
9847
9848 assert((Subtarget.hasSSSE3() && VT.is128BitVector()) ||
9849 (Subtarget.hasAVX2() && VT.is256BitVector()) ||
9850 (Subtarget.hasBWI() && VT.is512BitVector()));
9851
9852 SmallVector<SDValue, 64> PSHUFBMask(NumBytes);
9853 // Sign bit set in i8 mask means zero element.
9854 SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);
9855
9856 SDValue V;
9857 for (int i = 0; i < NumBytes; ++i) {
9858 int M = Mask[i / NumEltBytes];
9859 if (M < 0) {
9860 PSHUFBMask[i] = DAG.getUNDEF(MVT::i8);
9861 continue;
9862 }
9863 if (Zeroable[i / NumEltBytes]) {
9864 PSHUFBMask[i] = ZeroMask;
9865 continue;
9866 }
9867
9868 // We can only use a single input of V1 or V2.
9869 SDValue SrcV = (M >= Size ? V2 : V1);
9870 if (V && V != SrcV)
9871 return SDValue();
9872 V = SrcV;
9873 M %= Size;
9874
9875 // PSHUFB can't cross lanes, ensure this doesn't happen.
9876 if ((M / LaneSize) != ((i / NumEltBytes) / LaneSize))
9877 return SDValue();
9878
9879 M = M % LaneSize;
9880 M = M * NumEltBytes + (i % NumEltBytes);
9881 PSHUFBMask[i] = DAG.getConstant(M, DL, MVT::i8);
9882 }
9883 assert(V && "Failed to find a source input");
9884
9885 MVT I8VT = MVT::getVectorVT(MVT::i8, NumBytes);
9886 return DAG.getBitcast(
9887 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V),
9888 DAG.getBuildVector(I8VT, DL, PSHUFBMask)));
9889}
9890
9891static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
9892 const X86Subtarget &Subtarget, SelectionDAG &DAG,
9893 const SDLoc &dl);
9894
9895// X86 has dedicated shuffle that can be lowered to VEXPAND
9896static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT,
9897 const APInt &Zeroable,
9898 ArrayRef<int> Mask, SDValue &V1,
9899 SDValue &V2, SelectionDAG &DAG,
9900 const X86Subtarget &Subtarget) {
9901 bool IsLeftZeroSide = true;
9902 if (!isNonZeroElementsInOrder(Zeroable, Mask, V1.getValueType(),
9903 IsLeftZeroSide))
9904 return SDValue();
9905 unsigned VEXPANDMask = (~Zeroable).getZExtValue();
9906 MVT IntegerType =
9907 MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));
9908 SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType);
9909 unsigned NumElts = VT.getVectorNumElements();
9910 assert((NumElts == 4 || NumElts == 8 || NumElts == 16) &&
9911 "Unexpected number of vector elements");
9912 SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts),
9913 Subtarget, DAG, DL);
9914 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
9915 SDValue ExpandedVector = IsLeftZeroSide ? V2 : V1;
9916 return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask);
9917}
9918
9919static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
9920 unsigned &UnpackOpcode, bool IsUnary,
9921 ArrayRef<int> TargetMask, const SDLoc &DL,
9922 SelectionDAG &DAG,
9923 const X86Subtarget &Subtarget) {
9924 int NumElts = VT.getVectorNumElements();
9925
9926 bool Undef1 = true, Undef2 = true, Zero1 = true, Zero2 = true;
9927 for (int i = 0; i != NumElts; i += 2) {
9928 int M1 = TargetMask[i + 0];
9929 int M2 = TargetMask[i + 1];
9930 Undef1 &= (SM_SentinelUndef == M1);
9931 Undef2 &= (SM_SentinelUndef == M2);
9932 Zero1 &= isUndefOrZero(M1);
9933 Zero2 &= isUndefOrZero(M2);
9934 }
9935 assert(!((Undef1 || Zero1) && (Undef2 || Zero2)) &&
9936 "Zeroable shuffle detected");
9937
9938 // Attempt to match the target mask against the unpack lo/hi mask patterns.
9939 SmallVector<int, 64> Unpckl, Unpckh;
9940 createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, IsUnary);
9941 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG, V1,
9942 (IsUnary ? V1 : V2))) {
9943 UnpackOpcode = X86ISD::UNPCKL;
9944 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
9945 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
9946 return true;
9947 }
9948
9949 createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, IsUnary);
9950 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG, V1,
9951 (IsUnary ? V1 : V2))) {
9952 UnpackOpcode = X86ISD::UNPCKH;
9953 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
9954 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
9955 return true;
9956 }
9957
9958 // If an unary shuffle, attempt to match as an unpack lo/hi with zero.
9959 if (IsUnary && (Zero1 || Zero2)) {
9960 // Don't bother if we can blend instead.
9961 if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) &&
9962 isSequentialOrUndefOrZeroInRange(TargetMask, 0, NumElts, 0))
9963 return false;
9964
9965 bool MatchLo = true, MatchHi = true;
9966 for (int i = 0; (i != NumElts) && (MatchLo || MatchHi); ++i) {
9967 int M = TargetMask[i];
9968
9969 // Ignore if the input is known to be zero or the index is undef.
9970 if ((((i & 1) == 0) && Zero1) || (((i & 1) == 1) && Zero2) ||
9971 (M == SM_SentinelUndef))
9972 continue;
9973
9974 MatchLo &= (M == Unpckl[i]);
9975 MatchHi &= (M == Unpckh[i]);
9976 }
9977
9978 if (MatchLo || MatchHi) {
9979 UnpackOpcode = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
9980 V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
9981 V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
9982 return true;
9983 }
9984 }
9985
9986 // If a binary shuffle, commute and try again.
9987 if (!IsUnary) {
9988 ShuffleVectorSDNode::commuteMask(Unpckl);
9989 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG)) {
9990 UnpackOpcode = X86ISD::UNPCKL;
9991 std::swap(V1, V2);
9992 return true;
9993 }
9994
9995 ShuffleVectorSDNode::commuteMask(Unpckh);
9996 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG)) {
9997 UnpackOpcode = X86ISD::UNPCKH;
9998 std::swap(V1, V2);
9999 return true;
10000 }
10001 }
10002
10003 return false;
10004}
10005
10006// X86 has dedicated unpack instructions that can handle specific blend
10007// operations: UNPCKH and UNPCKL.
10008static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT, SDValue V1,
10009 SDValue V2, ArrayRef<int> Mask,
10010 SelectionDAG &DAG) {
10011 SmallVector<int, 8> Unpckl;
10012 createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false);
10013 if (isShuffleEquivalent(Mask, Unpckl, V1, V2))
10014 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
10015
10016 SmallVector<int, 8> Unpckh;
10017 createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, /* Unary = */ false);
10018 if (isShuffleEquivalent(Mask, Unpckh, V1, V2))
10019 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
10020
10021 // Commute and try again.
10022 ShuffleVectorSDNode::commuteMask(Unpckl);
10023 if (isShuffleEquivalent(Mask, Unpckl, V1, V2))
10024 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);
10025
10026 ShuffleVectorSDNode::commuteMask(Unpckh);
10027 if (isShuffleEquivalent(Mask, Unpckh, V1, V2))
10028 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);
10029
10030 return SDValue();
10031}
10032
10033/// Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit)
10034/// followed by unpack 256-bit.
10035static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, SDValue V1,
10036 SDValue V2, ArrayRef<int> Mask,
10037 SelectionDAG &DAG) {
10038 SmallVector<int, 32> Unpckl, Unpckh;
10039 createSplat2ShuffleMask(VT, Unpckl, /* Lo */ true);
10040 createSplat2ShuffleMask(VT, Unpckh, /* Lo */ false);
10041
10042 unsigned UnpackOpcode;
10043 if (isShuffleEquivalent(Mask, Unpckl, V1, V2))
10044 UnpackOpcode = X86ISD::UNPCKL;
10045 else if (isShuffleEquivalent(Mask, Unpckh, V1, V2))
10046 UnpackOpcode = X86ISD::UNPCKH;
10047 else
10048 return SDValue();
10049
10050 // This is a "natural" unpack operation (rather than the 128-bit sectored
10051 // operation implemented by AVX). We need to rearrange 64-bit chunks of the
10052 // input in order to use the x86 instruction.
10053 V1 = DAG.getVectorShuffle(MVT::v4f64, DL, DAG.getBitcast(MVT::v4f64, V1),
10054 DAG.getUNDEF(MVT::v4f64), {0, 2, 1, 3});
10055 V1 = DAG.getBitcast(VT, V1);
10056 return DAG.getNode(UnpackOpcode, DL, VT, V1, V1);
10057}
10058
10059// Check if the mask can be mapped to a TRUNCATE or VTRUNC, truncating the
10060// source into the lower elements and zeroing the upper elements.
10061static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT,
10062 ArrayRef<int> Mask, const APInt &Zeroable,
10063 const X86Subtarget &Subtarget) {
10064 if (!VT.is512BitVector() && !Subtarget.hasVLX())
10065 return false;
10066
10067 unsigned NumElts = Mask.size();
10068 unsigned EltSizeInBits = VT.getScalarSizeInBits();
10069 unsigned MaxScale = 64 / EltSizeInBits;
10070
10071 for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
10072 unsigned SrcEltBits = EltSizeInBits * Scale;
10073 if (SrcEltBits < 32 && !Subtarget.hasBWI())
10074 continue;
10075 unsigned NumSrcElts = NumElts / Scale;
10076 if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale))
10077 continue;
10078 unsigned UpperElts = NumElts - NumSrcElts;
10079 if (!Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10080 continue;
10081 SrcVT = MVT::getIntegerVT(EltSizeInBits * Scale);
10082 SrcVT = MVT::getVectorVT(SrcVT, NumSrcElts);
10083 DstVT = MVT::getIntegerVT(EltSizeInBits);
10084 if ((NumSrcElts * EltSizeInBits) >= 128) {
10085 // ISD::TRUNCATE
10086 DstVT = MVT::getVectorVT(DstVT, NumSrcElts);
10087 } else {
10088 // X86ISD::VTRUNC
10089 DstVT = MVT::getVectorVT(DstVT, 128 / EltSizeInBits);
10090 }
10091 return true;
10092 }
10093
10094 return false;
10095}
10096
10097// Helper to create TRUNCATE/VTRUNC nodes, optionally with zero/undef upper
10098// element padding to the final DstVT.
10099static SDValue getAVX512TruncNode(const SDLoc &DL, MVT DstVT, SDValue Src,
10100 const X86Subtarget &Subtarget,
10101 SelectionDAG &DAG, bool ZeroUppers) {
10102 MVT SrcVT = Src.getSimpleValueType();
10103 MVT DstSVT = DstVT.getScalarType();
10104 unsigned NumDstElts = DstVT.getVectorNumElements();
10105 unsigned NumSrcElts = SrcVT.getVectorNumElements();
10106 unsigned DstEltSizeInBits = DstVT.getScalarSizeInBits();
10107
10108 if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT))
10109 return SDValue();
10110
10111 // Perform a direct ISD::TRUNCATE if possible.
10112 if (NumSrcElts == NumDstElts)
10113 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Src);
10114
10115 if (NumSrcElts > NumDstElts) {
10116 MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts);
10117 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
10118 return extractSubVector(Trunc, 0, DAG, DL, DstVT.getSizeInBits());
10119 }
10120
10121 if ((NumSrcElts * DstEltSizeInBits) >= 128) {
10122 MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts);
10123 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src);
10124 return widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL,
10125 DstVT.getSizeInBits());
10126 }
10127
10128 // Non-VLX targets must truncate from a 512-bit type, so we need to
10129 // widen, truncate and then possibly extract the original subvector.
10130 if (!Subtarget.hasVLX() && !SrcVT.is512BitVector()) {
10131 SDValue NewSrc = widenSubVector(Src, ZeroUppers, Subtarget, DAG, DL, 512);
10132 return getAVX512TruncNode(DL, DstVT, NewSrc, Subtarget, DAG, ZeroUppers);
10133 }
10134
10135 // Fallback to a X86ISD::VTRUNC, padding if necessary.
10136 MVT TruncVT = MVT::getVectorVT(DstSVT, 128 / DstEltSizeInBits);
10137 SDValue Trunc = DAG.getNode(X86ISD::VTRUNC, DL, TruncVT, Src);
10138 if (DstVT != TruncVT)
10139 Trunc = widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL,
10140 DstVT.getSizeInBits());
10141 return Trunc;
10142}
10143
10144// Try to lower trunc+vector_shuffle to a vpmovdb or a vpmovdw instruction.
10145//
10146// An example is the following:
10147//
10148// t0: ch = EntryToken
10149// t2: v4i64,ch = CopyFromReg t0, Register:v4i64 %0
10150// t25: v4i32 = truncate t2
10151// t41: v8i16 = bitcast t25
10152// t21: v8i16 = BUILD_VECTOR undef:i16, undef:i16, undef:i16, undef:i16,
10153// Constant:i16<0>, Constant:i16<0>, Constant:i16<0>, Constant:i16<0>
10154// t51: v8i16 = vector_shuffle<0,2,4,6,12,13,14,15> t41, t21
10155// t18: v2i64 = bitcast t51
10156//
10157// One can just use a single vpmovdw instruction, without avx512vl we need to
10158// use the zmm variant and extract the lower subvector, padding with zeroes.
10159// TODO: Merge with lowerShuffleAsVTRUNC.
10160static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1,
10161 SDValue V2, ArrayRef<int> Mask,
10162 const APInt &Zeroable,
10163 const X86Subtarget &Subtarget,
10164 SelectionDAG &DAG) {
10165 assert((VT == MVT::v16i8 || VT == MVT::v8i16) && "Unexpected VTRUNC type");
10166 if (!Subtarget.hasAVX512())
10167 return SDValue();
10168
10169 unsigned NumElts = VT.getVectorNumElements();
10170 unsigned EltSizeInBits = VT.getScalarSizeInBits();
10171 unsigned MaxScale = 64 / EltSizeInBits;
10172 for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
10173 unsigned SrcEltBits = EltSizeInBits * Scale;
10174 unsigned NumSrcElts = NumElts / Scale;
10175 unsigned UpperElts = NumElts - NumSrcElts;
10176 if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale) ||
10177 !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10178 continue;
10179
10180 // Attempt to find a matching source truncation, but as a fall back VLX
10181 // cases can use the VPMOV directly.
10182 SDValue Src = peekThroughBitcasts(V1);
10183 if (Src.getOpcode() == ISD::TRUNCATE &&
10184 Src.getScalarValueSizeInBits() == SrcEltBits) {
10185 Src = Src.getOperand(0);
10186 } else if (Subtarget.hasVLX()) {
10187 MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
10188 MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
10189 Src = DAG.getBitcast(SrcVT, Src);
10190 // Don't do this if PACKSS/PACKUS could perform it cheaper.
10191 if (Scale == 2 &&
10192 ((DAG.ComputeNumSignBits(Src) > EltSizeInBits) ||
10193 (DAG.computeKnownBits(Src).countMinLeadingZeros() >= EltSizeInBits)))
10194 return SDValue();
10195 } else
10196 return SDValue();
10197
10198 // VPMOVWB is only available with avx512bw.
10199 if (!Subtarget.hasBWI() && Src.getScalarValueSizeInBits() < 32)
10200 return SDValue();
10201
10202 bool UndefUppers = isUndefInRange(Mask, NumSrcElts, UpperElts);
10203 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers);
10204 }
10205
10206 return SDValue();
10207}
10208
10209// Attempt to match binary shuffle patterns as a truncate.
10210static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1,
10211 SDValue V2, ArrayRef<int> Mask,
10212 const APInt &Zeroable,
10213 const X86Subtarget &Subtarget,
10214 SelectionDAG &DAG) {
10215 assert((VT.is128BitVector() || VT.is256BitVector()) &&
10216 "Unexpected VTRUNC type");
10217 if (!Subtarget.hasAVX512())
10218 return SDValue();
10219
10220 unsigned NumElts = VT.getVectorNumElements();
10221 unsigned EltSizeInBits = VT.getScalarSizeInBits();
10222 unsigned MaxScale = 64 / EltSizeInBits;
10223 for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) {
10224 // TODO: Support non-BWI VPMOVWB truncations?
10225 unsigned SrcEltBits = EltSizeInBits * Scale;
10226 if (SrcEltBits < 32 && !Subtarget.hasBWI())
10227 continue;
10228
10229 // Match shuffle <Ofs,Ofs+Scale,Ofs+2*Scale,..,undef_or_zero,undef_or_zero>
10230 // Bail if the V2 elements are undef.
10231 unsigned NumHalfSrcElts = NumElts / Scale;
10232 unsigned NumSrcElts = 2 * NumHalfSrcElts;
10233 for (unsigned Offset = 0; Offset != Scale; ++Offset) {
10234 if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, Offset, Scale) ||
10235 isUndefInRange(Mask, NumHalfSrcElts, NumHalfSrcElts))
10236 continue;
10237
10238 // The elements beyond the truncation must be undef/zero.
10239 unsigned UpperElts = NumElts - NumSrcElts;
10240 if (UpperElts > 0 &&
10241 !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnes())
10242 continue;
10243 bool UndefUppers =
10244 UpperElts > 0 && isUndefInRange(Mask, NumSrcElts, UpperElts);
10245
10246 // For offset truncations, ensure that the concat is cheap.
10247 if (Offset) {
10248 auto IsCheapConcat = [&](SDValue Lo, SDValue Hi) {
10249 if (Lo.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
10250 Hi.getOpcode() == ISD::EXTRACT_SUBVECTOR)
10251 return Lo.getOperand(0) == Hi.getOperand(0);
10252 if (ISD::isNormalLoad(Lo.getNode()) &&
10253 ISD::isNormalLoad(Hi.getNode())) {
10254 auto *LDLo = cast<LoadSDNode>(Lo);
10255 auto *LDHi = cast<LoadSDNode>(Hi);
10256 return DAG.areNonVolatileConsecutiveLoads(
10257 LDHi, LDLo, Lo.getValueType().getStoreSize(), 1);
10258 }
10259 return false;
10260 };
10261 if (!IsCheapConcat(V1, V2))
10262 continue;
10263 }
10264
10265 // As we're using both sources then we need to concat them together
10266 // and truncate from the double-sized src.
10267 MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2);
10268 SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
10269
10270 MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
10271 MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
10272 Src = DAG.getBitcast(SrcVT, Src);
10273
10274 // Shift the offset'd elements into place for the truncation.
10275 // TODO: Use getTargetVShiftByConstNode.
10276 if (Offset)
10277 Src = DAG.getNode(
10278 X86ISD::VSRLI, DL, SrcVT, Src,
10279 DAG.getTargetConstant(Offset * EltSizeInBits, DL, MVT::i8));
10280
10281 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers);
10282 }
10283 }
10284
10285 return SDValue();
10286}
10287
10288/// Check whether a compaction lowering can be done by dropping even/odd
10289/// elements and compute how many times even/odd elements must be dropped.
10290///
10291/// This handles shuffles which take every Nth element where N is a power of
10292/// two. Example shuffle masks:
10293///
10294/// (even)
10295/// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14
10296/// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
10297/// N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12
10298/// N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28
10299/// N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8
10300/// N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24
10301///
10302/// (odd)
10303/// N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 14
10304/// N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
10305///
10306/// Any of these lanes can of course be undef.
10307///
10308/// This routine only supports N <= 3.
10309/// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here
10310/// for larger N.
10311///
10312/// \returns N above, or the number of times even/odd elements must be dropped
10313/// if there is such a number. Otherwise returns zero.
10314static int canLowerByDroppingElements(ArrayRef<int> Mask, bool MatchEven,
10315 bool IsSingleInput) {
10316 // The modulus for the shuffle vector entries is based on whether this is
10317 // a single input or not.
10318 int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2);
10319 assert(isPowerOf2_32((uint32_t)ShuffleModulus) &&
10320 "We should only be called with masks with a power-of-2 size!");
10321
10322 uint64_t ModMask = (uint64_t)ShuffleModulus - 1;
10323 int Offset = MatchEven ? 0 : 1;
10324
10325 // We track whether the input is viable for all power-of-2 strides 2^1, 2^2,
10326 // and 2^3 simultaneously. This is because we may have ambiguity with
10327 // partially undef inputs.
10328 bool ViableForN[3] = {true, true, true};
10329
10330 for (int i = 0, e = Mask.size(); i < e; ++i) {
10331 // Ignore undef lanes, we'll optimistically collapse them to the pattern we
10332 // want.
10333 if (Mask[i] < 0)
10334 continue;
10335
10336 bool IsAnyViable = false;
10337 for (unsigned j = 0; j != std::size(ViableForN); ++j)
10338 if (ViableForN[j]) {
10339 uint64_t N = j + 1;
10340
10341 // The shuffle mask must be equal to (i * 2^N) % M.
10342 if ((uint64_t)(Mask[i] - Offset) == (((uint64_t)i << N) & ModMask))
10343 IsAnyViable = true;
10344 else
10345 ViableForN[j] = false;
10346 }
10347 // Early exit if we exhaust the possible powers of two.
10348 if (!IsAnyViable)
10349 break;
10350 }
10351
10352 for (unsigned j = 0; j != std::size(ViableForN); ++j)
10353 if (ViableForN[j])
10354 return j + 1;
10355
10356 // Return 0 as there is no viable power of two.
10357 return 0;
10358}
10359
10360// X86 has dedicated pack instructions that can handle specific truncation
10361// operations: PACKSS and PACKUS.
10362// Checks for compaction shuffle masks if MaxStages > 1.
10363// TODO: Add support for matching multiple PACKSS/PACKUS stages.
10364static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
10365 unsigned &PackOpcode, ArrayRef<int> TargetMask,
10366 const SelectionDAG &DAG,
10367 const X86Subtarget &Subtarget,
10368 unsigned MaxStages = 1) {
10369 unsigned NumElts = VT.getVectorNumElements();
10370 unsigned BitSize = VT.getScalarSizeInBits();
10371 assert(0 < MaxStages && MaxStages <= 3 && (BitSize << MaxStages) <= 64 &&
10372 "Illegal maximum compaction");
10373
10374 auto MatchPACK = [&](SDValue N1, SDValue N2, MVT PackVT) {
10375 unsigned NumSrcBits = PackVT.getScalarSizeInBits();
10376 unsigned NumPackedBits = NumSrcBits - BitSize;
10377 N1 = peekThroughBitcasts(N1);
10378 N2 = peekThroughBitcasts(N2);
10379 unsigned NumBits1 = N1.getScalarValueSizeInBits();
10380 unsigned NumBits2 = N2.getScalarValueSizeInBits();
10381 bool IsZero1 = llvm::isNullOrNullSplat(N1, /*AllowUndefs*/ false);
10382 bool IsZero2 = llvm::isNullOrNullSplat(N2, /*AllowUndefs*/ false);
10383 if ((!N1.isUndef() && !IsZero1 && NumBits1 != NumSrcBits) ||
10384 (!N2.isUndef() && !IsZero2 && NumBits2 != NumSrcBits))
10385 return false;
10386 if (Subtarget.hasSSE41() || BitSize == 8) {
10387 APInt ZeroMask = APInt::getHighBitsSet(NumSrcBits, NumPackedBits);
10388 if ((N1.isUndef() || IsZero1 || DAG.MaskedValueIsZero(N1, ZeroMask)) &&
10389 (N2.isUndef() || IsZero2 || DAG.MaskedValueIsZero(N2, ZeroMask))) {
10390 V1 = N1;
10391 V2 = N2;
10392 SrcVT = PackVT;
10393 PackOpcode = X86ISD::PACKUS;
10394 return true;
10395 }
10396 }
10397 bool IsAllOnes1 = llvm::isAllOnesOrAllOnesSplat(N1, /*AllowUndefs*/ false);
10398 bool IsAllOnes2 = llvm::isAllOnesOrAllOnesSplat(N2, /*AllowUndefs*/ false);
10399 if ((N1.isUndef() || IsZero1 || IsAllOnes1 ||
10400 DAG.ComputeNumSignBits(N1) > NumPackedBits) &&
10401 (N2.isUndef() || IsZero2 || IsAllOnes2 ||
10402 DAG.ComputeNumSignBits(N2) > NumPackedBits)) {
10403 V1 = N1;
10404 V2 = N2;
10405 SrcVT = PackVT;
10406 PackOpcode = X86ISD::PACKSS;
10407 return true;
10408 }
10409 return false;
10410 };
10411
10412 // Attempt to match against wider and wider compaction patterns.
10413 for (unsigned NumStages = 1; NumStages <= MaxStages; ++NumStages) {
10414 MVT PackSVT = MVT::getIntegerVT(BitSize << NumStages);
10415 MVT PackVT = MVT::getVectorVT(PackSVT, NumElts >> NumStages);
10416
10417 // Try binary shuffle.
10418 SmallVector<int, 32> BinaryMask;
10419 createPackShuffleMask(VT, BinaryMask, false, NumStages);
10420 if (isTargetShuffleEquivalent(VT, TargetMask, BinaryMask, DAG, V1, V2))
10421 if (MatchPACK(V1, V2, PackVT))
10422 return true;
10423
10424 // Try unary shuffle.
10425 SmallVector<int, 32> UnaryMask;
10426 createPackShuffleMask(VT, UnaryMask, true, NumStages);
10427 if (isTargetShuffleEquivalent(VT, TargetMask, UnaryMask, DAG, V1))
10428 if (MatchPACK(V1, V1, PackVT))
10429 return true;
10430 }
10431
10432 return false;
10433}
10434
10435static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, SDValue V1,
10436 SDValue V2, ArrayRef<int> Mask,
10437 const X86Subtarget &Subtarget,
10438 SelectionDAG &DAG) {
10439 MVT PackVT;
10440 unsigned PackOpcode;
10441 unsigned SizeBits = VT.getSizeInBits();
10442 unsigned EltBits = VT.getScalarSizeInBits();
10443 unsigned MaxStages = Log2_32(64 / EltBits);
10444 if (!matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG,
10445 Subtarget, MaxStages))
10446 return SDValue();
10447
10448 unsigned CurrentEltBits = PackVT.getScalarSizeInBits();
10449 unsigned NumStages = Log2_32(CurrentEltBits / EltBits);
10450
10451 // Don't lower multi-stage packs on AVX512, truncation is better.
10452 if (NumStages != 1 && SizeBits == 128 && Subtarget.hasVLX())
10453 return SDValue();
10454
10455 // Pack to the largest type possible:
10456 // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB.
10457 unsigned MaxPackBits = 16;
10458 if (CurrentEltBits > 16 &&
10459 (PackOpcode == X86ISD::PACKSS || Subtarget.hasSSE41()))
10460 MaxPackBits = 32;
10461
10462 // Repeatedly pack down to the target size.
10463 SDValue Res;
10464 for (unsigned i = 0; i != NumStages; ++i) {
10465 unsigned SrcEltBits = std::min(MaxPackBits, CurrentEltBits);
10466 unsigned NumSrcElts = SizeBits / SrcEltBits;
10467 MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
10468 MVT DstSVT = MVT::getIntegerVT(SrcEltBits / 2);
10469 MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
10470 MVT DstVT = MVT::getVectorVT(DstSVT, NumSrcElts * 2);
10471 Res = DAG.getNode(PackOpcode, DL, DstVT, DAG.getBitcast(SrcVT, V1),
10472 DAG.getBitcast(SrcVT, V2));
10473 V1 = V2 = Res;
10474 CurrentEltBits /= 2;
10475 }
10476 assert(Res && Res.getValueType() == VT &&
10477 "Failed to lower compaction shuffle");
10478 return Res;
10479}
10480
10481/// Try to emit a bitmask instruction for a shuffle.
10482///
10483/// This handles cases where we can model a blend exactly as a bitmask due to
10484/// one of the inputs being zeroable.
10485static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
10486 SDValue V2, ArrayRef<int> Mask,
10487 const APInt &Zeroable,
10488 const X86Subtarget &Subtarget,
10489 SelectionDAG &DAG) {
10490 MVT MaskVT = VT;
10491 MVT EltVT = VT.getVectorElementType();
10492 SDValue Zero, AllOnes;
10493 // Use f64 if i64 isn't legal.
10494 if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
10495 EltVT = MVT::f64;
10496 MaskVT = MVT::getVectorVT(EltVT, Mask.size());
10497 }
10498
10499 MVT LogicVT = VT;
10500 if (EltVT == MVT::f32 || EltVT == MVT::f64) {
10501 Zero = DAG.getConstantFP(0.0, DL, EltVT);
10502 APFloat AllOnesValue = APFloat::getAllOnesValue(EltVT.getFltSemantics());
10503 AllOnes = DAG.getConstantFP(AllOnesValue, DL, EltVT);
10504 LogicVT =
10505 MVT::getVectorVT(EltVT == MVT::f64 ? MVT::i64 : MVT::i32, Mask.size());
10506 } else {
10507 Zero = DAG.getConstant(0, DL, EltVT);
10508 AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10509 }
10510
10511 SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
10512 SDValue V;
10513 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10514 if (Zeroable[i])
10515 continue;
10516 if (Mask[i] % Size != i)
10517 return SDValue(); // Not a blend.
10518 if (!V)
10519 V = Mask[i] < Size ? V1 : V2;
10520 else if (V != (Mask[i] < Size ? V1 : V2))
10521 return SDValue(); // Can only let one input through the mask.
10522
10523 VMaskOps[i] = AllOnes;
10524 }
10525 if (!V)
10526 return SDValue(); // No non-zeroable elements!
10527
10528 SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps);
10529 VMask = DAG.getBitcast(LogicVT, VMask);
10530 V = DAG.getBitcast(LogicVT, V);
10531 SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask);
10532 return DAG.getBitcast(VT, And);
10533}
10534
10535/// Try to emit a blend instruction for a shuffle using bit math.
10536///
10537/// This is used as a fallback approach when first class blend instructions are
10538/// unavailable. Currently it is only suitable for integer vectors, but could
10539/// be generalized for floating point vectors if desirable.
10540static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
10541 SDValue V2, ArrayRef<int> Mask,
10542 SelectionDAG &DAG) {
10543 assert(VT.isInteger() && "Only supports integer vector types!");
10544 MVT EltVT = VT.getVectorElementType();
10545 SDValue Zero = DAG.getConstant(0, DL, EltVT);
10546 SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT);
10547 SmallVector<SDValue, 16> MaskOps;
10548 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10549 if (Mask[i] >= 0 && Mask[i] != i && Mask[i] != i + Size)
10550 return SDValue(); // Shuffled input!
10551 MaskOps.push_back(Mask[i] < Size ? AllOnes : Zero);
10552 }
10553
10554 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);
10555 return getBitSelect(DL, VT, V1, V2, V1Mask, DAG);
10556}
10557
10558static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
10559 SDValue PreservedSrc,
10560 const X86Subtarget &Subtarget,
10561 SelectionDAG &DAG);
10562
10563static bool matchShuffleAsBlend(MVT VT, SDValue V1, SDValue V2,
10564 MutableArrayRef<int> Mask,
10565 const APInt &Zeroable, bool &ForceV1Zero,
10566 bool &ForceV2Zero, uint64_t &BlendMask) {
10567 bool V1IsZeroOrUndef =
10568 V1.isUndef() || ISD::isBuildVectorAllZeros(V1.getNode());
10569 bool V2IsZeroOrUndef =
10570 V2.isUndef() || ISD::isBuildVectorAllZeros(V2.getNode());
10571
10572 BlendMask = 0;
10573 ForceV1Zero = false, ForceV2Zero = false;
10574 assert(Mask.size() <= 64 && "Shuffle mask too big for blend mask");
10575
10576 int NumElts = Mask.size();
10577 int NumLanes = VT.getSizeInBits() / 128;
10578 int NumEltsPerLane = NumElts / NumLanes;
10579 assert((NumLanes * NumEltsPerLane) == NumElts && "Value type mismatch");
10580
10581 // For 32/64-bit elements, if we only reference one input (plus any undefs),
10582 // then ensure the blend mask part for that lane just references that input.
10583 bool ForceWholeLaneMasks =
10584 VT.is256BitVector() && VT.getScalarSizeInBits() >= 32;
10585
10586 // Attempt to generate the binary blend mask. If an input is zero then
10587 // we can use any lane.
10588 for (int Lane = 0; Lane != NumLanes; ++Lane) {
10589 // Keep track of the inputs used per lane.
10590 bool LaneV1InUse = false;
10591 bool LaneV2InUse = false;
10592 uint64_t LaneBlendMask = 0;
10593 for (int LaneElt = 0; LaneElt != NumEltsPerLane; ++LaneElt) {
10594 int Elt = (Lane * NumEltsPerLane) + LaneElt;
10595 int M = Mask[Elt];
10596 if (M == SM_SentinelUndef)
10597 continue;
10598 if (M == Elt || (0 <= M && M < NumElts &&
10599 IsElementEquivalent(NumElts, V1, V1, M, Elt))) {
10600 Mask[Elt] = Elt;
10601 LaneV1InUse = true;
10602 continue;
10603 }
10604 if (M == (Elt + NumElts) ||
10605 (NumElts <= M &&
10606 IsElementEquivalent(NumElts, V2, V2, M - NumElts, Elt))) {
10607 LaneBlendMask |= 1ull << LaneElt;
10608 Mask[Elt] = Elt + NumElts;
10609 LaneV2InUse = true;
10610 continue;
10611 }
10612 if (Zeroable[Elt]) {
10613 if (V1IsZeroOrUndef) {
10614 ForceV1Zero = true;
10615 Mask[Elt] = Elt;
10616 LaneV1InUse = true;
10617 continue;
10618 }
10619 if (V2IsZeroOrUndef) {
10620 ForceV2Zero = true;
10621 LaneBlendMask |= 1ull << LaneElt;
10622 Mask[Elt] = Elt + NumElts;
10623 LaneV2InUse = true;
10624 continue;
10625 }
10626 }
10627 return false;
10628 }
10629
10630 // If we only used V2 then splat the lane blend mask to avoid any demanded
10631 // elts from V1 in this lane (the V1 equivalent is implicit with a zero
10632 // blend mask bit).
10633 if (ForceWholeLaneMasks && LaneV2InUse && !LaneV1InUse)
10634 LaneBlendMask = (1ull << NumEltsPerLane) - 1;
10635
10636 BlendMask |= LaneBlendMask << (Lane * NumEltsPerLane);
10637 }
10638 return true;
10639}
10640
10641/// Try to emit a blend instruction for a shuffle.
10642///
10643/// This doesn't do any checks for the availability of instructions for blending
10644/// these values. It relies on the availability of the X86ISD::BLENDI pattern to
10645/// be matched in the backend with the type given. What it does check for is
10646/// that the shuffle mask is a blend, or convertible into a blend with zero.
10647static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
10648 SDValue V2, ArrayRef<int> Original,
10649 const APInt &Zeroable,
10650 const X86Subtarget &Subtarget,
10651 SelectionDAG &DAG) {
10652 uint64_t BlendMask = 0;
10653 bool ForceV1Zero = false, ForceV2Zero = false;
10654 SmallVector<int, 64> Mask(Original);
10655 if (!matchShuffleAsBlend(VT, V1, V2, Mask, Zeroable, ForceV1Zero, ForceV2Zero,
10656 BlendMask))
10657 return SDValue();
10658
10659 // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs.
10660 if (ForceV1Zero)
10661 V1 = getZeroVector(VT, Subtarget, DAG, DL);
10662 if (ForceV2Zero)
10663 V2 = getZeroVector(VT, Subtarget, DAG, DL);
10664
10665 unsigned NumElts = VT.getVectorNumElements();
10666
10667 switch (VT.SimpleTy) {
10668 case MVT::v4i64:
10669 case MVT::v8i32:
10670 assert(Subtarget.hasAVX2() && "256-bit integer blends require AVX2!");
10671 [[fallthrough]];
10672 case MVT::v4f64:
10673 case MVT::v8f32:
10674 assert(Subtarget.hasAVX() && "256-bit float blends require AVX!");
10675 [[fallthrough]];
10676 case MVT::v2f64:
10677 case MVT::v2i64:
10678 case MVT::v4f32:
10679 case MVT::v4i32:
10680 case MVT::v8i16:
10681 assert(Subtarget.hasSSE41() && "128-bit blends require SSE41!");
10682 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
10683 DAG.getTargetConstant(BlendMask, DL, MVT::i8));
10684 case MVT::v16i16: {
10685 assert(Subtarget.hasAVX2() && "v16i16 blends require AVX2!");
10686 SmallVector<int, 8> RepeatedMask;
10687 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
10688 // We can lower these with PBLENDW which is mirrored across 128-bit lanes.
10689 assert(RepeatedMask.size() == 8 && "Repeated mask size doesn't match!");
10690 BlendMask = 0;
10691 for (int i = 0; i < 8; ++i)
10692 if (RepeatedMask[i] >= 8)
10693 BlendMask |= 1ull << i;
10694 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10695 DAG.getTargetConstant(BlendMask, DL, MVT::i8));
10696 }
10697 // Use PBLENDW for lower/upper lanes and then blend lanes.
10698 // TODO - we should allow 2 PBLENDW here and leave shuffle combine to
10699 // merge to VSELECT where useful.
10700 uint64_t LoMask = BlendMask & 0xFF;
10701 uint64_t HiMask = (BlendMask >> 8) & 0xFF;
10702 if (LoMask == 0 || LoMask == 255 || HiMask == 0 || HiMask == 255) {
10703 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10704 DAG.getTargetConstant(LoMask, DL, MVT::i8));
10705 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10706 DAG.getTargetConstant(HiMask, DL, MVT::i8));
10707 return DAG.getVectorShuffle(
10708 MVT::v16i16, DL, Lo, Hi,
10709 {0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31});
10710 }
10711 [[fallthrough]];
10712 }
10713 case MVT::v32i8:
10714 assert(Subtarget.hasAVX2() && "256-bit byte-blends require AVX2!");
10715 [[fallthrough]];
10716 case MVT::v16i8: {
10717 assert(Subtarget.hasSSE41() && "128-bit byte-blends require SSE41!");
10718
10719 // Attempt to lower to a bitmask if we can. VPAND is faster than VPBLENDVB.
10720 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
10721 Subtarget, DAG))
10722 return Masked;
10723
10724 if (Subtarget.hasBWI() && Subtarget.hasVLX()) {
10725 MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8));
10726 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
10727 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
10728 }
10729
10730 // If we have VPTERNLOG, we can use that as a bit blend.
10731 if (Subtarget.hasVLX())
10732 if (SDValue BitBlend =
10733 lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
10734 return BitBlend;
10735
10736 // Scale the blend by the number of bytes per element.
10737 int Scale = VT.getScalarSizeInBits() / 8;
10738
10739 // This form of blend is always done on bytes. Compute the byte vector
10740 // type.
10741 MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
10742
10743 // x86 allows load folding with blendvb from the 2nd source operand. But
10744 // we are still using LLVM select here (see comment below), so that's V1.
10745 // If V2 can be load-folded and V1 cannot be load-folded, then commute to
10746 // allow that load-folding possibility.
10747 if (!ISD::isNormalLoad(V1.getNode()) && ISD::isNormalLoad(V2.getNode())) {
10748 ShuffleVectorSDNode::commuteMask(Mask);
10749 std::swap(V1, V2);
10750 }
10751
10752 // Compute the VSELECT mask. Note that VSELECT is really confusing in the
10753 // mix of LLVM's code generator and the x86 backend. We tell the code
10754 // generator that boolean values in the elements of an x86 vector register
10755 // are -1 for true and 0 for false. We then use the LLVM semantics of 'true'
10756 // mapping a select to operand #1, and 'false' mapping to operand #2. The
10757 // reality in x86 is that vector masks (pre-AVX-512) use only the high bit
10758 // of the element (the remaining are ignored) and 0 in that high bit would
10759 // mean operand #1 while 1 in the high bit would mean operand #2. So while
10760 // the LLVM model for boolean values in vector elements gets the relevant
10761 // bit set, it is set backwards and over constrained relative to x86's
10762 // actual model.
10763 SmallVector<SDValue, 32> VSELECTMask;
10764 for (int i = 0, Size = Mask.size(); i < Size; ++i)
10765 for (int j = 0; j < Scale; ++j)
10766 VSELECTMask.push_back(
10767 Mask[i] < 0
10768 ? DAG.getUNDEF(MVT::i8)
10769 : DAG.getSignedConstant(Mask[i] < Size ? -1 : 0, DL, MVT::i8));
10770
10771 V1 = DAG.getBitcast(BlendVT, V1);
10772 V2 = DAG.getBitcast(BlendVT, V2);
10773 return DAG.getBitcast(
10774 VT,
10775 DAG.getSelect(DL, BlendVT, DAG.getBuildVector(BlendVT, DL, VSELECTMask),
10776 V1, V2));
10777 }
10778 case MVT::v16f32:
10779 case MVT::v8f64:
10780 case MVT::v8i64:
10781 case MVT::v16i32:
10782 case MVT::v32i16:
10783 case MVT::v64i8: {
10784 // Attempt to lower to a bitmask if we can. Only if not optimizing for size.
10785 bool OptForSize = DAG.shouldOptForSize();
10786 if (!OptForSize) {
10787 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
10788 Subtarget, DAG))
10789 return Masked;
10790 }
10791
10792 // Otherwise load an immediate into a GPR, cast to k-register, and use a
10793 // masked move.
10794 MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8));
10795 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
10796 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
10797 }
10798 default:
10799 llvm_unreachable("Not a supported integer vector type!");
10800 }
10801}
10802
10803/// Try to lower as a blend of elements from two inputs followed by
10804/// a single-input permutation.
10805///
10806/// This matches the pattern where we can blend elements from two inputs and
10807/// then reduce the shuffle to a single-input permutation.
10808static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
10809 SDValue V1, SDValue V2,
10810 ArrayRef<int> Mask,
10811 SelectionDAG &DAG,
10812 bool ImmBlends = false) {
10813 // We build up the blend mask while checking whether a blend is a viable way
10814 // to reduce the shuffle.
10815 SmallVector<int, 32> BlendMask(Mask.size(), -1);
10816 SmallVector<int, 32> PermuteMask(Mask.size(), -1);
10817
10818 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10819 if (Mask[i] < 0)
10820 continue;
10821
10822 assert(Mask[i] < Size * 2 && "Shuffle input is out of bounds.");
10823
10824 if (BlendMask[Mask[i] % Size] < 0)
10825 BlendMask[Mask[i] % Size] = Mask[i];
10826 else if (BlendMask[Mask[i] % Size] != Mask[i])
10827 return SDValue(); // Can't blend in the needed input!
10828
10829 PermuteMask[i] = Mask[i] % Size;
10830 }
10831
10832 // If only immediate blends, then bail if the blend mask can't be widened to
10833 // i16.
10834 unsigned EltSize = VT.getScalarSizeInBits();
10835 if (ImmBlends && EltSize == 8 && !canWidenShuffleElements(BlendMask))
10836 return SDValue();
10837
10838 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
10839 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask);
10840}
10841
10842/// Try to lower as an unpack of elements from two inputs followed by
10843/// a single-input permutation.
10844///
10845/// This matches the pattern where we can unpack elements from two inputs and
10846/// then reduce the shuffle to a single-input (wider) permutation.
10847static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
10848 SDValue V1, SDValue V2,
10849 ArrayRef<int> Mask,
10850 SelectionDAG &DAG) {
10851 int NumElts = Mask.size();
10852 int NumLanes = VT.getSizeInBits() / 128;
10853 int NumLaneElts = NumElts / NumLanes;
10854 int NumHalfLaneElts = NumLaneElts / 2;
10855
10856 bool MatchLo = true, MatchHi = true;
10857 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
10858
10859 // Determine UNPCKL/UNPCKH type and operand order.
10860 for (int Elt = 0; Elt != NumElts; ++Elt) {
10861 int M = Mask[Elt];
10862 if (M < 0)
10863 continue;
10864
10865 // Normalize the mask value depending on whether it's V1 or V2.
10866 int NormM = M;
10867 SDValue &Op = Ops[Elt & 1];
10868 if (M < NumElts && (Op.isUndef() || Op == V1))
10869 Op = V1;
10870 else if (NumElts <= M && (Op.isUndef() || Op == V2)) {
10871 Op = V2;
10872 NormM -= NumElts;
10873 } else
10874 return SDValue();
10875
10876 bool MatchLoAnyLane = false, MatchHiAnyLane = false;
10877 for (int Lane = 0; Lane != NumElts; Lane += NumLaneElts) {
10878 int Lo = Lane, Mid = Lane + NumHalfLaneElts, Hi = Lane + NumLaneElts;
10879 MatchLoAnyLane |= isUndefOrInRange(NormM, Lo, Mid);
10880 MatchHiAnyLane |= isUndefOrInRange(NormM, Mid, Hi);
10881 if (MatchLoAnyLane || MatchHiAnyLane) {
10882 assert((MatchLoAnyLane ^ MatchHiAnyLane) &&
10883 "Failed to match UNPCKLO/UNPCKHI");
10884 break;
10885 }
10886 }
10887 MatchLo &= MatchLoAnyLane;
10888 MatchHi &= MatchHiAnyLane;
10889 if (!MatchLo && !MatchHi)
10890 return SDValue();
10891 }
10892 assert((MatchLo ^ MatchHi) && "Failed to match UNPCKLO/UNPCKHI");
10893
10894 // Element indices have changed after unpacking. Calculate permute mask
10895 // so that they will be put back to the position as dictated by the
10896 // original shuffle mask indices.
10897 SmallVector<int, 32> PermuteMask(NumElts, -1);
10898 for (int Elt = 0; Elt != NumElts; ++Elt) {
10899 int M = Mask[Elt];
10900 if (M < 0)
10901 continue;
10902 int NormM = M;
10903 if (NumElts <= M)
10904 NormM -= NumElts;
10905 bool IsFirstOp = M < NumElts;
10906 int BaseMaskElt =
10907 NumLaneElts * (NormM / NumLaneElts) + (2 * (NormM % NumHalfLaneElts));
10908 if ((IsFirstOp && V1 == Ops[0]) || (!IsFirstOp && V2 == Ops[0]))
10909 PermuteMask[Elt] = BaseMaskElt;
10910 else if ((IsFirstOp && V1 == Ops[1]) || (!IsFirstOp && V2 == Ops[1]))
10911 PermuteMask[Elt] = BaseMaskElt + 1;
10912 assert(PermuteMask[Elt] != -1 &&
10913 "Input mask element is defined but failed to assign permute mask");
10914 }
10915
10916 unsigned UnpckOp = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
10917 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);
10918 return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask);
10919}
10920
10921/// Try to lower a shuffle as a permute of the inputs followed by an
10922/// UNPCK instruction.
10923///
10924/// This specifically targets cases where we end up with alternating between
10925/// the two inputs, and so can permute them into something that feeds a single
10926/// UNPCK instruction. Note that this routine only targets integer vectors
10927/// because for floating point vectors we have a generalized SHUFPS lowering
10928/// strategy that handles everything that doesn't *exactly* match an unpack,
10929/// making this clever lowering unnecessary.
10930static SDValue lowerShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT,
10931 SDValue V1, SDValue V2,
10932 ArrayRef<int> Mask,
10933 const X86Subtarget &Subtarget,
10934 SelectionDAG &DAG) {
10935 int Size = Mask.size();
10936 assert(Mask.size() >= 2 && "Single element masks are invalid.");
10937
10938 // This routine only supports 128-bit integer dual input vectors.
10939 if (VT.isFloatingPoint() || !VT.is128BitVector() || V2.isUndef())
10940 return SDValue();
10941
10942 int NumLoInputs =
10943 count_if(Mask, [Size](int M) { return M >= 0 && M % Size < Size / 2; });
10944 int NumHiInputs =
10945 count_if(Mask, [Size](int M) { return M % Size >= Size / 2; });
10946
10947 bool UnpackLo = NumLoInputs >= NumHiInputs;
10948
10949 auto TryUnpack = [&](int ScalarSize, int Scale) {
10950 SmallVector<int, 16> V1Mask((unsigned)Size, -1);
10951 SmallVector<int, 16> V2Mask((unsigned)Size, -1);
10952
10953 for (int i = 0; i < Size; ++i) {
10954 if (Mask[i] < 0)
10955 continue;
10956
10957 // Each element of the unpack contains Scale elements from this mask.
10958 int UnpackIdx = i / Scale;
10959
10960 // We only handle the case where V1 feeds the first slots of the unpack.
10961 // We rely on canonicalization to ensure this is the case.
10962 if ((UnpackIdx % 2 == 0) != (Mask[i] < Size))
10963 return SDValue();
10964
10965 // Setup the mask for this input. The indexing is tricky as we have to
10966 // handle the unpack stride.
10967 SmallVectorImpl<int> &VMask = (UnpackIdx % 2 == 0) ? V1Mask : V2Mask;
10968 VMask[(UnpackIdx / 2) * Scale + i % Scale + (UnpackLo ? 0 : Size / 2)] =
10969 Mask[i] % Size;
10970 }
10971
10972 // If we will have to shuffle both inputs to use the unpack, check whether
10973 // we can just unpack first and shuffle the result. If so, skip this unpack.
10974 if ((NumLoInputs == 0 || NumHiInputs == 0) && !isNoopShuffleMask(V1Mask) &&
10975 !isNoopShuffleMask(V2Mask))
10976 return SDValue();
10977
10978 // Shuffle the inputs into place.
10979 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
10980 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
10981
10982 // Cast the inputs to the type we will use to unpack them.
10983 MVT UnpackVT =
10984 MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), Size / Scale);
10985 V1 = DAG.getBitcast(UnpackVT, V1);
10986 V2 = DAG.getBitcast(UnpackVT, V2);
10987
10988 // Unpack the inputs and cast the result back to the desired type.
10989 return DAG.getBitcast(
10990 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
10991 UnpackVT, V1, V2));
10992 };
10993
10994 // We try each unpack from the largest to the smallest to try and find one
10995 // that fits this mask.
10996 int OrigScalarSize = VT.getScalarSizeInBits();
10997 for (int ScalarSize = 64; ScalarSize >= OrigScalarSize; ScalarSize /= 2)
10998 if (SDValue Unpack = TryUnpack(ScalarSize, ScalarSize / OrigScalarSize))
10999 return Unpack;
11000
11001 // If we're shuffling with a zero vector then we're better off not doing
11002 // VECTOR_SHUFFLE(UNPCK()) as we lose track of those zero elements.
11003 if (ISD::isBuildVectorAllZeros(V1.getNode()) ||
11004 ISD::isBuildVectorAllZeros(V2.getNode()))
11005 return SDValue();
11006
11007 // If none of the unpack-rooted lowerings worked (or were profitable) try an
11008 // initial unpack.
11009 if (NumLoInputs == 0 || NumHiInputs == 0) {
11010 assert((NumLoInputs > 0 || NumHiInputs > 0) &&
11011 "We have to have *some* inputs!");
11012 int HalfOffset = NumLoInputs == 0 ? Size / 2 : 0;
11013
11014 // FIXME: We could consider the total complexity of the permute of each
11015 // possible unpacking. Or at the least we should consider how many
11016 // half-crossings are created.
11017 // FIXME: We could consider commuting the unpacks.
11018
11019 SmallVector<int, 32> PermMask((unsigned)Size, -1);
11020 for (int i = 0; i < Size; ++i) {
11021 if (Mask[i] < 0)
11022 continue;
11023
11024 assert(Mask[i] % Size >= HalfOffset && "Found input from wrong half!");
11025
11026 PermMask[i] =
11027 2 * ((Mask[i] % Size) - HalfOffset) + (Mask[i] < Size ? 0 : 1);
11028 }
11029 return DAG.getVectorShuffle(
11030 VT, DL,
11031 DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, DL, VT,
11032 V1, V2),
11033 DAG.getUNDEF(VT), PermMask);
11034 }
11035
11036 return SDValue();
11037}
11038
11039/// Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then
11040/// permuting the elements of the result in place.
11041static SDValue lowerShuffleAsByteRotateAndPermute(
11042 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11043 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11044 if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) ||
11045 (VT.is256BitVector() && !Subtarget.hasAVX2()) ||
11046 (VT.is512BitVector() && !Subtarget.hasBWI()))
11047 return SDValue();
11048
11049 // We don't currently support lane crossing permutes.
11050 if (is128BitLaneCrossingShuffleMask(VT, Mask))
11051 return SDValue();
11052
11053 int Scale = VT.getScalarSizeInBits() / 8;
11054 int NumLanes = VT.getSizeInBits() / 128;
11055 int NumElts = VT.getVectorNumElements();
11056 int NumEltsPerLane = NumElts / NumLanes;
11057
11058 // Determine range of mask elts.
11059 bool Blend1 = true;
11060 bool Blend2 = true;
11061 std::pair<int, int> Range1 = std::make_pair(INT_MAX, INT_MIN);
11062 std::pair<int, int> Range2 = std::make_pair(INT_MAX, INT_MIN);
11063 for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) {
11064 for (int Elt = 0; Elt != NumEltsPerLane; ++Elt) {
11065 int M = Mask[Lane + Elt];
11066 if (M < 0)
11067 continue;
11068 if (M < NumElts) {
11069 Blend1 &= (M == (Lane + Elt));
11070 assert(Lane <= M && M < (Lane + NumEltsPerLane) && "Out of range mask");
11071 M = M % NumEltsPerLane;
11072 Range1.first = std::min(Range1.first, M);
11073 Range1.second = std::max(Range1.second, M);
11074 } else {
11075 M -= NumElts;
11076 Blend2 &= (M == (Lane + Elt));
11077 assert(Lane <= M && M < (Lane + NumEltsPerLane) && "Out of range mask");
11078 M = M % NumEltsPerLane;
11079 Range2.first = std::min(Range2.first, M);
11080 Range2.second = std::max(Range2.second, M);
11081 }
11082 }
11083 }
11084
11085 // Bail if we don't need both elements.
11086 // TODO - it might be worth doing this for unary shuffles if the permute
11087 // can be widened.
11088 if (!(0 <= Range1.first && Range1.second < NumEltsPerLane) ||
11089 !(0 <= Range2.first && Range2.second < NumEltsPerLane))
11090 return SDValue();
11091
11092 if (VT.getSizeInBits() > 128 && (Blend1 || Blend2))
11093 return SDValue();
11094
11095 // Rotate the 2 ops so we can access both ranges, then permute the result.
11096 auto RotateAndPermute = [&](SDValue Lo, SDValue Hi, int RotAmt, int Ofs) {
11097 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
11098 SDValue Rotate = DAG.getBitcast(
11099 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi),
11100 DAG.getBitcast(ByteVT, Lo),
11101 DAG.getTargetConstant(Scale * RotAmt, DL, MVT::i8)));
11102 SmallVector<int, 64> PermMask(NumElts, SM_SentinelUndef);
11103 for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) {
11104 for (int Elt = 0; Elt != NumEltsPerLane; ++Elt) {
11105 int M = Mask[Lane + Elt];
11106 if (M < 0)
11107 continue;
11108 if (M < NumElts)
11109 PermMask[Lane + Elt] = Lane + ((M + Ofs - RotAmt) % NumEltsPerLane);
11110 else
11111 PermMask[Lane + Elt] = Lane + ((M - Ofs - RotAmt) % NumEltsPerLane);
11112 }
11113 }
11114 return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask);
11115 };
11116
11117 // Check if the ranges are small enough to rotate from either direction.
11118 if (Range2.second < Range1.first)
11119 return RotateAndPermute(V1, V2, Range1.first, 0);
11120 if (Range1.second < Range2.first)
11121 return RotateAndPermute(V2, V1, Range2.first, NumElts);
11122 return SDValue();
11123}
11124
11125static bool isBroadcastShuffleMask(ArrayRef<int> Mask) {
11126 return isUndefOrEqual(Mask, 0);
11127}
11128
11129static bool isNoopOrBroadcastShuffleMask(ArrayRef<int> Mask) {
11130 return isNoopShuffleMask(Mask) || isBroadcastShuffleMask(Mask);
11131}
11132
11133/// Check if the Mask consists of the same element repeated multiple times.
11134static bool isSingleElementRepeatedMask(ArrayRef<int> Mask) {
11135 size_t NumUndefs = 0;
11136 std::optional<int> UniqueElt;
11137 for (int Elt : Mask) {
11138 if (Elt == SM_SentinelUndef) {
11139 NumUndefs++;
11140 continue;
11141 }
11142 if (UniqueElt.has_value() && UniqueElt.value() != Elt)
11143 return false;
11144 UniqueElt = Elt;
11145 }
11146 // Make sure the element is repeated enough times by checking the number of
11147 // undefs is small.
11148 return NumUndefs <= Mask.size() / 2 && UniqueElt.has_value();
11149}
11150
11151/// Generic routine to decompose a shuffle and blend into independent
11152/// blends and permutes.
11153///
11154/// This matches the extremely common pattern for handling combined
11155/// shuffle+blend operations on newer X86 ISAs where we have very fast blend
11156/// operations. It will try to pick the best arrangement of shuffles and
11157/// blends. For vXi8/vXi16 shuffles we may use unpack instead of blend.
11158static SDValue lowerShuffleAsDecomposedShuffleMerge(
11159 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11160 const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11161 int NumElts = Mask.size();
11162 int NumLanes = VT.getSizeInBits() / 128;
11163 int NumEltsPerLane = NumElts / NumLanes;
11164
11165 // Shuffle the input elements into the desired positions in V1 and V2 and
11166 // unpack/blend them together.
11167 bool IsAlternating = true;
11168 bool V1Zero = true, V2Zero = true;
11169 SmallVector<int, 32> V1Mask(NumElts, -1);
11170 SmallVector<int, 32> V2Mask(NumElts, -1);
11171 SmallVector<int, 32> FinalMask(NumElts, -1);
11172 for (int i = 0; i < NumElts; ++i) {
11173 int M = Mask[i];
11174 if (M >= 0 && M < NumElts) {
11175 V1Mask[i] = M;
11176 FinalMask[i] = i;
11177 V1Zero &= Zeroable[i];
11178 IsAlternating &= (i & 1) == 0;
11179 } else if (M >= NumElts) {
11180 V2Mask[i] = M - NumElts;
11181 FinalMask[i] = i + NumElts;
11182 V2Zero &= Zeroable[i];
11183 IsAlternating &= (i & 1) == 1;
11184 }
11185 }
11186
11187 // If we effectively only demand the 0'th element of \p Input, and not only
11188 // as 0'th element, then broadcast said input,
11189 // and change \p InputMask to be a no-op (identity) mask.
11190 auto canonicalizeBroadcastableInput = [DL, VT, &Subtarget,
11191 &DAG](SDValue &Input,
11192 MutableArrayRef<int> InputMask) {
11193 unsigned EltSizeInBits = Input.getScalarValueSizeInBits();
11194 if (!Subtarget.hasAVX2() && (!Subtarget.hasAVX() || EltSizeInBits < 32 ||
11195 !X86::mayFoldLoad(Input, Subtarget)))
11196 return;
11197 if (isNoopShuffleMask(InputMask))
11198 return;
11199 assert(isBroadcastShuffleMask(InputMask) &&
11200 "Expected to demand only the 0'th element.");
11201 Input = DAG.getNode(X86ISD::VBROADCAST, DL, VT, Input);
11202 for (auto I : enumerate(InputMask)) {
11203 int &InputMaskElt = I.value();
11204 if (InputMaskElt >= 0)
11205 InputMaskElt = I.index();
11206 }
11207 };
11208
11209 // Currently, we may need to produce one shuffle per input, and blend results.
11210 // It is possible that the shuffle for one of the inputs is already a no-op.
11211 // See if we can simplify non-no-op shuffles into broadcasts,
11212 // which we consider to be strictly better than an arbitrary shuffle.
11213 if (isNoopOrBroadcastShuffleMask(V1Mask) &&
11214 isNoopOrBroadcastShuffleMask(V2Mask)) {
11215 canonicalizeBroadcastableInput(V1, V1Mask);
11216 canonicalizeBroadcastableInput(V2, V2Mask);
11217 }
11218
11219 // Try to lower with the simpler initial blend/unpack/rotate strategies unless
11220 // one of the input shuffles would be a no-op. We prefer to shuffle inputs as
11221 // the shuffle may be able to fold with a load or other benefit. However, when
11222 // we'll have to do 2x as many shuffles in order to achieve this, a 2-input
11223 // pre-shuffle first is a better strategy.
11224 if (!isNoopShuffleMask(V1Mask) && !isNoopShuffleMask(V2Mask)) {
11225 // Only prefer immediate blends to unpack/rotate.
11226 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11227 DAG, true))
11228 return BlendPerm;
11229 // If either input vector provides only a single element which is repeated
11230 // multiple times, unpacking from both input vectors would generate worse
11231 // code. e.g. for
11232 // t5: v16i8 = vector_shuffle<16,0,16,1,16,2,16,3,16,4,16,5,16,6,16,7> t2, t4
11233 // it is better to process t4 first to create a vector of t4[0], then unpack
11234 // that vector with t2.
11235 if (!V1Zero && !V2Zero && !isSingleElementRepeatedMask(V1Mask) &&
11236 !isSingleElementRepeatedMask(V2Mask))
11237 if (SDValue UnpackPerm =
11238 lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG))
11239 return UnpackPerm;
11240 if (SDValue RotatePerm = lowerShuffleAsByteRotateAndPermute(
11241 DL, VT, V1, V2, Mask, Subtarget, DAG))
11242 return RotatePerm;
11243 // Unpack/rotate failed - try again with variable blends.
11244 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11245 DAG))
11246 return BlendPerm;
11247 if (VT.getScalarSizeInBits() >= 32)
11248 if (SDValue PermUnpack = lowerShuffleAsPermuteAndUnpack(
11249 DL, VT, V1, V2, Mask, Subtarget, DAG))
11250 return PermUnpack;
11251 }
11252
11253 // If the final mask is an alternating blend of vXi8/vXi16, convert to an
11254 // UNPCKL(SHUFFLE, SHUFFLE) pattern.
11255 // TODO: It doesn't have to be alternating - but each lane mustn't have more
11256 // than half the elements coming from each source.
11257 if (IsAlternating && VT.getScalarSizeInBits() < 32) {
11258 V1Mask.assign(NumElts, -1);
11259 V2Mask.assign(NumElts, -1);
11260 FinalMask.assign(NumElts, -1);
11261 for (int i = 0; i != NumElts; i += NumEltsPerLane)
11262 for (int j = 0; j != NumEltsPerLane; ++j) {
11263 int M = Mask[i + j];
11264 if (M >= 0 && M < NumElts) {
11265 V1Mask[i + (j / 2)] = M;
11266 FinalMask[i + j] = i + (j / 2);
11267 } else if (M >= NumElts) {
11268 V2Mask[i + (j / 2)] = M - NumElts;
11269 FinalMask[i + j] = i + (j / 2) + NumElts;
11270 }
11271 }
11272 }
11273
11274 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
11275 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
11276 return DAG.getVectorShuffle(VT, DL, V1, V2, FinalMask);
11277}
11278
11279static int matchShuffleAsBitRotate(MVT &RotateVT, int EltSizeInBits,
11280 const X86Subtarget &Subtarget,
11281 ArrayRef<int> Mask) {
11282 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11283 assert(EltSizeInBits < 64 && "Can't rotate 64-bit integers");
11284
11285 // AVX512 only has vXi32/vXi64 rotates, so limit the rotation sub group size.
11286 int MinSubElts = Subtarget.hasAVX512() ? std::max(32 / EltSizeInBits, 2) : 2;
11287 int MaxSubElts = 64 / EltSizeInBits;
11288 unsigned RotateAmt, NumSubElts;
11289 if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, MinSubElts,
11290 MaxSubElts, NumSubElts, RotateAmt))
11291 return -1;
11292 unsigned NumElts = Mask.size();
11293 MVT RotateSVT = MVT::getIntegerVT(EltSizeInBits * NumSubElts);
11294 RotateVT = MVT::getVectorVT(RotateSVT, NumElts / NumSubElts);
11295 return RotateAmt;
11296}
11297
11298/// Lower shuffle using X86ISD::VROTLI rotations.
11299static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1,
11300 ArrayRef<int> Mask,
11301 const X86Subtarget &Subtarget,
11302 SelectionDAG &DAG) {
11303 // Only XOP + AVX512 targets have bit rotation instructions.
11304 // If we at least have SSSE3 (PSHUFB) then we shouldn't attempt to use this.
11305 bool IsLegal =
11306 (VT.is128BitVector() && Subtarget.hasXOP()) || Subtarget.hasAVX512();
11307 if (!IsLegal && Subtarget.hasSSE3())
11308 return SDValue();
11309
11310 MVT RotateVT;
11311 int RotateAmt = matchShuffleAsBitRotate(RotateVT, VT.getScalarSizeInBits(),
11312 Subtarget, Mask);
11313 if (RotateAmt < 0)
11314 return SDValue();
11315
11316 // For pre-SSSE3 targets, if we are shuffling vXi8 elts then ISD::ROTL,
11317 // expanded to OR(SRL,SHL), will be more efficient, but if they can
11318 // widen to vXi16 or more then existing lowering should will be better.
11319 if (!IsLegal) {
11320 if ((RotateAmt % 16) == 0)
11321 return SDValue();
11322 // TODO: Use getTargetVShiftByConstNode.
11323 unsigned ShlAmt = RotateAmt;
11324 unsigned SrlAmt = RotateVT.getScalarSizeInBits() - RotateAmt;
11325 V1 = DAG.getBitcast(RotateVT, V1);
11326 SDValue SHL = DAG.getNode(X86ISD::VSHLI, DL, RotateVT, V1,
11327 DAG.getTargetConstant(ShlAmt, DL, MVT::i8));
11328 SDValue SRL = DAG.getNode(X86ISD::VSRLI, DL, RotateVT, V1,
11329 DAG.getTargetConstant(SrlAmt, DL, MVT::i8));
11330 SDValue Rot = DAG.getNode(ISD::OR, DL, RotateVT, SHL, SRL);
11331 return DAG.getBitcast(VT, Rot);
11332 }
11333
11334 SDValue Rot =
11335 DAG.getNode(X86ISD::VROTLI, DL, RotateVT, DAG.getBitcast(RotateVT, V1),
11336 DAG.getTargetConstant(RotateAmt, DL, MVT::i8));
11337 return DAG.getBitcast(VT, Rot);
11338}
11339
11340/// Try to match a vector shuffle as an element rotation.
11341///
11342/// This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.
11343static int matchShuffleAsElementRotate(SDValue &V1, SDValue &V2,
11344 ArrayRef<int> Mask) {
11345 int NumElts = Mask.size();
11346
11347 // We need to detect various ways of spelling a rotation:
11348 // [11, 12, 13, 14, 15, 0, 1, 2]
11349 // [-1, 12, 13, 14, -1, -1, 1, -1]
11350 // [-1, -1, -1, -1, -1, -1, 1, 2]
11351 // [ 3, 4, 5, 6, 7, 8, 9, 10]
11352 // [-1, 4, 5, 6, -1, -1, 9, -1]
11353 // [-1, 4, 5, 6, -1, -1, -1, -1]
11354 int Rotation = 0;
11355 SDValue Lo, Hi;
11356 for (int i = 0; i < NumElts; ++i) {
11357 int M = Mask[i];
11358 assert((M == SM_SentinelUndef || (0 <= M && M < (2*NumElts))) &&
11359 "Unexpected mask index.");
11360 if (M < 0)
11361 continue;
11362
11363 // Determine where a rotated vector would have started.
11364 int StartIdx = i - (M % NumElts);
11365 if (StartIdx == 0)
11366 // The identity rotation isn't interesting, stop.
11367 return -1;
11368
11369 // If we found the tail of a vector the rotation must be the missing
11370 // front. If we found the head of a vector, it must be how much of the
11371 // head.
11372 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
11373
11374 if (Rotation == 0)
11375 Rotation = CandidateRotation;
11376 else if (Rotation != CandidateRotation)
11377 // The rotations don't match, so we can't match this mask.
11378 return -1;
11379
11380 // Compute which value this mask is pointing at.
11381 SDValue MaskV = M < NumElts ? V1 : V2;
11382
11383 // Compute which of the two target values this index should be assigned
11384 // to. This reflects whether the high elements are remaining or the low
11385 // elements are remaining.
11386 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
11387
11388 // Either set up this value if we've not encountered it before, or check
11389 // that it remains consistent.
11390 if (!TargetV)
11391 TargetV = MaskV;
11392 else if (TargetV != MaskV)
11393 // This may be a rotation, but it pulls from the inputs in some
11394 // unsupported interleaving.
11395 return -1;
11396 }
11397
11398 // Check that we successfully analyzed the mask, and normalize the results.
11399 assert(Rotation != 0 && "Failed to locate a viable rotation!");
11400 assert((Lo || Hi) && "Failed to find a rotated input vector!");
11401 if (!Lo)
11402 Lo = Hi;
11403 else if (!Hi)
11404 Hi = Lo;
11405
11406 V1 = Lo;
11407 V2 = Hi;
11408
11409 return Rotation;
11410}
11411
11412/// Try to lower a vector shuffle as a byte rotation.
11413///
11414/// SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary
11415/// byte-rotation of the concatenation of two vectors; pre-SSSE3 can use
11416/// a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will
11417/// try to generically lower a vector shuffle through such an pattern. It
11418/// does not check for the profitability of lowering either as PALIGNR or
11419/// PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form.
11420/// This matches shuffle vectors that look like:
11421///
11422/// v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
11423///
11424/// Essentially it concatenates V1 and V2, shifts right by some number of
11425/// elements, and takes the low elements as the result. Note that while this is
11426/// specified as a *right shift* because x86 is little-endian, it is a *left
11427/// rotate* of the vector lanes.
11428static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
11429 ArrayRef<int> Mask) {
11430 // Don't accept any shuffles with zero elements.
11431 if (isAnyZero(Mask))
11432 return -1;
11433
11434 // PALIGNR works on 128-bit lanes.
11435 SmallVector<int, 16> RepeatedMask;
11436 if (!is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask))
11437 return -1;
11438
11439 int Rotation = matchShuffleAsElementRotate(V1, V2, RepeatedMask);
11440 if (Rotation <= 0)
11441 return -1;
11442
11443 // PALIGNR rotates bytes, so we need to scale the
11444 // rotation based on how many bytes are in the vector lane.
11445 int NumElts = RepeatedMask.size();
11446 int Scale = 16 / NumElts;
11447 return Rotation * Scale;
11448}
11449
11450static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
11451 SDValue V2, ArrayRef<int> Mask,
11452 const X86Subtarget &Subtarget,
11453 SelectionDAG &DAG) {
11454 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11455
11456 SDValue Lo = V1, Hi = V2;
11457 int ByteRotation = matchShuffleAsByteRotate(VT, Lo, Hi, Mask);
11458 if (ByteRotation <= 0)
11459 return SDValue();
11460
11461 // Cast the inputs to i8 vector of correct length to match PALIGNR or
11462 // PSLLDQ/PSRLDQ.
11463 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
11464 Lo = DAG.getBitcast(ByteVT, Lo);
11465 Hi = DAG.getBitcast(ByteVT, Hi);
11466
11467 // SSSE3 targets can use the palignr instruction.
11468 if (Subtarget.hasSSSE3()) {
11469 assert((!VT.is512BitVector() || Subtarget.hasBWI()) &&
11470 "512-bit PALIGNR requires BWI instructions");
11471 return DAG.getBitcast(
11472 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,
11473 DAG.getTargetConstant(ByteRotation, DL, MVT::i8)));
11474 }
11475
11476 assert(VT.is128BitVector() &&
11477 "Rotate-based lowering only supports 128-bit lowering!");
11478 assert(Mask.size() <= 16 &&
11479 "Can shuffle at most 16 bytes in a 128-bit vector!");
11480 assert(ByteVT == MVT::v16i8 &&
11481 "SSE2 rotate lowering only needed for v16i8!");
11482
11483 // Default SSE2 implementation
11484 int LoByteShift = 16 - ByteRotation;
11485 int HiByteShift = ByteRotation;
11486
11487 SDValue LoShift =
11488 DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
11489 DAG.getTargetConstant(LoByteShift, DL, MVT::i8));
11490 SDValue HiShift =
11491 DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
11492 DAG.getTargetConstant(HiByteShift, DL, MVT::i8));
11493 return DAG.getBitcast(VT,
11494 DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift));
11495}
11496
11497/// Try to lower a vector shuffle as a dword/qword rotation.
11498///
11499/// AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary
11500/// rotation of the concatenation of two vectors; This routine will
11501/// try to generically lower a vector shuffle through such an pattern.
11502///
11503/// Essentially it concatenates V1 and V2, shifts right by some number of
11504/// elements, and takes the low elements as the result. Note that while this is
11505/// specified as a *right shift* because x86 is little-endian, it is a *left
11506/// rotate* of the vector lanes.
11507static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,
11508 SDValue V2, ArrayRef<int> Mask,
11509 const APInt &Zeroable,
11510 const X86Subtarget &Subtarget,
11511 SelectionDAG &DAG) {
11512 assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
11513 "Only 32-bit and 64-bit elements are supported!");
11514
11515 // 128/256-bit vectors are only supported with VLX.
11516 assert((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector()))
11517 && "VLX required for 128/256-bit vectors");
11518
11519 SDValue Lo = V1, Hi = V2;
11520 int Rotation = matchShuffleAsElementRotate(Lo, Hi, Mask);
11521 if (0 < Rotation)
11522 return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,
11523 DAG.getTargetConstant(Rotation, DL, MVT::i8));
11524
11525 // See if we can use VALIGN as a cross-lane version of VSHLDQ/VSRLDQ.
11526 // TODO: Pull this out as a matchShuffleAsElementShift helper?
11527 // TODO: We can probably make this more aggressive and use shift-pairs like
11528 // lowerShuffleAsByteShiftMask.
11529 unsigned NumElts = Mask.size();
11530 unsigned ZeroLo = Zeroable.countr_one();
11531 unsigned ZeroHi = Zeroable.countl_one();
11532 assert((ZeroLo + ZeroHi) < NumElts && "Zeroable shuffle detected");
11533 if (!ZeroLo && !ZeroHi)
11534 return SDValue();
11535
11536 if (ZeroLo) {
11537 SDValue Src = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11538 int Low = Mask[ZeroLo] < (int)NumElts ? 0 : NumElts;
11539 if (isSequentialOrUndefInRange(Mask, ZeroLo, NumElts - ZeroLo, Low))
11540 return DAG.getNode(X86ISD::VALIGN, DL, VT, Src,
11541 getZeroVector(VT, Subtarget, DAG, DL),
11542 DAG.getTargetConstant(NumElts - ZeroLo, DL, MVT::i8));
11543 }
11544
11545 if (ZeroHi) {
11546 SDValue Src = Mask[0] < (int)NumElts ? V1 : V2;
11547 int Low = Mask[0] < (int)NumElts ? 0 : NumElts;
11548 if (isSequentialOrUndefInRange(Mask, 0, NumElts - ZeroHi, Low + ZeroHi))
11549 return DAG.getNode(X86ISD::VALIGN, DL, VT,
11550 getZeroVector(VT, Subtarget, DAG, DL), Src,
11551 DAG.getTargetConstant(ZeroHi, DL, MVT::i8));
11552 }
11553
11554 return SDValue();
11555}
11556
11557/// Try to lower a vector shuffle as a byte shift sequence.
11558static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1,
11559 SDValue V2, ArrayRef<int> Mask,
11560 const APInt &Zeroable,
11561 const X86Subtarget &Subtarget,
11562 SelectionDAG &DAG) {
11563 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11564 assert(VT.is128BitVector() && "Only 128-bit vectors supported");
11565
11566 // We need a shuffle that has zeros at one/both ends and a sequential
11567 // shuffle from one source within.
11568 unsigned ZeroLo = Zeroable.countr_one();
11569 unsigned ZeroHi = Zeroable.countl_one();
11570 if (!ZeroLo && !ZeroHi)
11571 return SDValue();
11572
11573 unsigned NumElts = Mask.size();
11574 unsigned Len = NumElts - (ZeroLo + ZeroHi);
11575 if (!isSequentialOrUndefInRange(Mask, ZeroLo, Len, Mask[ZeroLo]))
11576 return SDValue();
11577
11578 unsigned Scale = VT.getScalarSizeInBits() / 8;
11579 ArrayRef<int> StubMask = Mask.slice(ZeroLo, Len);
11580 if (!isUndefOrInRange(StubMask, 0, NumElts) &&
11581 !isUndefOrInRange(StubMask, NumElts, 2 * NumElts))
11582 return SDValue();
11583
11584 SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11585 Res = DAG.getBitcast(MVT::v16i8, Res);
11586
11587 // Use VSHLDQ/VSRLDQ ops to zero the ends of a vector and leave an
11588 // inner sequential set of elements, possibly offset:
11589 // 01234567 --> zzzzzz01 --> 1zzzzzzz
11590 // 01234567 --> 4567zzzz --> zzzzz456
11591 // 01234567 --> z0123456 --> 3456zzzz --> zz3456zz
11592 if (ZeroLo == 0) {
11593 unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
11594 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11595 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11596 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11597 DAG.getTargetConstant(Scale * ZeroHi, DL, MVT::i8));
11598 } else if (ZeroHi == 0) {
11599 unsigned Shift = Mask[ZeroLo] % NumElts;
11600 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11601 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11602 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11603 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11604 } else if (!Subtarget.hasSSSE3()) {
11605 // If we don't have PSHUFB then its worth avoiding an AND constant mask
11606 // by performing 3 byte shifts. Shuffle combining can kick in above that.
11607 // TODO: There may be some cases where VSH{LR}DQ+PAND is still better.
11608 unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
11609 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11610 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11611 Shift += Mask[ZeroLo] % NumElts;
11612 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11613 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11614 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11615 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11616 } else
11617 return SDValue();
11618
11619 return DAG.getBitcast(VT, Res);
11620}
11621
11622/// Try to lower a vector shuffle as a bit shift (shifts in zeros).
11623///
11624/// Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and
11625/// PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function
11626/// matches elements from one of the input vectors shuffled to the left or
11627/// right with zeroable elements 'shifted in'. It handles both the strictly
11628/// bit-wise element shifts and the byte shift across an entire 128-bit double
11629/// quad word lane.
11630///
11631/// PSHL : (little-endian) left bit shift.
11632/// [ zz, 0, zz, 2 ]
11633/// [ -1, 4, zz, -1 ]
11634/// PSRL : (little-endian) right bit shift.
11635/// [ 1, zz, 3, zz]
11636/// [ -1, -1, 7, zz]
11637/// PSLLDQ : (little-endian) left byte shift
11638/// [ zz, 0, 1, 2, 3, 4, 5, 6]
11639/// [ zz, zz, -1, -1, 2, 3, 4, -1]
11640/// [ zz, zz, zz, zz, zz, zz, -1, 1]
11641/// PSRLDQ : (little-endian) right byte shift
11642/// [ 5, 6, 7, zz, zz, zz, zz, zz]
11643/// [ -1, 5, 6, 7, zz, zz, zz, zz]
11644/// [ 1, 2, -1, -1, -1, -1, zz, zz]
11645static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
11646 unsigned ScalarSizeInBits, ArrayRef<int> Mask,
11647 int MaskOffset, const APInt &Zeroable,
11648 const X86Subtarget &Subtarget) {
11649 int Size = Mask.size();
11650 unsigned SizeInBits = Size * ScalarSizeInBits;
11651
11652 auto CheckZeros = [&](int Shift, int Scale, bool Left) {
11653 for (int i = 0; i < Size; i += Scale)
11654 for (int j = 0; j < Shift; ++j)
11655 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
11656 return false;
11657
11658 return true;
11659 };
11660
11661 auto MatchShift = [&](int Shift, int Scale, bool Left) {
11662 for (int i = 0; i != Size; i += Scale) {
11663 unsigned Pos = Left ? i + Shift : i;
11664 unsigned Low = Left ? i : i + Shift;
11665 unsigned Len = Scale - Shift;
11666 if (!isSequentialOrUndefInRange(Mask, Pos, Len, Low + MaskOffset))
11667 return -1;
11668 }
11669
11670 int ShiftEltBits = ScalarSizeInBits * Scale;
11671 bool ByteShift = ShiftEltBits > 64;
11672 Opcode = Left ? (ByteShift ? X86ISD::VSHLDQ : X86ISD::VSHLI)
11673 : (ByteShift ? X86ISD::VSRLDQ : X86ISD::VSRLI);
11674 int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
11675
11676 // Normalize the scale for byte shifts to still produce an i64 element
11677 // type.
11678 Scale = ByteShift ? Scale / 2 : Scale;
11679
11680 // We need to round trip through the appropriate type for the shift.
11681 MVT ShiftSVT = MVT::getIntegerVT(ScalarSizeInBits * Scale);
11682 ShiftVT = ByteShift ? MVT::getVectorVT(MVT::i8, SizeInBits / 8)
11683 : MVT::getVectorVT(ShiftSVT, Size / Scale);
11684 return (int)ShiftAmt;
11685 };
11686
11687 // SSE/AVX supports logical shifts up to 64-bit integers - so we can just
11688 // keep doubling the size of the integer elements up to that. We can
11689 // then shift the elements of the integer vector by whole multiples of
11690 // their width within the elements of the larger integer vector. Test each
11691 // multiple to see if we can find a match with the moved element indices
11692 // and that the shifted in elements are all zeroable.
11693 unsigned MaxWidth = ((SizeInBits == 512) && !Subtarget.hasBWI() ? 64 : 128);
11694 for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
11695 for (int Shift = 1; Shift != Scale; ++Shift)
11696 for (bool Left : {true, false})
11697 if (CheckZeros(Shift, Scale, Left)) {
11698 int ShiftAmt = MatchShift(Shift, Scale, Left);
11699 if (0 < ShiftAmt)
11700 return ShiftAmt;
11701 }
11702
11703 // no match
11704 return -1;
11705}
11706
11707static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
11708 SDValue V2, ArrayRef<int> Mask,
11709 const APInt &Zeroable,
11710 const X86Subtarget &Subtarget,
11711 SelectionDAG &DAG, bool BitwiseOnly) {
11712 int Size = Mask.size();
11713 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
11714
11715 MVT ShiftVT;
11716 SDValue V = V1;
11717 unsigned Opcode;
11718
11719 // Try to match shuffle against V1 shift.
11720 int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
11721 Mask, 0, Zeroable, Subtarget);
11722
11723 // If V1 failed, try to match shuffle against V2 shift.
11724 if (ShiftAmt < 0) {
11725 ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
11726 Mask, Size, Zeroable, Subtarget);
11727 V = V2;
11728 }
11729
11730 if (ShiftAmt < 0)
11731 return SDValue();
11732
11733 if (BitwiseOnly && (Opcode == X86ISD::VSHLDQ || Opcode == X86ISD::VSRLDQ))
11734 return SDValue();
11735
11736 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
11737 "Illegal integer vector type");
11738 V = DAG.getBitcast(ShiftVT, V);
11739 V = DAG.getNode(Opcode, DL, ShiftVT, V,
11740 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
11741 return DAG.getBitcast(VT, V);
11742}
11743
11744// EXTRQ: Extract Len elements from lower half of source, starting at Idx.
11745// Remainder of lower half result is zero and upper half is all undef.
11746static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
11747 ArrayRef<int> Mask, uint64_t &BitLen,
11748 uint64_t &BitIdx, const APInt &Zeroable) {
11749 int Size = Mask.size();
11750 int HalfSize = Size / 2;
11751 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
11752 assert(!Zeroable.isAllOnes() && "Fully zeroable shuffle mask");
11753
11754 // Upper half must be undefined.
11755 if (!isUndefUpperHalf(Mask))
11756 return false;
11757
11758 // Determine the extraction length from the part of the
11759 // lower half that isn't zeroable.
11760 int Len = HalfSize;
11761 for (; Len > 0; --Len)
11762 if (!Zeroable[Len - 1])
11763 break;
11764 assert(Len > 0 && "Zeroable shuffle mask");
11765
11766 // Attempt to match first Len sequential elements from the lower half.
11767 SDValue Src;
11768 int Idx = -1;
11769 for (int i = 0; i != Len; ++i) {
11770 int M = Mask[i];
11771 if (M == SM_SentinelUndef)
11772 continue;
11773 SDValue &V = (M < Size ? V1 : V2);
11774 M = M % Size;
11775
11776 // The extracted elements must start at a valid index and all mask
11777 // elements must be in the lower half.
11778 if (i > M || M >= HalfSize)
11779 return false;
11780
11781 if (Idx < 0 || (Src == V && Idx == (M - i))) {
11782 Src = V;
11783 Idx = M - i;
11784 continue;
11785 }
11786 return false;
11787 }
11788
11789 if (!Src || Idx < 0)
11790 return false;
11791
11792 assert((Idx + Len) <= HalfSize && "Illegal extraction mask");
11793 BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
11794 BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
11795 V1 = Src;
11796 return true;
11797}
11798
11799// INSERTQ: Extract lowest Len elements from lower half of second source and
11800// insert over first source, starting at Idx.
11801// { A[0], .., A[Idx-1], B[0], .., B[Len-1], A[Idx+Len], .., UNDEF, ... }
11802static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
11803 ArrayRef<int> Mask, uint64_t &BitLen,
11804 uint64_t &BitIdx) {
11805 int Size = Mask.size();
11806 int HalfSize = Size / 2;
11807 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
11808
11809 // Upper half must be undefined.
11810 if (!isUndefUpperHalf(Mask))
11811 return false;
11812
11813 for (int Idx = 0; Idx != HalfSize; ++Idx) {
11814 SDValue Base;
11815
11816 // Attempt to match first source from mask before insertion point.
11817 if (isUndefInRange(Mask, 0, Idx)) {
11818 /* EMPTY */
11819 } else if (isSequentialOrUndefInRange(Mask, 0, Idx, 0)) {
11820 Base = V1;
11821 } else if (isSequentialOrUndefInRange(Mask, 0, Idx, Size)) {
11822 Base = V2;
11823 } else {
11824 continue;
11825 }
11826
11827 // Extend the extraction length looking to match both the insertion of
11828 // the second source and the remaining elements of the first.
11829 for (int Hi = Idx + 1; Hi <= HalfSize; ++Hi) {
11830 SDValue Insert;
11831 int Len = Hi - Idx;
11832
11833 // Match insertion.
11834 if (isSequentialOrUndefInRange(Mask, Idx, Len, 0)) {
11835 Insert = V1;
11836 } else if (isSequentialOrUndefInRange(Mask, Idx, Len, Size)) {
11837 Insert = V2;
11838 } else {
11839 continue;
11840 }
11841
11842 // Match the remaining elements of the lower half.
11843 if (isUndefInRange(Mask, Hi, HalfSize - Hi)) {
11844 /* EMPTY */
11845 } else if ((!Base || (Base == V1)) &&
11846 isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, Hi)) {
11847 Base = V1;
11848 } else if ((!Base || (Base == V2)) &&
11849 isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi,
11850 Size + Hi)) {
11851 Base = V2;
11852 } else {
11853 continue;
11854 }
11855
11856 BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
11857 BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
11858 V1 = Base;
11859 V2 = Insert;
11860 return true;
11861 }
11862 }
11863
11864 return false;
11865}
11866
11867/// Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
11868static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
11869 SDValue V2, ArrayRef<int> Mask,
11870 const APInt &Zeroable, SelectionDAG &DAG) {
11871 uint64_t BitLen, BitIdx;
11872 if (matchShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))
11873 return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,
11874 DAG.getTargetConstant(BitLen, DL, MVT::i8),
11875 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
11876
11877 if (matchShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx))
11878 return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),
11879 V2 ? V2 : DAG.getUNDEF(VT),
11880 DAG.getTargetConstant(BitLen, DL, MVT::i8),
11881 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
11882
11883 return SDValue();
11884}
11885
11886/// Lower a vector shuffle as a zero or any extension.
11887///
11888/// Given a specific number of elements, element bit width, and extension
11889/// stride, produce either a zero or any extension based on the available
11890/// features of the subtarget. The extended elements are consecutive and
11891/// begin and can start from an offsetted element index in the input; to
11892/// avoid excess shuffling the offset must either being in the bottom lane
11893/// or at the start of a higher lane. All extended elements must be from
11894/// the same lane.
11895static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
11896 const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,
11897 ArrayRef<int> Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11898 assert(Scale > 1 && "Need a scale to extend.");
11899 int EltBits = VT.getScalarSizeInBits();
11900 int NumElements = VT.getVectorNumElements();
11901 int NumEltsPerLane = 128 / EltBits;
11902 int OffsetLane = Offset / NumEltsPerLane;
11903 assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
11904 "Only 8, 16, and 32 bit elements can be extended.");
11905 assert(Scale * EltBits <= 64 && "Cannot zero extend past 64 bits.");
11906 assert(0 <= Offset && "Extension offset must be positive.");
11907 assert((Offset < NumEltsPerLane || Offset % NumEltsPerLane == 0) &&
11908 "Extension offset must be in the first lane or start an upper lane.");
11909
11910 // Check that an index is in same lane as the base offset.
11911 auto SafeOffset = [&](int Idx) {
11912 return OffsetLane == (Idx / NumEltsPerLane);
11913 };
11914
11915 // Shift along an input so that the offset base moves to the first element.
11916 auto ShuffleOffset = [&](SDValue V) {
11917 if (!Offset)
11918 return V;
11919
11920 SmallVector<int, 8> ShMask((unsigned)NumElements, -1);
11921 for (int i = 0; i * Scale < NumElements; ++i) {
11922 int SrcIdx = i + Offset;
11923 ShMask[i] = SafeOffset(SrcIdx) ? SrcIdx : -1;
11924 }
11925 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask);
11926 };
11927
11928 // Found a valid a/zext mask! Try various lowering strategies based on the
11929 // input type and available ISA extensions.
11930 if (Subtarget.hasSSE41()) {
11931 // Not worth offsetting 128-bit vectors if scale == 2, a pattern using
11932 // PUNPCK will catch this in a later shuffle match.
11933 if (Offset && Scale == 2 && VT.is128BitVector())
11934 return SDValue();
11935 MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale),
11936 NumElements / Scale);
11937 InputV = DAG.getBitcast(VT, InputV);
11938 InputV = ShuffleOffset(InputV);
11939 InputV = getEXTEND_VECTOR_INREG(AnyExt ? ISD::ANY_EXTEND : ISD::ZERO_EXTEND,
11940 DL, ExtVT, InputV, DAG);
11941 return DAG.getBitcast(VT, InputV);
11942 }
11943
11944 assert(VT.is128BitVector() && "Only 128-bit vectors can be extended.");
11945 InputV = DAG.getBitcast(VT, InputV);
11946
11947 // For any extends we can cheat for larger element sizes and use shuffle
11948 // instructions that can fold with a load and/or copy.
11949 if (AnyExt && EltBits == 32) {
11950 int PSHUFDMask[4] = {Offset, -1, SafeOffset(Offset + 1) ? Offset + 1 : -1,
11951 -1};
11952 return DAG.getBitcast(
11953 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
11954 DAG.getBitcast(MVT::v4i32, InputV),
11955 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
11956 }
11957 if (AnyExt && EltBits == 16 && Scale > 2) {
11958 int PSHUFDMask[4] = {Offset / 2, -1,
11959 SafeOffset(Offset + 1) ? (Offset + 1) / 2 : -1, -1};
11960 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
11961 DAG.getBitcast(MVT::v4i32, InputV),
11962 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
11963 int PSHUFWMask[4] = {1, -1, -1, -1};
11964 unsigned OddEvenOp = (Offset & 1) ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
11965 return DAG.getBitcast(
11966 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16,
11967 DAG.getBitcast(MVT::v8i16, InputV),
11968 getV4X86ShuffleImm8ForMask(PSHUFWMask, DL, DAG)));
11969 }
11970
11971 // The SSE4A EXTRQ instruction can efficiently extend the first 2 lanes
11972 // to 64-bits.
11973 if ((Scale * EltBits) == 64 && EltBits < 32 && Subtarget.hasSSE4A()) {
11974 assert(NumElements == (int)Mask.size() && "Unexpected shuffle mask size!");
11975 assert(VT.is128BitVector() && "Unexpected vector width!");
11976
11977 int LoIdx = Offset * EltBits;
11978 SDValue Lo = DAG.getBitcast(
11979 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
11980 DAG.getTargetConstant(EltBits, DL, MVT::i8),
11981 DAG.getTargetConstant(LoIdx, DL, MVT::i8)));
11982
11983 if (isUndefUpperHalf(Mask) || !SafeOffset(Offset + 1))
11984 return DAG.getBitcast(VT, Lo);
11985
11986 int HiIdx = (Offset + 1) * EltBits;
11987 SDValue Hi = DAG.getBitcast(
11988 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
11989 DAG.getTargetConstant(EltBits, DL, MVT::i8),
11990 DAG.getTargetConstant(HiIdx, DL, MVT::i8)));
11991 return DAG.getBitcast(VT,
11992 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
11993 }
11994
11995 // If this would require more than 2 unpack instructions to expand, use
11996 // pshufb when available. We can only use more than 2 unpack instructions
11997 // when zero extending i8 elements which also makes it easier to use pshufb.
11998 if (Scale > 4 && EltBits == 8 && Subtarget.hasSSSE3()) {
11999 assert(NumElements == 16 && "Unexpected byte vector width!");
12000 SDValue PSHUFBMask[16];
12001 for (int i = 0; i < 16; ++i) {
12002 int Idx = Offset + (i / Scale);
12003 if ((i % Scale == 0 && SafeOffset(Idx))) {
12004 PSHUFBMask[i] = DAG.getConstant(Idx, DL, MVT::i8);
12005 continue;
12006 }
12007 PSHUFBMask[i] =
12008 AnyExt ? DAG.getUNDEF(MVT::i8) : DAG.getConstant(0x80, DL, MVT::i8);
12009 }
12010 InputV = DAG.getBitcast(MVT::v16i8, InputV);
12011 return DAG.getBitcast(
12012 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
12013 DAG.getBuildVector(MVT::v16i8, DL, PSHUFBMask)));
12014 }
12015
12016 // If we are extending from an offset, ensure we start on a boundary that
12017 // we can unpack from.
12018 int AlignToUnpack = Offset % (NumElements / Scale);
12019 if (AlignToUnpack) {
12020 SmallVector<int, 8> ShMask((unsigned)NumElements, -1);
12021 for (int i = AlignToUnpack; i < NumElements; ++i)
12022 ShMask[i - AlignToUnpack] = i;
12023 InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask);
12024 Offset -= AlignToUnpack;
12025 }
12026
12027 // Otherwise emit a sequence of unpacks.
12028 do {
12029 unsigned UnpackLoHi = X86ISD::UNPCKL;
12030 if (Offset >= (NumElements / 2)) {
12031 UnpackLoHi = X86ISD::UNPCKH;
12032 Offset -= (NumElements / 2);
12033 }
12034
12035 MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements);
12036 SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
12037 : getZeroVector(InputVT, Subtarget, DAG, DL);
12038 InputV = DAG.getBitcast(InputVT, InputV);
12039 InputV = DAG.getNode(UnpackLoHi, DL, InputVT, InputV, Ext);
12040 Scale /= 2;
12041 EltBits *= 2;
12042 NumElements /= 2;
12043 } while (Scale > 1);
12044 return DAG.getBitcast(VT, InputV);
12045}
12046
12047/// Try to lower a vector shuffle as a zero extension on any microarch.
12048///
12049/// This routine will try to do everything in its power to cleverly lower
12050/// a shuffle which happens to match the pattern of a zero extend. It doesn't
12051/// check for the profitability of this lowering, it tries to aggressively
12052/// match this pattern. It will use all of the micro-architectural details it
12053/// can to emit an efficient lowering. It handles both blends with all-zero
12054/// inputs to explicitly zero-extend and undef-lanes (sometimes undef due to
12055/// masking out later).
12056///
12057/// The reason we have dedicated lowering for zext-style shuffles is that they
12058/// are both incredibly common and often quite performance sensitive.
12059static SDValue lowerShuffleAsZeroOrAnyExtend(
12060 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12061 const APInt &Zeroable, const X86Subtarget &Subtarget,
12062 SelectionDAG &DAG) {
12063 int Bits = VT.getSizeInBits();
12064 int NumLanes = Bits / 128;
12065 int NumElements = VT.getVectorNumElements();
12066 int NumEltsPerLane = NumElements / NumLanes;
12067 assert(VT.getScalarSizeInBits() <= 32 &&
12068 "Exceeds 32-bit integer zero extension limit");
12069 assert((int)Mask.size() == NumElements && "Unexpected shuffle mask size");
12070
12071 // Define a helper function to check a particular ext-scale and lower to it if
12072 // valid.
12073 auto Lower = [&](int Scale) -> SDValue {
12074 SDValue InputV;
12075 bool AnyExt = true;
12076 int Offset = 0;
12077 int Matches = 0;
12078 for (int i = 0; i < NumElements; ++i) {
12079 int M = Mask[i];
12080 if (M < 0)
12081 continue; // Valid anywhere but doesn't tell us anything.
12082 if (i % Scale != 0) {
12083 // Each of the extended elements need to be zeroable.
12084 if (!Zeroable[i])
12085 return SDValue();
12086
12087 // We no longer are in the anyext case.
12088 AnyExt = false;
12089 continue;
12090 }
12091
12092 // Each of the base elements needs to be consecutive indices into the
12093 // same input vector.
12094 SDValue V = M < NumElements ? V1 : V2;
12095 M = M % NumElements;
12096 if (!InputV) {
12097 InputV = V;
12098 Offset = M - (i / Scale);
12099 } else if (InputV != V)
12100 return SDValue(); // Flip-flopping inputs.
12101
12102 // Offset must start in the lowest 128-bit lane or at the start of an
12103 // upper lane.
12104 // FIXME: Is it ever worth allowing a negative base offset?
12105 if (!((0 <= Offset && Offset < NumEltsPerLane) ||
12106 (Offset % NumEltsPerLane) == 0))
12107 return SDValue();
12108
12109 // If we are offsetting, all referenced entries must come from the same
12110 // lane.
12111 if (Offset && (Offset / NumEltsPerLane) != (M / NumEltsPerLane))
12112 return SDValue();
12113
12114 if ((M % NumElements) != (Offset + (i / Scale)))
12115 return SDValue(); // Non-consecutive strided elements.
12116 Matches++;
12117 }
12118
12119 // If we fail to find an input, we have a zero-shuffle which should always
12120 // have already been handled.
12121 // FIXME: Maybe handle this here in case during blending we end up with one?
12122 if (!InputV)
12123 return SDValue();
12124
12125 // If we are offsetting, don't extend if we only match a single input, we
12126 // can always do better by using a basic PSHUF or PUNPCK.
12127 if (Offset != 0 && Matches < 2)
12128 return SDValue();
12129
12130 return lowerShuffleAsSpecificZeroOrAnyExtend(DL, VT, Scale, Offset, AnyExt,
12131 InputV, Mask, Subtarget, DAG);
12132 };
12133
12134 // The widest scale possible for extending is to a 64-bit integer.
12135 assert(Bits % 64 == 0 &&
12136 "The number of bits in a vector must be divisible by 64 on x86!");
12137 int NumExtElements = Bits / 64;
12138
12139 // Each iteration, try extending the elements half as much, but into twice as
12140 // many elements.
12141 for (; NumExtElements < NumElements; NumExtElements *= 2) {
12142 assert(NumElements % NumExtElements == 0 &&
12143 "The input vector size must be divisible by the extended size.");
12144 if (SDValue V = Lower(NumElements / NumExtElements))
12145 return V;
12146 }
12147
12148 // General extends failed, but 128-bit vectors may be able to use MOVQ.
12149 if (Bits != 128)
12150 return SDValue();
12151
12152 // Returns one of the source operands if the shuffle can be reduced to a
12153 // MOVQ, copying the lower 64-bits and zero-extending to the upper 64-bits.
12154 auto CanZExtLowHalf = [&]() {
12155 for (int i = NumElements / 2; i != NumElements; ++i)
12156 if (!Zeroable[i])
12157 return SDValue();
12158 if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, 0))
12159 return V1;
12160 if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, NumElements))
12161 return V2;
12162 return SDValue();
12163 };
12164
12165 if (SDValue V = CanZExtLowHalf()) {
12166 V = DAG.getBitcast(MVT::v2i64, V);
12167 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
12168 return DAG.getBitcast(VT, V);
12169 }
12170
12171 // No viable ext lowering found.
12172 return SDValue();
12173}
12174
12175/// Try to get a scalar value for a specific element of a vector.
12176///
12177/// Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
12178static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
12179 SelectionDAG &DAG) {
12180 MVT VT = V.getSimpleValueType();
12181 MVT EltVT = VT.getVectorElementType();
12182 V = peekThroughBitcasts(V);
12183
12184 // If the bitcasts shift the element size, we can't extract an equivalent
12185 // element from it.
12186 MVT NewVT = V.getSimpleValueType();
12187 if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
12188 return SDValue();
12189
12190 if (V.getOpcode() == ISD::BUILD_VECTOR ||
12191 (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR)) {
12192 // Ensure the scalar operand is the same size as the destination.
12193 // FIXME: Add support for scalar truncation where possible.
12194 SDValue S = V.getOperand(Idx);
12195 if (EltVT.getSizeInBits() == S.getSimpleValueType().getSizeInBits())
12196 return DAG.getBitcast(EltVT, S);
12197 }
12198
12199 return SDValue();
12200}
12201
12202/// Helper to test for a load that can be folded with x86 shuffles.
12203///
12204/// This is particularly important because the set of instructions varies
12205/// significantly based on whether the operand is a load or not.
12206static bool isShuffleFoldableLoad(SDValue V) {
12207 return V->hasOneUse() &&
12208 ISD::isNON_EXTLoad(peekThroughOneUseBitcasts(V).getNode());
12209}
12210
12211template<typename T>
12212static bool isSoftF16(T VT, const X86Subtarget &Subtarget) {
12213 T EltVT = VT.getScalarType();
12214 return EltVT == MVT::bf16 || (EltVT == MVT::f16 && !Subtarget.hasFP16());
12215}
12216
12217/// Try to lower insertion of a single element into a zero vector.
12218///
12219/// This is a common pattern that we have especially efficient patterns to lower
12220/// across all subtarget feature sets.
12221static SDValue lowerShuffleAsElementInsertion(
12222 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12223 const APInt &Zeroable, const X86Subtarget &Subtarget,
12224 SelectionDAG &DAG) {
12225 MVT ExtVT = VT;
12226 MVT EltVT = VT.getVectorElementType();
12227 unsigned NumElts = VT.getVectorNumElements();
12228 unsigned EltBits = VT.getScalarSizeInBits();
12229
12230 if (isSoftF16(EltVT, Subtarget))
12231 return SDValue();
12232
12233 int V2Index =
12234 find_if(Mask, [&Mask](int M) { return M >= (int)Mask.size(); }) -
12235 Mask.begin();
12236 bool IsV1Constant = getTargetConstantFromNode(V1) != nullptr;
12237 bool IsV1Zeroable = true;
12238 for (int i = 0, Size = Mask.size(); i < Size; ++i)
12239 if (i != V2Index && !Zeroable[i]) {
12240 IsV1Zeroable = false;
12241 break;
12242 }
12243
12244 // Bail if a non-zero V1 isn't used in place.
12245 if (!IsV1Zeroable) {
12246 SmallVector<int, 8> V1Mask(Mask);
12247 V1Mask[V2Index] = -1;
12248 if (!isNoopShuffleMask(V1Mask))
12249 return SDValue();
12250 }
12251
12252 // Check for a single input from a SCALAR_TO_VECTOR node.
12253 // FIXME: All of this should be canonicalized into INSERT_VECTOR_ELT and
12254 // all the smarts here sunk into that routine. However, the current
12255 // lowering of BUILD_VECTOR makes that nearly impossible until the old
12256 // vector shuffle lowering is dead.
12257 SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
12258 DAG);
12259 if (V2S && DAG.getTargetLoweringInfo().isTypeLegal(V2S.getValueType())) {
12260 // We need to zext the scalar if it is smaller than an i32.
12261 V2S = DAG.getBitcast(EltVT, V2S);
12262 if (EltVT == MVT::i8 || (EltVT == MVT::i16 && !Subtarget.hasFP16())) {
12263 // Using zext to expand a narrow element won't work for non-zero
12264 // insertions. But we can use a masked constant vector if we're
12265 // inserting V2 into the bottom of V1.
12266 if (!IsV1Zeroable && !(IsV1Constant && V2Index == 0))
12267 return SDValue();
12268
12269 // Zero-extend directly to i32.
12270 ExtVT = MVT::getVectorVT(MVT::i32, ExtVT.getSizeInBits() / 32);
12271 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S);
12272
12273 // If we're inserting into a constant, mask off the inserted index
12274 // and OR with the zero-extended scalar.
12275 if (!IsV1Zeroable) {
12276 SmallVector<APInt> Bits(NumElts, APInt::getAllOnes(EltBits));
12277 Bits[V2Index] = APInt::getZero(EltBits);
12278 SDValue BitMask = getConstVector(Bits, VT, DAG, DL);
12279 V1 = DAG.getNode(ISD::AND, DL, VT, V1, BitMask);
12280 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
12281 V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2));
12282 return DAG.getNode(ISD::OR, DL, VT, V1, V2);
12283 }
12284 }
12285 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
12286 } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 ||
12287 EltVT == MVT::i16) {
12288 // Either not inserting from the low element of the input or the input
12289 // element size is too small to use VZEXT_MOVL to clear the high bits.
12290 return SDValue();
12291 }
12292
12293 if (!IsV1Zeroable) {
12294 // If V1 can't be treated as a zero vector we have fewer options to lower
12295 // this. We can't support integer vectors or non-zero targets cheaply.
12296 assert(VT == ExtVT && "Cannot change extended type when non-zeroable!");
12297 if (!VT.isFloatingPoint() || V2Index != 0)
12298 return SDValue();
12299 if (!VT.is128BitVector())
12300 return SDValue();
12301
12302 // Otherwise, use MOVSD, MOVSS or MOVSH.
12303 unsigned MovOpc = 0;
12304 if (EltVT == MVT::f16)
12305 MovOpc = X86ISD::MOVSH;
12306 else if (EltVT == MVT::f32)
12307 MovOpc = X86ISD::MOVSS;
12308 else if (EltVT == MVT::f64)
12309 MovOpc = X86ISD::MOVSD;
12310 else
12311 llvm_unreachable("Unsupported floating point element type to handle!");
12312 return DAG.getNode(MovOpc, DL, ExtVT, V1, V2);
12313 }
12314
12315 // This lowering only works for the low element with floating point vectors.
12316 if (VT.isFloatingPoint() && V2Index != 0)
12317 return SDValue();
12318
12319 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
12320 if (ExtVT != VT)
12321 V2 = DAG.getBitcast(VT, V2);
12322
12323 if (V2Index != 0) {
12324 // If we have 4 or fewer lanes we can cheaply shuffle the element into
12325 // the desired position. Otherwise it is more efficient to do a vector
12326 // shift left. We know that we can do a vector shift left because all
12327 // the inputs are zero.
12328 if (VT.isFloatingPoint() || NumElts <= 4) {
12329 SmallVector<int, 4> V2Shuffle(Mask.size(), 1);
12330 V2Shuffle[V2Index] = 0;
12331 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
12332 } else {
12333 V2 = DAG.getBitcast(MVT::v16i8, V2);
12334 V2 = DAG.getNode(
12335 X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
12336 DAG.getTargetConstant(V2Index * EltBits / 8, DL, MVT::i8));
12337 V2 = DAG.getBitcast(VT, V2);
12338 }
12339 }
12340 return V2;
12341}
12342
12343/// Try to lower broadcast of a single - truncated - integer element,
12344/// coming from a scalar_to_vector/build_vector node \p V0 with larger elements.
12345///
12346/// This assumes we have AVX2.
12347static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
12348 int BroadcastIdx,
12349 const X86Subtarget &Subtarget,
12350 SelectionDAG &DAG) {
12351 assert(Subtarget.hasAVX2() &&
12352 "We can only lower integer broadcasts with AVX2!");
12353
12354 MVT EltVT = VT.getVectorElementType();
12355 MVT V0VT = V0.getSimpleValueType();
12356
12357 assert(VT.isInteger() && "Unexpected non-integer trunc broadcast!");
12358 assert(V0VT.isVector() && "Unexpected non-vector vector-sized value!");
12359
12360 MVT V0EltVT = V0VT.getVectorElementType();
12361 if (!V0EltVT.isInteger())
12362 return SDValue();
12363
12364 const unsigned EltSize = EltVT.getSizeInBits();
12365 const unsigned V0EltSize = V0EltVT.getSizeInBits();
12366
12367 // This is only a truncation if the original element type is larger.
12368 if (V0EltSize <= EltSize)
12369 return SDValue();
12370
12371 assert(((V0EltSize % EltSize) == 0) &&
12372 "Scalar type sizes must all be powers of 2 on x86!");
12373
12374 const unsigned V0Opc = V0.getOpcode();
12375 const unsigned Scale = V0EltSize / EltSize;
12376 const unsigned V0BroadcastIdx = BroadcastIdx / Scale;
12377
12378 if ((V0Opc != ISD::SCALAR_TO_VECTOR || V0BroadcastIdx != 0) &&
12379 V0Opc != ISD::BUILD_VECTOR)
12380 return SDValue();
12381
12382 SDValue Scalar = V0.getOperand(V0BroadcastIdx);
12383
12384 // If we're extracting non-least-significant bits, shift so we can truncate.
12385 // Hopefully, we can fold away the trunc/srl/load into the broadcast.
12386 // Even if we can't (and !isShuffleFoldableLoad(Scalar)), prefer
12387 // vpbroadcast+vmovd+shr to vpshufb(m)+vmovd.
12388 if (const int OffsetIdx = BroadcastIdx % Scale)
12389 Scalar = DAG.getNode(ISD::SRL, DL, Scalar.getValueType(), Scalar,
12390 DAG.getConstant(OffsetIdx * EltSize, DL, MVT::i8));
12391
12392 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
12393 DAG.getNode(ISD::TRUNCATE, DL, EltVT, Scalar));
12394}
12395
12396/// Test whether this can be lowered with a single SHUFPS instruction.
12397///
12398/// This is used to disable more specialized lowerings when the shufps lowering
12399/// will happen to be efficient.
12400static bool isSingleSHUFPSMask(ArrayRef<int> Mask) {
12401 // This routine only handles 128-bit shufps.
12402 assert(Mask.size() == 4 && "Unsupported mask size!");
12403 assert(Mask[0] >= -1 && Mask[0] < 8 && "Out of bound mask element!");
12404 assert(Mask[1] >= -1 && Mask[1] < 8 && "Out of bound mask element!");
12405 assert(Mask[2] >= -1 && Mask[2] < 8 && "Out of bound mask element!");
12406 assert(Mask[3] >= -1 && Mask[3] < 8 && "Out of bound mask element!");
12407
12408 // To lower with a single SHUFPS we need to have the low half and high half
12409 // each requiring a single input.
12410 if (Mask[0] >= 0 && Mask[1] >= 0 && (Mask[0] < 4) != (Mask[1] < 4))
12411 return false;
12412 if (Mask[2] >= 0 && Mask[3] >= 0 && (Mask[2] < 4) != (Mask[3] < 4))
12413 return false;
12414
12415 return true;
12416}
12417
12418/// Test whether the specified input (0 or 1) is in-place blended by the
12419/// given mask.
12420///
12421/// This returns true if the elements from a particular input are already in the
12422/// slot required by the given mask and require no permutation.
12423static bool isShuffleMaskInputInPlace(int Input, ArrayRef<int> Mask) {
12424 assert((Input == 0 || Input == 1) && "Only two inputs to shuffles.");
12425 int Size = Mask.size();
12426 for (int i = 0; i < Size; ++i)
12427 if (Mask[i] >= 0 && Mask[i] / Size == Input && Mask[i] % Size != i)
12428 return false;
12429
12430 return true;
12431}
12432
12433/// If we are extracting two 128-bit halves of a vector and shuffling the
12434/// result, match that to a 256-bit AVX2 vperm* instruction to avoid a
12435/// multi-shuffle lowering.
12436static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
12437 SDValue N1, ArrayRef<int> Mask,
12438 SelectionDAG &DAG) {
12439 MVT VT = N0.getSimpleValueType();
12440 assert((VT.is128BitVector() &&
12441 (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) &&
12442 "VPERM* family of shuffles requires 32-bit or 64-bit elements");
12443
12444 // Check that both sources are extracts of the same source vector.
12445 if (N0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12446 N1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12447 N0.getOperand(0) != N1.getOperand(0) ||
12448 !N0.hasOneUse() || !N1.hasOneUse())
12449 return SDValue();
12450
12451 SDValue WideVec = N0.getOperand(0);
12452 MVT WideVT = WideVec.getSimpleValueType();
12453 if (!WideVT.is256BitVector())
12454 return SDValue();
12455
12456 // Match extracts of each half of the wide source vector. Commute the shuffle
12457 // if the extract of the low half is N1.
12458 unsigned NumElts = VT.getVectorNumElements();
12459 SmallVector<int, 4> NewMask(Mask);
12460 const APInt &ExtIndex0 = N0.getConstantOperandAPInt(1);
12461 const APInt &ExtIndex1 = N1.getConstantOperandAPInt(1);
12462 if (ExtIndex1 == 0 && ExtIndex0 == NumElts)
12463 ShuffleVectorSDNode::commuteMask(NewMask);
12464 else if (ExtIndex0 != 0 || ExtIndex1 != NumElts)
12465 return SDValue();
12466
12467 // Final bailout: if the mask is simple, we are better off using an extract
12468 // and a simple narrow shuffle. Prefer extract+unpack(h/l)ps to vpermps
12469 // because that avoids a constant load from memory.
12470 if (NumElts == 4 &&
12471 (isSingleSHUFPSMask(NewMask) || is128BitUnpackShuffleMask(NewMask, DAG)))
12472 return SDValue();
12473
12474 // Extend the shuffle mask with undef elements.
12475 NewMask.append(NumElts, -1);
12476
12477 // shuf (extract X, 0), (extract X, 4), M --> extract (shuf X, undef, M'), 0
12478 SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
12479 NewMask);
12480 // This is free: ymm -> xmm.
12481 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf,
12482 DAG.getIntPtrConstant(0, DL));
12483}
12484
12485/// Try to lower broadcast of a single element.
12486///
12487/// For convenience, this code also bundles all of the subtarget feature set
12488/// filtering. While a little annoying to re-dispatch on type here, there isn't
12489/// a convenient way to factor it out.
12490static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
12491 SDValue V2, ArrayRef<int> Mask,
12492 const X86Subtarget &Subtarget,
12493 SelectionDAG &DAG) {
12494 MVT EltVT = VT.getVectorElementType();
12495 if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) ||
12496 (Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) ||
12497 (Subtarget.hasAVX2() && (VT.isInteger() || EltVT == MVT::f16))))
12498 return SDValue();
12499
12500 // With MOVDDUP (v2f64) we can broadcast from a register or a load, otherwise
12501 // we can only broadcast from a register with AVX2.
12502 unsigned NumEltBits = VT.getScalarSizeInBits();
12503 unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2())
12504 ? X86ISD::MOVDDUP
12505 : X86ISD::VBROADCAST;
12506 bool BroadcastFromReg = (Opcode == X86ISD::MOVDDUP) || Subtarget.hasAVX2();
12507
12508 // Check that the mask is a broadcast.
12509 int BroadcastIdx = getSplatIndex(Mask);
12510 if (BroadcastIdx < 0)
12511 return SDValue();
12512 assert(BroadcastIdx < (int)Mask.size() && "We only expect to be called with "
12513 "a sorted mask where the broadcast "
12514 "comes from V1.");
12515
12516 // Go up the chain of (vector) values to find a scalar load that we can
12517 // combine with the broadcast.
12518 // TODO: Combine this logic with findEltLoadSrc() used by
12519 // EltsFromConsecutiveLoads().
12520 int BitOffset = BroadcastIdx * NumEltBits;
12521 SDValue V = V1;
12522 for (;;) {
12523 switch (V.getOpcode()) {
12524 case ISD::BITCAST: {
12525 V = V.getOperand(0);
12526 continue;
12527 }
12528 case ISD::CONCAT_VECTORS: {
12529 int OpBitWidth = V.getOperand(0).getValueSizeInBits();
12530 int OpIdx = BitOffset / OpBitWidth;
12531 V = V.getOperand(OpIdx);
12532 BitOffset %= OpBitWidth;
12533 continue;
12534 }
12535 case ISD::EXTRACT_SUBVECTOR: {
12536 // The extraction index adds to the existing offset.
12537 unsigned EltBitWidth = V.getScalarValueSizeInBits();
12538 unsigned Idx = V.getConstantOperandVal(1);
12539 unsigned BeginOffset = Idx * EltBitWidth;
12540 BitOffset += BeginOffset;
12541 V = V.getOperand(0);
12542 continue;
12543 }
12544 case ISD::INSERT_SUBVECTOR: {
12545 SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1);
12546 int EltBitWidth = VOuter.getScalarValueSizeInBits();
12547 int Idx = (int)V.getConstantOperandVal(2);
12548 int NumSubElts = (int)VInner.getSimpleValueType().getVectorNumElements();
12549 int BeginOffset = Idx * EltBitWidth;
12550 int EndOffset = BeginOffset + NumSubElts * EltBitWidth;
12551 if (BeginOffset <= BitOffset && BitOffset < EndOffset) {
12552 BitOffset -= BeginOffset;
12553 V = VInner;
12554 } else {
12555 V = VOuter;
12556 }
12557 continue;
12558 }
12559 }
12560 break;
12561 }
12562 assert((BitOffset % NumEltBits) == 0 && "Illegal bit-offset");
12563 BroadcastIdx = BitOffset / NumEltBits;
12564
12565 // Do we need to bitcast the source to retrieve the original broadcast index?
12566 bool BitCastSrc = V.getScalarValueSizeInBits() != NumEltBits;
12567
12568 // Check if this is a broadcast of a scalar. We special case lowering
12569 // for scalars so that we can more effectively fold with loads.
12570 // If the original value has a larger element type than the shuffle, the
12571 // broadcast element is in essence truncated. Make that explicit to ease
12572 // folding.
12573 if (BitCastSrc && VT.isInteger())
12574 if (SDValue TruncBroadcast = lowerShuffleAsTruncBroadcast(
12575 DL, VT, V, BroadcastIdx, Subtarget, DAG))
12576 return TruncBroadcast;
12577
12578 // Also check the simpler case, where we can directly reuse the scalar.
12579 if (!BitCastSrc &&
12580 ((V.getOpcode() == ISD::BUILD_VECTOR && V.hasOneUse()) ||
12581 (V.getOpcode() == ISD::SCALAR_TO_VECTOR && BroadcastIdx == 0))) {
12582 V = V.getOperand(BroadcastIdx);
12583
12584 // If we can't broadcast from a register, check that the input is a load.
12585 if (!BroadcastFromReg && !isShuffleFoldableLoad(V))
12586 return SDValue();
12587 } else if (ISD::isNormalLoad(V.getNode()) &&
12588 cast<LoadSDNode>(V)->isSimple()) {
12589 // We do not check for one-use of the vector load because a broadcast load
12590 // is expected to be a win for code size, register pressure, and possibly
12591 // uops even if the original vector load is not eliminated.
12592
12593 // Reduce the vector load and shuffle to a broadcasted scalar load.
12594 LoadSDNode *Ld = cast<LoadSDNode>(V);
12595 SDValue BaseAddr = Ld->getOperand(1);
12596 MVT SVT = VT.getScalarType();
12597 unsigned Offset = BroadcastIdx * SVT.getStoreSize();
12598 assert((int)(Offset * 8) == BitOffset && "Unexpected bit-offset");
12599 SDValue NewAddr =
12600 DAG.getMemBasePlusOffset(BaseAddr, TypeSize::getFixed(Offset), DL);
12601
12602 // Directly form VBROADCAST_LOAD if we're using VBROADCAST opcode rather
12603 // than MOVDDUP.
12604 // FIXME: Should we add VBROADCAST_LOAD isel patterns for pre-AVX?
12605 if (Opcode == X86ISD::VBROADCAST) {
12606 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
12607 SDValue Ops[] = {Ld->getChain(), NewAddr};
12608 V = DAG.getMemIntrinsicNode(
12609 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SVT,
12610 DAG.getMachineFunction().getMachineMemOperand(
12611 Ld->getMemOperand(), Offset, SVT.getStoreSize()));
12612 DAG.makeEquivalentMemoryOrdering(Ld, V);
12613 return DAG.getBitcast(VT, V);
12614 }
12615 assert(SVT == MVT::f64 && "Unexpected VT!");
12616 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
12617 DAG.getMachineFunction().getMachineMemOperand(
12618 Ld->getMemOperand(), Offset, SVT.getStoreSize()));
12619 DAG.makeEquivalentMemoryOrdering(Ld, V);
12620 } else if (!BroadcastFromReg) {
12621 // We can't broadcast from a vector register.
12622 return SDValue();
12623 } else if (BitOffset != 0) {
12624 // We can only broadcast from the zero-element of a vector register,
12625 // but it can be advantageous to broadcast from the zero-element of a
12626 // subvector.
12627 if (!VT.is256BitVector() && !VT.is512BitVector())
12628 return SDValue();
12629
12630 // VPERMQ/VPERMPD can perform the cross-lane shuffle directly.
12631 if (VT == MVT::v4f64 || VT == MVT::v4i64)
12632 return SDValue();
12633
12634 // Only broadcast the zero-element of a 128-bit subvector.
12635 if ((BitOffset % 128) != 0)
12636 return SDValue();
12637
12638 assert((BitOffset % V.getScalarValueSizeInBits()) == 0 &&
12639 "Unexpected bit-offset");
12640 assert((V.getValueSizeInBits() == 256 || V.getValueSizeInBits() == 512) &&
12641 "Unexpected vector size");
12642 unsigned ExtractIdx = BitOffset / V.getScalarValueSizeInBits();
12643 V = extract128BitVector(V, ExtractIdx, DAG, DL);
12644 }
12645
12646 // On AVX we can use VBROADCAST directly for scalar sources.
12647 if (Opcode == X86ISD::MOVDDUP && !V.getValueType().isVector()) {
12648 V = DAG.getBitcast(MVT::f64, V);
12649 if (Subtarget.hasAVX()) {
12650 V = DAG.getNode(X86ISD::VBROADCAST, DL, MVT::v2f64, V);
12651 return DAG.getBitcast(VT, V);
12652 }
12653 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V);
12654 }
12655
12656 // If this is a scalar, do the broadcast on this type and bitcast.
12657 if (!V.getValueType().isVector()) {
12658 assert(V.getScalarValueSizeInBits() == NumEltBits &&
12659 "Unexpected scalar size");
12660 MVT BroadcastVT = MVT::getVectorVT(V.getSimpleValueType(),
12661 VT.getVectorNumElements());
12662 return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V));
12663 }
12664
12665 // We only support broadcasting from 128-bit vectors to minimize the
12666 // number of patterns we need to deal with in isel. So extract down to
12667 // 128-bits, removing as many bitcasts as possible.
12668 if (V.getValueSizeInBits() > 128)
12669 V = extract128BitVector(peekThroughBitcasts(V), 0, DAG, DL);
12670
12671 // Otherwise cast V to a vector with the same element type as VT, but
12672 // possibly narrower than VT. Then perform the broadcast.
12673 unsigned NumSrcElts = V.getValueSizeInBits() / NumEltBits;
12674 MVT CastVT = MVT::getVectorVT(VT.getVectorElementType(), NumSrcElts);
12675 return DAG.getNode(Opcode, DL, VT, DAG.getBitcast(CastVT, V));
12676}
12677
12678// Check for whether we can use INSERTPS to perform the shuffle. We only use
12679// INSERTPS when the V1 elements are already in the correct locations
12680// because otherwise we can just always use two SHUFPS instructions which
12681// are much smaller to encode than a SHUFPS and an INSERTPS. We can also
12682// perform INSERTPS if a single V1 element is out of place and all V2
12683// elements are zeroable.
12684static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
12685 unsigned &InsertPSMask,
12686 const APInt &Zeroable,
12687 ArrayRef<int> Mask, SelectionDAG &DAG) {
12688 assert(V1.getSimpleValueType().is128BitVector() && "Bad operand type!");
12689 assert(V2.getSimpleValueType().is128BitVector() && "Bad operand type!");
12690 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
12691
12692 // Attempt to match INSERTPS with one element from VA or VB being
12693 // inserted into VA (or undef). If successful, V1, V2 and InsertPSMask
12694 // are updated.
12695 auto matchAsInsertPS = [&](SDValue VA, SDValue VB,
12696 ArrayRef<int> CandidateMask) {
12697 unsigned ZMask = 0;
12698 int VADstIndex = -1;
12699 int VBDstIndex = -1;
12700 bool VAUsedInPlace = false;
12701
12702 for (int i = 0; i < 4; ++i) {
12703 // Synthesize a zero mask from the zeroable elements (includes undefs).
12704 if (Zeroable[i]) {
12705 ZMask |= 1 << i;
12706 continue;
12707 }
12708
12709 // Flag if we use any VA inputs in place.
12710 if (i == CandidateMask[i]) {
12711 VAUsedInPlace = true;
12712 continue;
12713 }
12714
12715 // We can only insert a single non-zeroable element.
12716 if (VADstIndex >= 0 || VBDstIndex >= 0)
12717 return false;
12718
12719 if (CandidateMask[i] < 4) {
12720 // VA input out of place for insertion.
12721 VADstIndex = i;
12722 } else {
12723 // VB input for insertion.
12724 VBDstIndex = i;
12725 }
12726 }
12727
12728 // Don't bother if we have no (non-zeroable) element for insertion.
12729 if (VADstIndex < 0 && VBDstIndex < 0)
12730 return false;
12731
12732 // Determine element insertion src/dst indices. The src index is from the
12733 // start of the inserted vector, not the start of the concatenated vector.
12734 unsigned VBSrcIndex = 0;
12735 if (VADstIndex >= 0) {
12736 // If we have a VA input out of place, we use VA as the V2 element
12737 // insertion and don't use the original V2 at all.
12738 VBSrcIndex = CandidateMask[VADstIndex];
12739 VBDstIndex = VADstIndex;
12740 VB = VA;
12741 } else {
12742 VBSrcIndex = CandidateMask[VBDstIndex] - 4;
12743 }
12744
12745 // If no V1 inputs are used in place, then the result is created only from
12746 // the zero mask and the V2 insertion - so remove V1 dependency.
12747 if (!VAUsedInPlace)
12748 VA = DAG.getUNDEF(MVT::v4f32);
12749
12750 // Update V1, V2 and InsertPSMask accordingly.
12751 V1 = VA;
12752 V2 = VB;
12753
12754 // Insert the V2 element into the desired position.
12755 InsertPSMask = VBSrcIndex << 6 | VBDstIndex << 4 | ZMask;
12756 assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
12757 return true;
12758 };
12759
12760 if (matchAsInsertPS(V1, V2, Mask))
12761 return true;
12762
12763 // Commute and try again.
12764 SmallVector<int, 4> CommutedMask(Mask);
12765 ShuffleVectorSDNode::commuteMask(CommutedMask);
12766 if (matchAsInsertPS(V2, V1, CommutedMask))
12767 return true;
12768
12769 return false;
12770}
12771
12772static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
12773 ArrayRef<int> Mask, const APInt &Zeroable,
12774 SelectionDAG &DAG) {
12775 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12776 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
12777
12778 // Attempt to match the insertps pattern.
12779 unsigned InsertPSMask = 0;
12780 if (!matchShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
12781 return SDValue();
12782
12783 // Insert the V2 element into the desired position.
12784 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
12785 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
12786}
12787
12788/// Handle lowering of 2-lane 64-bit floating point shuffles.
12789///
12790/// This is the basis function for the 2-lane 64-bit shuffles as we have full
12791/// support for floating point shuffles but not integer shuffles. These
12792/// instructions will incur a domain crossing penalty on some chips though so
12793/// it is better to avoid lowering through this for integer vectors where
12794/// possible.
12795static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12796 const APInt &Zeroable, SDValue V1, SDValue V2,
12797 const X86Subtarget &Subtarget,
12798 SelectionDAG &DAG) {
12799 assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
12800 assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
12801 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
12802
12803 if (V2.isUndef()) {
12804 // Check for being able to broadcast a single element.
12805 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,
12806 Mask, Subtarget, DAG))
12807 return Broadcast;
12808
12809 // Straight shuffle of a single input vector. Simulate this by using the
12810 // single input as both of the "inputs" to this instruction..
12811 unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1);
12812
12813 if (Subtarget.hasAVX()) {
12814 // If we have AVX, we can use VPERMILPS which will allow folding a load
12815 // into the shuffle.
12816 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
12817 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
12818 }
12819
12820 return DAG.getNode(
12821 X86ISD::SHUFP, DL, MVT::v2f64,
12822 Mask[0] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
12823 Mask[1] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
12824 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
12825 }
12826 assert(Mask[0] >= 0 && "No undef lanes in multi-input v2 shuffles!");
12827 assert(Mask[1] >= 0 && "No undef lanes in multi-input v2 shuffles!");
12828 assert(Mask[0] < 2 && "We sort V1 to be the first input.");
12829 assert(Mask[1] >= 2 && "We sort V2 to be the second input.");
12830
12831 if (Subtarget.hasAVX2())
12832 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
12833 return Extract;
12834
12835 // When loading a scalar and then shuffling it into a vector we can often do
12836 // the insertion cheaply.
12837 if (SDValue Insertion = lowerShuffleAsElementInsertion(
12838 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))
12839 return Insertion;
12840 // Try inverting the insertion since for v2 masks it is easy to do and we
12841 // can't reliably sort the mask one way or the other.
12842 int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2),
12843 Mask[1] < 0 ? -1 : (Mask[1] ^ 2)};
12844 if (SDValue Insertion = lowerShuffleAsElementInsertion(
12845 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
12846 return Insertion;
12847
12848 // Try to use one of the special instruction patterns to handle two common
12849 // blend patterns if a zero-blend above didn't work.
12850 if (isShuffleEquivalent(Mask, {0, 3}, V1, V2) ||
12851 isShuffleEquivalent(Mask, {1, 3}, V1, V2))
12852 if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
12853 // We can either use a special instruction to load over the low double or
12854 // to move just the low double.
12855 return DAG.getNode(
12856 X86ISD::MOVSD, DL, MVT::v2f64, V2,
12857 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));
12858
12859 if (Subtarget.hasSSE41())
12860 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
12861 Zeroable, Subtarget, DAG))
12862 return Blend;
12863
12864 // Use dedicated unpack instructions for masks that match their pattern.
12865 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, V1, V2, Mask, DAG))
12866 return V;
12867
12868 unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
12869 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
12870 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
12871}
12872
12873/// Handle lowering of 2-lane 64-bit integer shuffles.
12874///
12875/// Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by
12876/// the integer unit to minimize domain crossing penalties. However, for blends
12877/// it falls back to the floating point shuffle operation with appropriate bit
12878/// casting.
12879static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
12880 const APInt &Zeroable, SDValue V1, SDValue V2,
12881 const X86Subtarget &Subtarget,
12882 SelectionDAG &DAG) {
12883 assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
12884 assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
12885 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
12886
12887 if (V2.isUndef()) {
12888 // Check for being able to broadcast a single element.
12889 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,
12890 Mask, Subtarget, DAG))
12891 return Broadcast;
12892
12893 // Straight shuffle of a single input vector. For everything from SSE2
12894 // onward this has a single fast instruction with no scary immediates.
12895 // We have to map the mask as it is actually a v4i32 shuffle instruction.
12896 V1 = DAG.getBitcast(MVT::v4i32, V1);
12897 int WidenedMask[4] = {Mask[0] < 0 ? -1 : (Mask[0] * 2),
12898 Mask[0] < 0 ? -1 : ((Mask[0] * 2) + 1),
12899 Mask[1] < 0 ? -1 : (Mask[1] * 2),
12900 Mask[1] < 0 ? -1 : ((Mask[1] * 2) + 1)};
12901 return DAG.getBitcast(
12902 MVT::v2i64,
12903 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
12904 getV4X86ShuffleImm8ForMask(WidenedMask, DL, DAG)));
12905 }
12906 assert(Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!");
12907 assert(Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!");
12908 assert(Mask[0] < 2 && "We sort V1 to be the first input.");
12909 assert(Mask[1] >= 2 && "We sort V2 to be the second input.");
12910
12911 if (Subtarget.hasAVX2())
12912 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
12913 return Extract;
12914
12915 // Try to use shift instructions.
12916 if (SDValue Shift =
12917 lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget,
12918 DAG, /*BitwiseOnly*/ false))
12919 return Shift;
12920
12921 // When loading a scalar and then shuffling it into a vector we can often do
12922 // the insertion cheaply.
12923 if (SDValue Insertion = lowerShuffleAsElementInsertion(
12924 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))
12925 return Insertion;
12926 // Try inverting the insertion since for v2 masks it is easy to do and we
12927 // can't reliably sort the mask one way or the other.
12928 int InverseMask[2] = {Mask[0] ^ 2, Mask[1] ^ 2};
12929 if (SDValue Insertion = lowerShuffleAsElementInsertion(
12930 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
12931 return Insertion;
12932
12933 // We have different paths for blend lowering, but they all must use the
12934 // *exact* same predicate.
12935 bool IsBlendSupported = Subtarget.hasSSE41();
12936 if (IsBlendSupported)
12937 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
12938 Zeroable, Subtarget, DAG))
12939 return Blend;
12940
12941 // Use dedicated unpack instructions for masks that match their pattern.
12942 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, V1, V2, Mask, DAG))
12943 return V;
12944
12945 // Try to use byte rotation instructions.
12946 // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
12947 if (Subtarget.hasSSSE3()) {
12948 if (Subtarget.hasVLX())
12949 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v2i64, V1, V2, Mask,
12950 Zeroable, Subtarget, DAG))
12951 return Rotate;
12952
12953 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,
12954 Subtarget, DAG))
12955 return Rotate;
12956 }
12957
12958 // If we have direct support for blends, we should lower by decomposing into
12959 // a permute. That will be faster than the domain cross.
12960 if (IsBlendSupported)
12961 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v2i64, V1, V2, Mask,
12962 Zeroable, Subtarget, DAG);
12963
12964 // We implement this with SHUFPD which is pretty lame because it will likely
12965 // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
12966 // However, all the alternatives are still more cycles and newer chips don't
12967 // have this problem. It would be really nice if x86 had better shuffles here.
12968 V1 = DAG.getBitcast(MVT::v2f64, V1);
12969 V2 = DAG.getBitcast(MVT::v2f64, V2);
12970 return DAG.getBitcast(MVT::v2i64,
12971 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
12972}
12973
12974/// Lower a vector shuffle using the SHUFPS instruction.
12975///
12976/// This is a helper routine dedicated to lowering vector shuffles using SHUFPS.
12977/// It makes no assumptions about whether this is the *best* lowering, it simply
12978/// uses it.
12979static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
12980 ArrayRef<int> Mask, SDValue V1,
12981 SDValue V2, SelectionDAG &DAG) {
12982 SDValue LowV = V1, HighV = V2;
12983 SmallVector<int, 4> NewMask(Mask);
12984 int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
12985
12986 if (NumV2Elements == 1) {
12987 int V2Index = find_if(Mask, [](int M) { return M >= 4; }) - Mask.begin();
12988
12989 // Compute the index adjacent to V2Index and in the same half by toggling
12990 // the low bit.
12991 int V2AdjIndex = V2Index ^ 1;
12992
12993 if (Mask[V2AdjIndex] < 0) {
12994 // Handles all the cases where we have a single V2 element and an undef.
12995 // This will only ever happen in the high lanes because we commute the
12996 // vector otherwise.
12997 if (V2Index < 2)
12998 std::swap(LowV, HighV);
12999 NewMask[V2Index] -= 4;
13000 } else {
13001 // Handle the case where the V2 element ends up adjacent to a V1 element.
13002 // To make this work, blend them together as the first step.
13003 int V1Index = V2AdjIndex;
13004 int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0};
13005 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
13006 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
13007
13008 // Now proceed to reconstruct the final blend as we have the necessary
13009 // high or low half formed.
13010 if (V2Index < 2) {
13011 LowV = V2;
13012 HighV = V1;
13013 } else {
13014 HighV = V2;
13015 }
13016 NewMask[V1Index] = 2; // We put the V1 element in V2[2].
13017 NewMask[V2Index] = 0; // We shifted the V2 element into V2[0].
13018 }
13019 } else if (NumV2Elements == 2) {
13020 if (Mask[0] < 4 && Mask[1] < 4) {
13021 // Handle the easy case where we have V1 in the low lanes and V2 in the
13022 // high lanes.
13023 NewMask[2] -= 4;
13024 NewMask[3] -= 4;
13025 } else if (Mask[2] < 4 && Mask[3] < 4) {
13026 // We also handle the reversed case because this utility may get called
13027 // when we detect a SHUFPS pattern but can't easily commute the shuffle to
13028 // arrange things in the right direction.
13029 NewMask[0] -= 4;
13030 NewMask[1] -= 4;
13031 HighV = V1;
13032 LowV = V2;
13033 } else {
13034 // We have a mixture of V1 and V2 in both low and high lanes. Rather than
13035 // trying to place elements directly, just blend them and set up the final
13036 // shuffle to place them.
13037
13038 // The first two blend mask elements are for V1, the second two are for
13039 // V2.
13040 int BlendMask[4] = {Mask[0] < 4 ? Mask[0] : Mask[1],
13041 Mask[2] < 4 ? Mask[2] : Mask[3],
13042 (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4,
13043 (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4};
13044 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
13045 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
13046
13047 // Now we do a normal shuffle of V1 by giving V1 as both operands to
13048 // a blend.
13049 LowV = HighV = V1;
13050 NewMask[0] = Mask[0] < 4 ? 0 : 2;
13051 NewMask[1] = Mask[0] < 4 ? 2 : 0;
13052 NewMask[2] = Mask[2] < 4 ? 1 : 3;
13053 NewMask[3] = Mask[2] < 4 ? 3 : 1;
13054 }
13055 } else if (NumV2Elements == 3) {
13056 // Ideally canonicalizeShuffleMaskWithCommute should have caught this, but
13057 // we can get here due to other paths (e.g repeated mask matching) that we
13058 // don't want to do another round of lowerVECTOR_SHUFFLE.
13059 ShuffleVectorSDNode::commuteMask(NewMask);
13060 return lowerShuffleWithSHUFPS(DL, VT, NewMask, V2, V1, DAG);
13061 }
13062 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
13063 getV4X86ShuffleImm8ForMask(NewMask, DL, DAG));
13064}
13065
13066/// Lower 4-lane 32-bit floating point shuffles.
13067///
13068/// Uses instructions exclusively from the floating point unit to minimize
13069/// domain crossing penalties, as these are sufficient to implement all v4f32
13070/// shuffles.
13071static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13072 const APInt &Zeroable, SDValue V1, SDValue V2,
13073 const X86Subtarget &Subtarget,
13074 SelectionDAG &DAG) {
13075 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
13076 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
13077 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
13078
13079 if (Subtarget.hasSSE41())
13080 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
13081 Zeroable, Subtarget, DAG))
13082 return Blend;
13083
13084 int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
13085
13086 if (NumV2Elements == 0) {
13087 // Check for being able to broadcast a single element.
13088 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,
13089 Mask, Subtarget, DAG))
13090 return Broadcast;
13091
13092 // Use even/odd duplicate instructions for masks that match their pattern.
13093 if (Subtarget.hasSSE3()) {
13094 if (isShuffleEquivalent(Mask, {0, 0, 2, 2}, V1, V2))
13095 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
13096 if (isShuffleEquivalent(Mask, {1, 1, 3, 3}, V1, V2))
13097 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
13098 }
13099
13100 if (Subtarget.hasAVX()) {
13101 // If we have AVX, we can use VPERMILPS which will allow folding a load
13102 // into the shuffle.
13103 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
13104 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13105 }
13106
13107 // Use MOVLHPS/MOVHLPS to simulate unary shuffles. These are only valid
13108 // in SSE1 because otherwise they are widened to v2f64 and never get here.
13109 if (!Subtarget.hasSSE2()) {
13110 if (isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1, V2))
13111 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1);
13112 if (isShuffleEquivalent(Mask, {2, 3, 2, 3}, V1, V2))
13113 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1);
13114 }
13115
13116 // Otherwise, use a straight shuffle of a single input vector. We pass the
13117 // input vector to both operands to simulate this with a SHUFPS.
13118 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
13119 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13120 }
13121
13122 if (Subtarget.hasSSE2())
13123 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
13124 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) {
13125 ZExt = DAG.getBitcast(MVT::v4f32, ZExt);
13126 return ZExt;
13127 }
13128
13129 if (Subtarget.hasAVX2())
13130 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13131 return Extract;
13132
13133 // There are special ways we can lower some single-element blends. However, we
13134 // have custom ways we can lower more complex single-element blends below that
13135 // we defer to if both this and BLENDPS fail to match, so restrict this to
13136 // when the V2 input is targeting element 0 of the mask -- that is the fast
13137 // case here.
13138 if (NumV2Elements == 1 && Mask[0] >= 4)
13139 if (SDValue V = lowerShuffleAsElementInsertion(
13140 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13141 return V;
13142
13143 if (Subtarget.hasSSE41()) {
13144 // Use INSERTPS if we can complete the shuffle efficiently.
13145 if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13146 return V;
13147
13148 if (!isSingleSHUFPSMask(Mask))
13149 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,
13150 V2, Mask, DAG))
13151 return BlendPerm;
13152 }
13153
13154 // Use low/high mov instructions. These are only valid in SSE1 because
13155 // otherwise they are widened to v2f64 and never get here.
13156 if (!Subtarget.hasSSE2()) {
13157 if (isShuffleEquivalent(Mask, {0, 1, 4, 5}, V1, V2))
13158 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);
13159 if (isShuffleEquivalent(Mask, {2, 3, 6, 7}, V1, V2))
13160 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);
13161 }
13162
13163 // Use dedicated unpack instructions for masks that match their pattern.
13164 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, V1, V2, Mask, DAG))
13165 return V;
13166
13167 // Otherwise fall back to a SHUFPS lowering strategy.
13168 return lowerShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
13169}
13170
13171/// Lower 4-lane i32 vector shuffles.
13172///
13173/// We try to handle these with integer-domain shuffles where we can, but for
13174/// blends we use the floating point domain blend instructions.
13175static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13176 const APInt &Zeroable, SDValue V1, SDValue V2,
13177 const X86Subtarget &Subtarget,
13178 SelectionDAG &DAG) {
13179 assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
13180 assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
13181 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
13182
13183 // Whenever we can lower this as a zext, that instruction is strictly faster
13184 // than any alternative. It also allows us to fold memory operands into the
13185 // shuffle in many cases.
13186 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,
13187 Zeroable, Subtarget, DAG))
13188 return ZExt;
13189
13190 int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
13191
13192 // Try to use shift instructions if fast.
13193 if (Subtarget.preferLowerShuffleAsShift()) {
13194 if (SDValue Shift =
13195 lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, Zeroable,
13196 Subtarget, DAG, /*BitwiseOnly*/ true))
13197 return Shift;
13198 if (NumV2Elements == 0)
13199 if (SDValue Rotate =
13200 lowerShuffleAsBitRotate(DL, MVT::v4i32, V1, Mask, Subtarget, DAG))
13201 return Rotate;
13202 }
13203
13204 if (NumV2Elements == 0) {
13205 // Try to use broadcast unless the mask only has one non-undef element.
13206 if (count_if(Mask, [](int M) { return M >= 0 && M < 4; }) > 1) {
13207 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,
13208 Mask, Subtarget, DAG))
13209 return Broadcast;
13210 }
13211
13212 // Straight shuffle of a single input vector. For everything from SSE2
13213 // onward this has a single fast instruction with no scary immediates.
13214 // We coerce the shuffle pattern to be compatible with UNPCK instructions
13215 // but we aren't actually going to use the UNPCK instruction because doing
13216 // so prevents folding a load into this instruction or making a copy.
13217 const int UnpackLoMask[] = {0, 0, 1, 1};
13218 const int UnpackHiMask[] = {2, 2, 3, 3};
13219 if (isShuffleEquivalent(Mask, {0, 0, 1, 1}, V1, V2))
13220 Mask = UnpackLoMask;
13221 else if (isShuffleEquivalent(Mask, {2, 2, 3, 3}, V1, V2))
13222 Mask = UnpackHiMask;
13223
13224 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
13225 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13226 }
13227
13228 if (Subtarget.hasAVX2())
13229 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13230 return Extract;
13231
13232 // Try to use shift instructions.
13233 if (SDValue Shift =
13234 lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget,
13235 DAG, /*BitwiseOnly*/ false))
13236 return Shift;
13237
13238 // There are special ways we can lower some single-element blends.
13239 if (NumV2Elements == 1)
13240 if (SDValue V = lowerShuffleAsElementInsertion(
13241 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13242 return V;
13243
13244 // We have different paths for blend lowering, but they all must use the
13245 // *exact* same predicate.
13246 bool IsBlendSupported = Subtarget.hasSSE41();
13247 if (IsBlendSupported)
13248 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
13249 Zeroable, Subtarget, DAG))
13250 return Blend;
13251
13252 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
13253 Zeroable, Subtarget, DAG))
13254 return Masked;
13255
13256 // Use dedicated unpack instructions for masks that match their pattern.
13257 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, V1, V2, Mask, DAG))
13258 return V;
13259
13260 // Try to use byte rotation instructions.
13261 // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
13262 if (Subtarget.hasSSSE3()) {
13263 if (Subtarget.hasVLX())
13264 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i32, V1, V2, Mask,
13265 Zeroable, Subtarget, DAG))
13266 return Rotate;
13267
13268 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,
13269 Subtarget, DAG))
13270 return Rotate;
13271 }
13272
13273 // Assume that a single SHUFPS is faster than an alternative sequence of
13274 // multiple instructions (even if the CPU has a domain penalty).
13275 // If some CPU is harmed by the domain switch, we can fix it in a later pass.
13276 if (!isSingleSHUFPSMask(Mask)) {
13277 // If we have direct support for blends, we should lower by decomposing into
13278 // a permute. That will be faster than the domain cross.
13279 if (IsBlendSupported)
13280 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i32, V1, V2, Mask,
13281 Zeroable, Subtarget, DAG);
13282
13283 // Try to lower by permuting the inputs into an unpack instruction.
13284 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
13285 Mask, Subtarget, DAG))
13286 return Unpack;
13287 }
13288
13289 // We implement this with SHUFPS because it can blend from two vectors.
13290 // Because we're going to eventually use SHUFPS, we use SHUFPS even to build
13291 // up the inputs, bypassing domain shift penalties that we would incur if we
13292 // directly used PSHUFD on Nehalem and older. For newer chips, this isn't
13293 // relevant.
13294 SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1);
13295 SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2);
13296 SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);
13297 return DAG.getBitcast(MVT::v4i32, ShufPS);
13298}
13299
13300/// Lowering of single-input v8i16 shuffles is the cornerstone of SSE2
13301/// shuffle lowering, and the most complex part.
13302///
13303/// The lowering strategy is to try to form pairs of input lanes which are
13304/// targeted at the same half of the final vector, and then use a dword shuffle
13305/// to place them onto the right half, and finally unpack the paired lanes into
13306/// their final position.
13307///
13308/// The exact breakdown of how to form these dword pairs and align them on the
13309/// correct sides is really tricky. See the comments within the function for
13310/// more of the details.
13311///
13312/// This code also handles repeated 128-bit lanes of v8i16 shuffles, but each
13313/// lane must shuffle the *exact* same way. In fact, you must pass a v8 Mask to
13314/// this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16
13315/// vector, form the analogous 128-bit 8-element Mask.
13316static SDValue lowerV8I16GeneralSingleInputShuffle(
13317 const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,
13318 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
13319 assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!");
13320 MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);
13321
13322 assert(Mask.size() == 8 && "Shuffle mask length doesn't match!");
13323 MutableArrayRef<int> LoMask = Mask.slice(0, 4);
13324 MutableArrayRef<int> HiMask = Mask.slice(4, 4);
13325
13326 // Attempt to directly match PSHUFLW or PSHUFHW.
13327 if (isUndefOrInRange(LoMask, 0, 4) &&
13328 isSequentialOrUndefInRange(HiMask, 0, 4, 4)) {
13329 return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13330 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
13331 }
13332 if (isUndefOrInRange(HiMask, 4, 8) &&
13333 isSequentialOrUndefInRange(LoMask, 0, 4, 0)) {
13334 for (int i = 0; i != 4; ++i)
13335 HiMask[i] = (HiMask[i] < 0 ? HiMask[i] : (HiMask[i] - 4));
13336 return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13337 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
13338 }
13339
13340 SmallVector<int, 4> LoInputs;
13341 copy_if(LoMask, std::back_inserter(LoInputs), [](int M) { return M >= 0; });
13342 array_pod_sort(LoInputs.begin(), LoInputs.end());
13343 LoInputs.erase(llvm::unique(LoInputs), LoInputs.end());
13344 SmallVector<int, 4> HiInputs;
13345 copy_if(HiMask, std::back_inserter(HiInputs), [](int M) { return M >= 0; });
13346 array_pod_sort(HiInputs.begin(), HiInputs.end());
13347 HiInputs.erase(llvm::unique(HiInputs), HiInputs.end());
13348 int NumLToL = llvm::lower_bound(LoInputs, 4) - LoInputs.begin();
13349 int NumHToL = LoInputs.size() - NumLToL;
13350 int NumLToH = llvm::lower_bound(HiInputs, 4) - HiInputs.begin();
13351 int NumHToH = HiInputs.size() - NumLToH;
13352 MutableArrayRef<int> LToLInputs(LoInputs.data(), NumLToL);
13353 MutableArrayRef<int> LToHInputs(HiInputs.data(), NumLToH);
13354 MutableArrayRef<int> HToLInputs(LoInputs.data() + NumLToL, NumHToL);
13355 MutableArrayRef<int> HToHInputs(HiInputs.data() + NumLToH, NumHToH);
13356
13357 // If we are shuffling values from one half - check how many different DWORD
13358 // pairs we need to create. If only 1 or 2 then we can perform this as a
13359 // PSHUFLW/PSHUFHW + PSHUFD instead of the PSHUFD+PSHUFLW+PSHUFHW chain below.
13360 auto ShuffleDWordPairs = [&](ArrayRef<int> PSHUFHalfMask,
13361 ArrayRef<int> PSHUFDMask, unsigned ShufWOp) {
13362 V = DAG.getNode(ShufWOp, DL, VT, V,
13363 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13364 V = DAG.getBitcast(PSHUFDVT, V);
13365 V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V,
13366 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
13367 return DAG.getBitcast(VT, V);
13368 };
13369
13370 if ((NumHToL + NumHToH) == 0 || (NumLToL + NumLToH) == 0) {
13371 int PSHUFDMask[4] = { -1, -1, -1, -1 };
13372 SmallVector<std::pair<int, int>, 4> DWordPairs;
13373 int DOffset = ((NumHToL + NumHToH) == 0 ? 0 : 2);
13374
13375 // Collect the different DWORD pairs.
13376 for (int DWord = 0; DWord != 4; ++DWord) {
13377 int M0 = Mask[2 * DWord + 0];
13378 int M1 = Mask[2 * DWord + 1];
13379 M0 = (M0 >= 0 ? M0 % 4 : M0);
13380 M1 = (M1 >= 0 ? M1 % 4 : M1);
13381 if (M0 < 0 && M1 < 0)
13382 continue;
13383
13384 bool Match = false;
13385 for (int j = 0, e = DWordPairs.size(); j < e; ++j) {
13386 auto &DWordPair = DWordPairs[j];
13387 if ((M0 < 0 || isUndefOrEqual(DWordPair.first, M0)) &&
13388 (M1 < 0 || isUndefOrEqual(DWordPair.second, M1))) {
13389 DWordPair.first = (M0 >= 0 ? M0 : DWordPair.first);
13390 DWordPair.second = (M1 >= 0 ? M1 : DWordPair.second);
13391 PSHUFDMask[DWord] = DOffset + j;
13392 Match = true;
13393 break;
13394 }
13395 }
13396 if (!Match) {
13397 PSHUFDMask[DWord] = DOffset + DWordPairs.size();
13398 DWordPairs.push_back(std::make_pair(M0, M1));
13399 }
13400 }
13401
13402 if (DWordPairs.size() <= 2) {
13403 DWordPairs.resize(2, std::make_pair(-1, -1));
13404 int PSHUFHalfMask[4] = {DWordPairs[0].first, DWordPairs[0].second,
13405 DWordPairs[1].first, DWordPairs[1].second};
13406 if ((NumHToL + NumHToH) == 0)
13407 return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFLW);
13408 if ((NumLToL + NumLToH) == 0)
13409 return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFHW);
13410 }
13411 }
13412
13413 // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all
13414 // such inputs we can swap two of the dwords across the half mark and end up
13415 // with <=2 inputs to each half in each half. Once there, we can fall through
13416 // to the generic code below. For example:
13417 //
13418 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
13419 // Mask: [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5]
13420 //
13421 // However in some very rare cases we have a 1-into-3 or 3-into-1 on one half
13422 // and an existing 2-into-2 on the other half. In this case we may have to
13423 // pre-shuffle the 2-into-2 half to avoid turning it into a 3-into-1 or
13424 // 1-into-3 which could cause us to cycle endlessly fixing each side in turn.
13425 // Fortunately, we don't have to handle anything but a 2-into-2 pattern
13426 // because any other situation (including a 3-into-1 or 1-into-3 in the other
13427 // half than the one we target for fixing) will be fixed when we re-enter this
13428 // path. We will also combine away any sequence of PSHUFD instructions that
13429 // result into a single instruction. Here is an example of the tricky case:
13430 //
13431 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
13432 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -THIS-IS-BAD!!!!-> [5, 7, 1, 0, 4, 7, 5, 3]
13433 //
13434 // This now has a 1-into-3 in the high half! Instead, we do two shuffles:
13435 //
13436 // Input: [a, b, c, d, e, f, g, h] PSHUFHW[0,2,1,3]-> [a, b, c, d, e, g, f, h]
13437 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -----------------> [3, 7, 1, 0, 2, 7, 3, 6]
13438 //
13439 // Input: [a, b, c, d, e, g, f, h] -PSHUFD[0,2,1,3]-> [a, b, e, g, c, d, f, h]
13440 // Mask: [3, 7, 1, 0, 2, 7, 3, 6] -----------------> [5, 7, 1, 0, 4, 7, 5, 6]
13441 //
13442 // The result is fine to be handled by the generic logic.
13443 auto balanceSides = [&](ArrayRef<int> AToAInputs, ArrayRef<int> BToAInputs,
13444 ArrayRef<int> BToBInputs, ArrayRef<int> AToBInputs,
13445 int AOffset, int BOffset) {
13446 assert((AToAInputs.size() == 3 || AToAInputs.size() == 1) &&
13447 "Must call this with A having 3 or 1 inputs from the A half.");
13448 assert((BToAInputs.size() == 1 || BToAInputs.size() == 3) &&
13449 "Must call this with B having 1 or 3 inputs from the B half.");
13450 assert(AToAInputs.size() + BToAInputs.size() == 4 &&
13451 "Must call this with either 3:1 or 1:3 inputs (summing to 4).");
13452
13453 bool ThreeAInputs = AToAInputs.size() == 3;
13454
13455 // Compute the index of dword with only one word among the three inputs in
13456 // a half by taking the sum of the half with three inputs and subtracting
13457 // the sum of the actual three inputs. The difference is the remaining
13458 // slot.
13459 int ADWord = 0, BDWord = 0;
13460 int &TripleDWord = ThreeAInputs ? ADWord : BDWord;
13461 int &OneInputDWord = ThreeAInputs ? BDWord : ADWord;
13462 int TripleInputOffset = ThreeAInputs ? AOffset : BOffset;
13463 ArrayRef<int> TripleInputs = ThreeAInputs ? AToAInputs : BToAInputs;
13464 int OneInput = ThreeAInputs ? BToAInputs[0] : AToAInputs[0];
13465 int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset);
13466 int TripleNonInputIdx =
13467 TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
13468 TripleDWord = TripleNonInputIdx / 2;
13469
13470 // We use xor with one to compute the adjacent DWord to whichever one the
13471 // OneInput is in.
13472 OneInputDWord = (OneInput / 2) ^ 1;
13473
13474 // Check for one tricky case: We're fixing a 3<-1 or a 1<-3 shuffle for AToA
13475 // and BToA inputs. If there is also such a problem with the BToB and AToB
13476 // inputs, we don't try to fix it necessarily -- we'll recurse and see it in
13477 // the next pass. However, if we have a 2<-2 in the BToB and AToB inputs, it
13478 // is essential that we don't *create* a 3<-1 as then we might oscillate.
13479 if (BToBInputs.size() == 2 && AToBInputs.size() == 2) {
13480 // Compute how many inputs will be flipped by swapping these DWords. We
13481 // need
13482 // to balance this to ensure we don't form a 3-1 shuffle in the other
13483 // half.
13484 int NumFlippedAToBInputs = llvm::count(AToBInputs, 2 * ADWord) +
13485 llvm::count(AToBInputs, 2 * ADWord + 1);
13486 int NumFlippedBToBInputs = llvm::count(BToBInputs, 2 * BDWord) +
13487 llvm::count(BToBInputs, 2 * BDWord + 1);
13488 if ((NumFlippedAToBInputs == 1 &&
13489 (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) ||
13490 (NumFlippedBToBInputs == 1 &&
13491 (NumFlippedAToBInputs == 0 || NumFlippedAToBInputs == 2))) {
13492 // We choose whether to fix the A half or B half based on whether that
13493 // half has zero flipped inputs. At zero, we may not be able to fix it
13494 // with that half. We also bias towards fixing the B half because that
13495 // will more commonly be the high half, and we have to bias one way.
13496 auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord,
13497 ArrayRef<int> Inputs) {
13498 int FixIdx = PinnedIdx ^ 1; // The adjacent slot to the pinned slot.
13499 bool IsFixIdxInput = is_contained(Inputs, PinnedIdx ^ 1);
13500 // Determine whether the free index is in the flipped dword or the
13501 // unflipped dword based on where the pinned index is. We use this bit
13502 // in an xor to conditionally select the adjacent dword.
13503 int FixFreeIdx = 2 * (DWord ^ (PinnedIdx / 2 == DWord));
13504 bool IsFixFreeIdxInput = is_contained(Inputs, FixFreeIdx);
13505 if (IsFixIdxInput == IsFixFreeIdxInput)
13506 FixFreeIdx += 1;
13507 IsFixFreeIdxInput = is_contained(Inputs, FixFreeIdx);
13508 assert(IsFixIdxInput != IsFixFreeIdxInput &&
13509 "We need to be changing the number of flipped inputs!");
13510 int PSHUFHalfMask[] = {0, 1, 2, 3};
13511 std::swap(PSHUFHalfMask[FixFreeIdx % 4], PSHUFHalfMask[FixIdx % 4]);
13512 V = DAG.getNode(
13513 FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
13514 MVT::getVectorVT(MVT::i16, V.getValueSizeInBits() / 16), V,
13515 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13516
13517 for (int &M : Mask)
13518 if (M >= 0 && M == FixIdx)
13519 M = FixFreeIdx;
13520 else if (M >= 0 && M == FixFreeIdx)
13521 M = FixIdx;
13522 };
13523 if (NumFlippedBToBInputs != 0) {
13524 int BPinnedIdx =
13525 BToAInputs.size() == 3 ? TripleNonInputIdx : OneInput;
13526 FixFlippedInputs(BPinnedIdx, BDWord, BToBInputs);
13527 } else {
13528 assert(NumFlippedAToBInputs != 0 && "Impossible given predicates!");
13529 int APinnedIdx = ThreeAInputs ? TripleNonInputIdx : OneInput;
13530 FixFlippedInputs(APinnedIdx, ADWord, AToBInputs);
13531 }
13532 }
13533 }
13534
13535 int PSHUFDMask[] = {0, 1, 2, 3};
13536 PSHUFDMask[ADWord] = BDWord;
13537 PSHUFDMask[BDWord] = ADWord;
13538 V = DAG.getBitcast(
13539 VT,
13540 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
13541 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
13542
13543 // Adjust the mask to match the new locations of A and B.
13544 for (int &M : Mask)
13545 if (M >= 0 && M/2 == ADWord)
13546 M = 2 * BDWord + M % 2;
13547 else if (M >= 0 && M/2 == BDWord)
13548 M = 2 * ADWord + M % 2;
13549
13550 // Recurse back into this routine to re-compute state now that this isn't
13551 // a 3 and 1 problem.
13552 return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG);
13553 };
13554 if ((NumLToL == 3 && NumHToL == 1) || (NumLToL == 1 && NumHToL == 3))
13555 return balanceSides(LToLInputs, HToLInputs, HToHInputs, LToHInputs, 0, 4);
13556 if ((NumHToH == 3 && NumLToH == 1) || (NumHToH == 1 && NumLToH == 3))
13557 return balanceSides(HToHInputs, LToHInputs, LToLInputs, HToLInputs, 4, 0);
13558
13559 // At this point there are at most two inputs to the low and high halves from
13560 // each half. That means the inputs can always be grouped into dwords and
13561 // those dwords can then be moved to the correct half with a dword shuffle.
13562 // We use at most one low and one high word shuffle to collect these paired
13563 // inputs into dwords, and finally a dword shuffle to place them.
13564 int PSHUFLMask[4] = {-1, -1, -1, -1};
13565 int PSHUFHMask[4] = {-1, -1, -1, -1};
13566 int PSHUFDMask[4] = {-1, -1, -1, -1};
13567
13568 // First fix the masks for all the inputs that are staying in their
13569 // original halves. This will then dictate the targets of the cross-half
13570 // shuffles.
13571 auto fixInPlaceInputs =
13572 [&PSHUFDMask](ArrayRef<int> InPlaceInputs, ArrayRef<int> IncomingInputs,
13573 MutableArrayRef<int> SourceHalfMask,
13574 MutableArrayRef<int> HalfMask, int HalfOffset) {
13575 if (InPlaceInputs.empty())
13576 return;
13577 if (InPlaceInputs.size() == 1) {
13578 SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
13579 InPlaceInputs[0] - HalfOffset;
13580 PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
13581 return;
13582 }
13583 if (IncomingInputs.empty()) {
13584 // Just fix all of the in place inputs.
13585 for (int Input : InPlaceInputs) {
13586 SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
13587 PSHUFDMask[Input / 2] = Input / 2;
13588 }
13589 return;
13590 }
13591
13592 assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
13593 SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
13594 InPlaceInputs[0] - HalfOffset;
13595 // Put the second input next to the first so that they are packed into
13596 // a dword. We find the adjacent index by toggling the low bit.
13597 int AdjIndex = InPlaceInputs[0] ^ 1;
13598 SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
13599 std::replace(HalfMask.begin(), HalfMask.end(), InPlaceInputs[1], AdjIndex);
13600 PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
13601 };
13602 fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0);
13603 fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4);
13604
13605 // Now gather the cross-half inputs and place them into a free dword of
13606 // their target half.
13607 // FIXME: This operation could almost certainly be simplified dramatically to
13608 // look more like the 3-1 fixing operation.
13609 auto moveInputsToRightHalf = [&PSHUFDMask](
13610 MutableArrayRef<int> IncomingInputs, ArrayRef<int> ExistingInputs,
13611 MutableArrayRef<int> SourceHalfMask, MutableArrayRef<int> HalfMask,
13612 MutableArrayRef<int> FinalSourceHalfMask, int SourceOffset,
13613 int DestOffset) {
13614 auto isWordClobbered = [](ArrayRef<int> SourceHalfMask, int Word) {
13615 return SourceHalfMask[Word] >= 0 && SourceHalfMask[Word] != Word;
13616 };
13617 auto isDWordClobbered = [&isWordClobbered](ArrayRef<int> SourceHalfMask,
13618 int Word) {
13619 int LowWord = Word & ~1;
13620 int HighWord = Word | 1;
13621 return isWordClobbered(SourceHalfMask, LowWord) ||
13622 isWordClobbered(SourceHalfMask, HighWord);
13623 };
13624
13625 if (IncomingInputs.empty())
13626 return;
13627
13628 if (ExistingInputs.empty()) {
13629 // Map any dwords with inputs from them into the right half.
13630 for (int Input : IncomingInputs) {
13631 // If the source half mask maps over the inputs, turn those into
13632 // swaps and use the swapped lane.
13633 if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) {
13634 if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] < 0) {
13635 SourceHalfMask[SourceHalfMask[Input - SourceOffset]] =
13636 Input - SourceOffset;
13637 // We have to swap the uses in our half mask in one sweep.
13638 for (int &M : HalfMask)
13639 if (M == SourceHalfMask[Input - SourceOffset] + SourceOffset)
13640 M = Input;
13641 else if (M == Input)
13642 M = SourceHalfMask[Input - SourceOffset] + SourceOffset;
13643 } else {
13644 assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] ==
13645 Input - SourceOffset &&
13646 "Previous placement doesn't match!");
13647 }
13648 // Note that this correctly re-maps both when we do a swap and when
13649 // we observe the other side of the swap above. We rely on that to
13650 // avoid swapping the members of the input list directly.
13651 Input = SourceHalfMask[Input - SourceOffset] + SourceOffset;
13652 }
13653
13654 // Map the input's dword into the correct half.
13655 if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] < 0)
13656 PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2;
13657 else
13658 assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] ==
13659 Input / 2 &&
13660 "Previous placement doesn't match!");
13661 }
13662
13663 // And just directly shift any other-half mask elements to be same-half
13664 // as we will have mirrored the dword containing the element into the
13665 // same position within that half.
13666 for (int &M : HalfMask)
13667 if (M >= SourceOffset && M < SourceOffset + 4) {
13668 M = M - SourceOffset + DestOffset;
13669 assert(M >= 0 && "This should never wrap below zero!");
13670 }
13671 return;
13672 }
13673
13674 // Ensure we have the input in a viable dword of its current half. This
13675 // is particularly tricky because the original position may be clobbered
13676 // by inputs being moved and *staying* in that half.
13677 if (IncomingInputs.size() == 1) {
13678 if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
13679 int InputFixed = find(SourceHalfMask, -1) - std::begin(SourceHalfMask) +
13680 SourceOffset;
13681 SourceHalfMask[InputFixed - SourceOffset] =
13682 IncomingInputs[0] - SourceOffset;
13683 std::replace(HalfMask.begin(), HalfMask.end(), IncomingInputs[0],
13684 InputFixed);
13685 IncomingInputs[0] = InputFixed;
13686 }
13687 } else if (IncomingInputs.size() == 2) {
13688 if (IncomingInputs[0] / 2 != IncomingInputs[1] / 2 ||
13689 isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
13690 // We have two non-adjacent or clobbered inputs we need to extract from
13691 // the source half. To do this, we need to map them into some adjacent
13692 // dword slot in the source mask.
13693 int InputsFixed[2] = {IncomingInputs[0] - SourceOffset,
13694 IncomingInputs[1] - SourceOffset};
13695
13696 // If there is a free slot in the source half mask adjacent to one of
13697 // the inputs, place the other input in it. We use (Index XOR 1) to
13698 // compute an adjacent index.
13699 if (!isWordClobbered(SourceHalfMask, InputsFixed[0]) &&
13700 SourceHalfMask[InputsFixed[0] ^ 1] < 0) {
13701 SourceHalfMask[InputsFixed[0]] = InputsFixed[0];
13702 SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
13703 InputsFixed[1] = InputsFixed[0] ^ 1;
13704 } else if (!isWordClobbered(SourceHalfMask, InputsFixed[1]) &&
13705 SourceHalfMask[InputsFixed[1] ^ 1] < 0) {
13706 SourceHalfMask[InputsFixed[1]] = InputsFixed[1];
13707 SourceHalfMask[InputsFixed[1] ^ 1] = InputsFixed[0];
13708 InputsFixed[0] = InputsFixed[1] ^ 1;
13709 } else if (SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] < 0 &&
13710 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] < 0) {
13711 // The two inputs are in the same DWord but it is clobbered and the
13712 // adjacent DWord isn't used at all. Move both inputs to the free
13713 // slot.
13714 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] = InputsFixed[0];
13715 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] = InputsFixed[1];
13716 InputsFixed[0] = 2 * ((InputsFixed[0] / 2) ^ 1);
13717 InputsFixed[1] = 2 * ((InputsFixed[0] / 2) ^ 1) + 1;
13718 } else {
13719 // The only way we hit this point is if there is no clobbering
13720 // (because there are no off-half inputs to this half) and there is no
13721 // free slot adjacent to one of the inputs. In this case, we have to
13722 // swap an input with a non-input.
13723 for (int i = 0; i < 4; ++i)
13724 assert((SourceHalfMask[i] < 0 || SourceHalfMask[i] == i) &&
13725 "We can't handle any clobbers here!");
13726 assert(InputsFixed[1] != (InputsFixed[0] ^ 1) &&
13727 "Cannot have adjacent inputs here!");
13728
13729 SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
13730 SourceHalfMask[InputsFixed[1]] = InputsFixed[0] ^ 1;
13731
13732 // We also have to update the final source mask in this case because
13733 // it may need to undo the above swap.
13734 for (int &M : FinalSourceHalfMask)
13735 if (M == (InputsFixed[0] ^ 1) + SourceOffset)
13736 M = InputsFixed[1] + SourceOffset;
13737 else if (M == InputsFixed[1] + SourceOffset)
13738 M = (InputsFixed[0] ^ 1) + SourceOffset;
13739
13740 InputsFixed[1] = InputsFixed[0] ^ 1;
13741 }
13742
13743 // Point everything at the fixed inputs.
13744 for (int &M : HalfMask)
13745 if (M == IncomingInputs[0])
13746 M = InputsFixed[0] + SourceOffset;
13747 else if (M == IncomingInputs[1])
13748 M = InputsFixed[1] + SourceOffset;
13749
13750 IncomingInputs[0] = InputsFixed[0] + SourceOffset;
13751 IncomingInputs[1] = InputsFixed[1] + SourceOffset;
13752 }
13753 } else {
13754 llvm_unreachable("Unhandled input size!");
13755 }
13756
13757 // Now hoist the DWord down to the right half.
13758 int FreeDWord = (PSHUFDMask[DestOffset / 2] < 0 ? 0 : 1) + DestOffset / 2;
13759 assert(PSHUFDMask[FreeDWord] < 0 && "DWord not free");
13760 PSHUFDMask[FreeDWord] = IncomingInputs[0] / 2;
13761 for (int &M : HalfMask)
13762 for (int Input : IncomingInputs)
13763 if (M == Input)
13764 M = FreeDWord * 2 + Input % 2;
13765 };
13766 moveInputsToRightHalf(HToLInputs, LToLInputs, PSHUFHMask, LoMask, HiMask,
13767 /*SourceOffset*/ 4, /*DestOffset*/ 0);
13768 moveInputsToRightHalf(LToHInputs, HToHInputs, PSHUFLMask, HiMask, LoMask,
13769 /*SourceOffset*/ 0, /*DestOffset*/ 4);
13770
13771 // Now enact all the shuffles we've computed to move the inputs into their
13772 // target half.
13773 if (!isNoopShuffleMask(PSHUFLMask))
13774 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13775 getV4X86ShuffleImm8ForMask(PSHUFLMask, DL, DAG));
13776 if (!isNoopShuffleMask(PSHUFHMask))
13777 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13778 getV4X86ShuffleImm8ForMask(PSHUFHMask, DL, DAG));
13779 if (!isNoopShuffleMask(PSHUFDMask))
13780 V = DAG.getBitcast(
13781 VT,
13782 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
13783 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
13784
13785 // At this point, each half should contain all its inputs, and we can then
13786 // just shuffle them into their final position.
13787 assert(count_if(LoMask, [](int M) { return M >= 4; }) == 0 &&
13788 "Failed to lift all the high half inputs to the low mask!");
13789 assert(count_if(HiMask, [](int M) { return M >= 0 && M < 4; }) == 0 &&
13790 "Failed to lift all the low half inputs to the high mask!");
13791
13792 // Do a half shuffle for the low mask.
13793 if (!isNoopShuffleMask(LoMask))
13794 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13795 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
13796
13797 // Do a half shuffle with the high mask after shifting its values down.
13798 for (int &M : HiMask)
13799 if (M >= 0)
13800 M -= 4;
13801 if (!isNoopShuffleMask(HiMask))
13802 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13803 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
13804
13805 return V;
13806}
13807
13808/// Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the
13809/// blend if only one input is used.
13810static SDValue lowerShuffleAsBlendOfPSHUFBs(
13811 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
13812 const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) {
13813 assert(!is128BitLaneCrossingShuffleMask(VT, Mask) &&
13814 "Lane crossing shuffle masks not supported");
13815
13816 int NumBytes = VT.getSizeInBits() / 8;
13817 int Size = Mask.size();
13818 int Scale = NumBytes / Size;
13819
13820 SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8));
13821 SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8));
13822 V1InUse = false;
13823 V2InUse = false;
13824
13825 for (int i = 0; i < NumBytes; ++i) {
13826 int M = Mask[i / Scale];
13827 if (M < 0)
13828 continue;
13829
13830 const int ZeroMask = 0x80;
13831 int V1Idx = M < Size ? M * Scale + i % Scale : ZeroMask;
13832 int V2Idx = M < Size ? ZeroMask : (M - Size) * Scale + i % Scale;
13833 if (Zeroable[i / Scale])
13834 V1Idx = V2Idx = ZeroMask;
13835
13836 V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8);
13837 V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8);
13838 V1InUse |= (ZeroMask != V1Idx);
13839 V2InUse |= (ZeroMask != V2Idx);
13840 }
13841
13842 MVT ShufVT = MVT::getVectorVT(MVT::i8, NumBytes);
13843 if (V1InUse)
13844 V1 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V1),
13845 DAG.getBuildVector(ShufVT, DL, V1Mask));
13846 if (V2InUse)
13847 V2 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V2),
13848 DAG.getBuildVector(ShufVT, DL, V2Mask));
13849
13850 // If we need shuffled inputs from both, blend the two.
13851 SDValue V;
13852 if (V1InUse && V2InUse)
13853 V = DAG.getNode(ISD::OR, DL, ShufVT, V1, V2);
13854 else
13855 V = V1InUse ? V1 : V2;
13856
13857 // Cast the result back to the correct type.
13858 return DAG.getBitcast(VT, V);
13859}
13860
13861/// Generic lowering of 8-lane i16 shuffles.
13862///
13863/// This handles both single-input shuffles and combined shuffle/blends with
13864/// two inputs. The single input shuffles are immediately delegated to
13865/// a dedicated lowering routine.
13866///
13867/// The blends are lowered in one of three fundamental ways. If there are few
13868/// enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle
13869/// of the input is significantly cheaper when lowered as an interleaving of
13870/// the two inputs, try to interleave them. Otherwise, blend the low and high
13871/// halves of the inputs separately (making them have relatively few inputs)
13872/// and then concatenate them.
13873static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13874 const APInt &Zeroable, SDValue V1, SDValue V2,
13875 const X86Subtarget &Subtarget,
13876 SelectionDAG &DAG) {
13877 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
13878 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
13879 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
13880
13881 // Whenever we can lower this as a zext, that instruction is strictly faster
13882 // than any alternative.
13883 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,
13884 Zeroable, Subtarget, DAG))
13885 return ZExt;
13886
13887 // Try to use lower using a truncation.
13888 if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
13889 Subtarget, DAG))
13890 return V;
13891
13892 int NumV2Inputs = count_if(Mask, [](int M) { return M >= 8; });
13893
13894 if (NumV2Inputs == 0) {
13895 // Try to use shift instructions.
13896 if (SDValue Shift =
13897 lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask, Zeroable,
13898 Subtarget, DAG, /*BitwiseOnly*/ false))
13899 return Shift;
13900
13901 // Check for being able to broadcast a single element.
13902 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,
13903 Mask, Subtarget, DAG))
13904 return Broadcast;
13905
13906 // Try to use bit rotation instructions.
13907 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask,
13908 Subtarget, DAG))
13909 return Rotate;
13910
13911 // Use dedicated unpack instructions for masks that match their pattern.
13912 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
13913 return V;
13914
13915 // Use dedicated pack instructions for masks that match their pattern.
13916 if (SDValue V =
13917 lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
13918 return V;
13919
13920 // Try to use byte rotation instructions.
13921 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,
13922 Subtarget, DAG))
13923 return Rotate;
13924
13925 // Make a copy of the mask so it can be modified.
13926 SmallVector<int, 8> MutableMask(Mask);
13927 return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v8i16, V1, MutableMask,
13928 Subtarget, DAG);
13929 }
13930
13931 assert(llvm::any_of(Mask, [](int M) { return M >= 0 && M < 8; }) &&
13932 "All single-input shuffles should be canonicalized to be V1-input "
13933 "shuffles.");
13934
13935 // Try to use shift instructions.
13936 if (SDValue Shift =
13937 lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget,
13938 DAG, /*BitwiseOnly*/ false))
13939 return Shift;
13940
13941 // See if we can use SSE4A Extraction / Insertion.
13942 if (Subtarget.hasSSE4A())
13943 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
13944 Zeroable, DAG))
13945 return V;
13946
13947 // There are special ways we can lower some single-element blends.
13948 if (NumV2Inputs == 1)
13949 if (SDValue V = lowerShuffleAsElementInsertion(
13950 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
13951 return V;
13952
13953 // We have different paths for blend lowering, but they all must use the
13954 // *exact* same predicate.
13955 bool IsBlendSupported = Subtarget.hasSSE41();
13956 if (IsBlendSupported)
13957 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
13958 Zeroable, Subtarget, DAG))
13959 return Blend;
13960
13961 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
13962 Zeroable, Subtarget, DAG))
13963 return Masked;
13964
13965 // Use dedicated unpack instructions for masks that match their pattern.
13966 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
13967 return V;
13968
13969 // Use dedicated pack instructions for masks that match their pattern.
13970 if (SDValue V =
13971 lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
13972 return V;
13973
13974 // Try to use lower using a truncation.
13975 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v8i16, V1, V2, Mask, Zeroable,
13976 Subtarget, DAG))
13977 return V;
13978
13979 // Try to use byte rotation instructions.
13980 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,
13981 Subtarget, DAG))
13982 return Rotate;
13983
13984 if (SDValue BitBlend =
13985 lowerShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))
13986 return BitBlend;
13987
13988 // Try to use byte shift instructions to mask.
13989 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v8i16, V1, V2, Mask,
13990 Zeroable, Subtarget, DAG))
13991 return V;
13992
13993 // Attempt to lower using compaction, SSE41 is necessary for PACKUSDW.
13994 int NumEvenDrops = canLowerByDroppingElements(Mask, true, false);
13995 if ((NumEvenDrops == 1 || (NumEvenDrops == 2 && Subtarget.hasSSE41())) &&
13996 !Subtarget.hasVLX()) {
13997 // Check if this is part of a 256-bit vector truncation.
13998 unsigned PackOpc = 0;
13999 if (NumEvenDrops == 2 && Subtarget.hasAVX2() &&
14000 peekThroughBitcasts(V1).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
14001 peekThroughBitcasts(V2).getOpcode() == ISD::EXTRACT_SUBVECTOR) {
14002 SDValue V1V2 = concatSubVectors(V1, V2, DAG, DL);
14003 V1V2 = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1V2,
14004 getZeroVector(MVT::v16i16, Subtarget, DAG, DL),
14005 DAG.getTargetConstant(0xEE, DL, MVT::i8));
14006 V1V2 = DAG.getBitcast(MVT::v8i32, V1V2);
14007 V1 = extract128BitVector(V1V2, 0, DAG, DL);
14008 V2 = extract128BitVector(V1V2, 4, DAG, DL);
14009 PackOpc = X86ISD::PACKUS;
14010 } else if (Subtarget.hasSSE41()) {
14011 SmallVector<SDValue, 4> DWordClearOps(4,
14012 DAG.getConstant(0, DL, MVT::i32));
14013 for (unsigned i = 0; i != 4; i += 1 << (NumEvenDrops - 1))
14014 DWordClearOps[i] = DAG.getConstant(0xFFFF, DL, MVT::i32);
14015 SDValue DWordClearMask =
14016 DAG.getBuildVector(MVT::v4i32, DL, DWordClearOps);
14017 V1 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V1),
14018 DWordClearMask);
14019 V2 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V2),
14020 DWordClearMask);
14021 PackOpc = X86ISD::PACKUS;
14022 } else if (!Subtarget.hasSSSE3()) {
14023 SDValue ShAmt = DAG.getTargetConstant(16, DL, MVT::i8);
14024 V1 = DAG.getBitcast(MVT::v4i32, V1);
14025 V2 = DAG.getBitcast(MVT::v4i32, V2);
14026 V1 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V1, ShAmt);
14027 V2 = DAG.getNode(X86ISD::VSHLI, DL, MVT::v4i32, V2, ShAmt);
14028 V1 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V1, ShAmt);
14029 V2 = DAG.getNode(X86ISD::VSRAI, DL, MVT::v4i32, V2, ShAmt);
14030 PackOpc = X86ISD::PACKSS;
14031 }
14032 if (PackOpc) {
14033 // Now pack things back together.
14034 SDValue Result = DAG.getNode(PackOpc, DL, MVT::v8i16, V1, V2);
14035 if (NumEvenDrops == 2) {
14036 Result = DAG.getBitcast(MVT::v4i32, Result);
14037 Result = DAG.getNode(PackOpc, DL, MVT::v8i16, Result, Result);
14038 }
14039 return Result;
14040 }
14041 }
14042
14043 // When compacting odd (upper) elements, use PACKSS pre-SSE41.
14044 int NumOddDrops = canLowerByDroppingElements(Mask, false, false);
14045 if (NumOddDrops == 1) {
14046 bool HasSSE41 = Subtarget.hasSSE41();
14047 V1 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,
14048 DAG.getBitcast(MVT::v4i32, V1),
14049 DAG.getTargetConstant(16, DL, MVT::i8));
14050 V2 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,
14051 DAG.getBitcast(MVT::v4i32, V2),
14052 DAG.getTargetConstant(16, DL, MVT::i8));
14053 return DAG.getNode(HasSSE41 ? X86ISD::PACKUS : X86ISD::PACKSS, DL,
14054 MVT::v8i16, V1, V2);
14055 }
14056
14057 // Try to lower by permuting the inputs into an unpack instruction.
14058 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,
14059 Mask, Subtarget, DAG))
14060 return Unpack;
14061
14062 // If we can't directly blend but can use PSHUFB, that will be better as it
14063 // can both shuffle and set up the inefficient blend.
14064 if (!IsBlendSupported && Subtarget.hasSSSE3()) {
14065 bool V1InUse, V2InUse;
14066 return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,
14067 Zeroable, DAG, V1InUse, V2InUse);
14068 }
14069
14070 // We can always bit-blend if we have to so the fallback strategy is to
14071 // decompose into single-input permutes and blends/unpacks.
14072 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i16, V1, V2, Mask,
14073 Zeroable, Subtarget, DAG);
14074}
14075
14076/// Lower 8-lane 16-bit floating point shuffles.
14077static SDValue lowerV8F16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14078 const APInt &Zeroable, SDValue V1, SDValue V2,
14079 const X86Subtarget &Subtarget,
14080 SelectionDAG &DAG) {
14081 assert(V1.getSimpleValueType() == MVT::v8f16 && "Bad operand type!");
14082 assert(V2.getSimpleValueType() == MVT::v8f16 && "Bad operand type!");
14083 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
14084 int NumV2Elements = count_if(Mask, [](int M) { return M >= 8; });
14085
14086 if (Subtarget.hasFP16()) {
14087 if (NumV2Elements == 0) {
14088 // Check for being able to broadcast a single element.
14089 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f16, V1, V2,
14090 Mask, Subtarget, DAG))
14091 return Broadcast;
14092 }
14093 if (NumV2Elements == 1 && Mask[0] >= 8)
14094 if (SDValue V = lowerShuffleAsElementInsertion(
14095 DL, MVT::v8f16, V1, V2, Mask, Zeroable, Subtarget, DAG))
14096 return V;
14097 }
14098
14099 V1 = DAG.getBitcast(MVT::v8i16, V1);
14100 V2 = DAG.getBitcast(MVT::v8i16, V2);
14101 return DAG.getBitcast(MVT::v8f16,
14102 DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask));
14103}
14104
14105// Lowers unary/binary shuffle as VPERMV/VPERMV3, for non-VLX targets,
14106// sub-512-bit shuffles are padded to 512-bits for the shuffle and then
14107// the active subvector is extracted.
14108static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
14109 ArrayRef<int> Mask, SDValue V1, SDValue V2,
14110 const X86Subtarget &Subtarget,
14111 SelectionDAG &DAG) {
14112 MVT MaskVT = VT.changeTypeToInteger();
14113 SDValue MaskNode;
14114 MVT ShuffleVT = VT;
14115 if (!VT.is512BitVector() && !Subtarget.hasVLX()) {
14116 V1 = widenSubVector(V1, false, Subtarget, DAG, DL, 512);
14117 V2 = widenSubVector(V2, false, Subtarget, DAG, DL, 512);
14118 ShuffleVT = V1.getSimpleValueType();
14119
14120 // Adjust mask to correct indices for the second input.
14121 int NumElts = VT.getVectorNumElements();
14122 unsigned Scale = 512 / VT.getSizeInBits();
14123 SmallVector<int, 32> AdjustedMask(Mask);
14124 for (int &M : AdjustedMask)
14125 if (NumElts <= M)
14126 M += (Scale - 1) * NumElts;
14127 MaskNode = getConstVector(AdjustedMask, MaskVT, DAG, DL, true);
14128 MaskNode = widenSubVector(MaskNode, false, Subtarget, DAG, DL, 512);
14129 } else {
14130 MaskNode = getConstVector(Mask, MaskVT, DAG, DL, true);
14131 }
14132
14133 SDValue Result;
14134 if (V2.isUndef())
14135 Result = DAG.getNode(X86ISD::VPERMV, DL, ShuffleVT, MaskNode, V1);
14136 else
14137 Result = DAG.getNode(X86ISD::VPERMV3, DL, ShuffleVT, V1, MaskNode, V2);
14138
14139 if (VT != ShuffleVT)
14140 Result = extractSubVector(Result, 0, DAG, DL, VT.getSizeInBits());
14141
14142 return Result;
14143}
14144
14145/// Generic lowering of v16i8 shuffles.
14146///
14147/// This is a hybrid strategy to lower v16i8 vectors. It first attempts to
14148/// detect any complexity reducing interleaving. If that doesn't help, it uses
14149/// UNPCK to spread the i8 elements across two i16-element vectors, and uses
14150/// the existing lowering for v8i16 blends on each half, finally PACK-ing them
14151/// back together.
14152static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14153 const APInt &Zeroable, SDValue V1, SDValue V2,
14154 const X86Subtarget &Subtarget,
14155 SelectionDAG &DAG) {
14156 assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
14157 assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
14158 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
14159
14160 // Try to use shift instructions.
14161 if (SDValue Shift =
14162 lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget,
14163 DAG, /*BitwiseOnly*/ false))
14164 return Shift;
14165
14166 // Try to use byte rotation instructions.
14167 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,
14168 Subtarget, DAG))
14169 return Rotate;
14170
14171 // Use dedicated pack instructions for masks that match their pattern.
14172 if (SDValue V =
14173 lowerShuffleWithPACK(DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14174 return V;
14175
14176 // Try to use a zext lowering.
14177 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,
14178 Zeroable, Subtarget, DAG))
14179 return ZExt;
14180
14181 // Try to use lower using a truncation.
14182 if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14183 Subtarget, DAG))
14184 return V;
14185
14186 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i8, V1, V2, Mask, Zeroable,
14187 Subtarget, DAG))
14188 return V;
14189
14190 // See if we can use SSE4A Extraction / Insertion.
14191 if (Subtarget.hasSSE4A())
14192 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
14193 Zeroable, DAG))
14194 return V;
14195
14196 int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
14197
14198 // For single-input shuffles, there are some nicer lowering tricks we can use.
14199 if (NumV2Elements == 0) {
14200 // Check for being able to broadcast a single element.
14201 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,
14202 Mask, Subtarget, DAG))
14203 return Broadcast;
14204
14205 // Try to use bit rotation instructions.
14206 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask,
14207 Subtarget, DAG))
14208 return Rotate;
14209
14210 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
14211 return V;
14212
14213 // Check whether we can widen this to an i16 shuffle by duplicating bytes.
14214 // Notably, this handles splat and partial-splat shuffles more efficiently.
14215 // However, it only makes sense if the pre-duplication shuffle simplifies
14216 // things significantly. Currently, this means we need to be able to
14217 // express the pre-duplication shuffle as an i16 shuffle.
14218 //
14219 // FIXME: We should check for other patterns which can be widened into an
14220 // i16 shuffle as well.
14221 auto canWidenViaDuplication = [](ArrayRef<int> Mask) {
14222 for (int i = 0; i < 16; i += 2)
14223 if (Mask[i] >= 0 && Mask[i + 1] >= 0 && Mask[i] != Mask[i + 1])
14224 return false;
14225
14226 return true;
14227 };
14228 auto tryToWidenViaDuplication = [&]() -> SDValue {
14229 if (!canWidenViaDuplication(Mask))
14230 return SDValue();
14231 SmallVector<int, 4> LoInputs;
14232 copy_if(Mask, std::back_inserter(LoInputs),
14233 [](int M) { return M >= 0 && M < 8; });
14234 array_pod_sort(LoInputs.begin(), LoInputs.end());
14235 LoInputs.erase(llvm::unique(LoInputs), LoInputs.end());
14236 SmallVector<int, 4> HiInputs;
14237 copy_if(Mask, std::back_inserter(HiInputs), [](int M) { return M >= 8; });
14238 array_pod_sort(HiInputs.begin(), HiInputs.end());
14239 HiInputs.erase(llvm::unique(HiInputs), HiInputs.end());
14240
14241 bool TargetLo = LoInputs.size() >= HiInputs.size();
14242 ArrayRef<int> InPlaceInputs = TargetLo ? LoInputs : HiInputs;
14243 ArrayRef<int> MovingInputs = TargetLo ? HiInputs : LoInputs;
14244
14245 int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1};
14246 SmallDenseMap<int, int, 8> LaneMap;
14247 for (int I : InPlaceInputs) {
14248 PreDupI16Shuffle[I/2] = I/2;
14249 LaneMap[I] = I;
14250 }
14251 int j = TargetLo ? 0 : 4, je = j + 4;
14252 for (int i = 0, ie = MovingInputs.size(); i < ie; ++i) {
14253 // Check if j is already a shuffle of this input. This happens when
14254 // there are two adjacent bytes after we move the low one.
14255 if (PreDupI16Shuffle[j] != MovingInputs[i] / 2) {
14256 // If we haven't yet mapped the input, search for a slot into which
14257 // we can map it.
14258 while (j < je && PreDupI16Shuffle[j] >= 0)
14259 ++j;
14260
14261 if (j == je)
14262 // We can't place the inputs into a single half with a simple i16 shuffle, so bail.
14263 return SDValue();
14264
14265 // Map this input with the i16 shuffle.
14266 PreDupI16Shuffle[j] = MovingInputs[i] / 2;
14267 }
14268
14269 // Update the lane map based on the mapping we ended up with.
14270 LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2;
14271 }
14272 V1 = DAG.getBitcast(
14273 MVT::v16i8,
14274 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14275 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));
14276
14277 // Unpack the bytes to form the i16s that will be shuffled into place.
14278 bool EvenInUse = false, OddInUse = false;
14279 for (int i = 0; i < 16; i += 2) {
14280 EvenInUse |= (Mask[i + 0] >= 0);
14281 OddInUse |= (Mask[i + 1] >= 0);
14282 if (EvenInUse && OddInUse)
14283 break;
14284 }
14285 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
14286 MVT::v16i8, EvenInUse ? V1 : DAG.getUNDEF(MVT::v16i8),
14287 OddInUse ? V1 : DAG.getUNDEF(MVT::v16i8));
14288
14289 int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
14290 for (int i = 0; i < 16; ++i)
14291 if (Mask[i] >= 0) {
14292 int MappedMask = LaneMap[Mask[i]] - (TargetLo ? 0 : 8);
14293 assert(MappedMask < 8 && "Invalid v8 shuffle mask!");
14294 if (PostDupI16Shuffle[i / 2] < 0)
14295 PostDupI16Shuffle[i / 2] = MappedMask;
14296 else
14297 assert(PostDupI16Shuffle[i / 2] == MappedMask &&
14298 "Conflicting entries in the original shuffle!");
14299 }
14300 return DAG.getBitcast(
14301 MVT::v16i8,
14302 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14303 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));
14304 };
14305 if (SDValue V = tryToWidenViaDuplication())
14306 return V;
14307 }
14308
14309 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
14310 Zeroable, Subtarget, DAG))
14311 return Masked;
14312
14313 // Use dedicated unpack instructions for masks that match their pattern.
14314 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
14315 return V;
14316
14317 // Try to use byte shift instructions to mask.
14318 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v16i8, V1, V2, Mask,
14319 Zeroable, Subtarget, DAG))
14320 return V;
14321
14322 // Check for compaction patterns.
14323 bool IsSingleInput = V2.isUndef();
14324 int NumEvenDrops = canLowerByDroppingElements(Mask, true, IsSingleInput);
14325
14326 // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly
14327 // with PSHUFB. It is important to do this before we attempt to generate any
14328 // blends but after all of the single-input lowerings. If the single input
14329 // lowerings can find an instruction sequence that is faster than a PSHUFB, we
14330 // want to preserve that and we can DAG combine any longer sequences into
14331 // a PSHUFB in the end. But once we start blending from multiple inputs,
14332 // the complexity of DAG combining bad patterns back into PSHUFB is too high,
14333 // and there are *very* few patterns that would actually be faster than the
14334 // PSHUFB approach because of its ability to zero lanes.
14335 //
14336 // If the mask is a binary compaction, we can more efficiently perform this
14337 // as a PACKUS(AND(),AND()) - which is quicker than UNPACK(PSHUFB(),PSHUFB()).
14338 //
14339 // FIXME: The only exceptions to the above are blends which are exact
14340 // interleavings with direct instructions supporting them. We currently don't
14341 // handle those well here.
14342 if (Subtarget.hasSSSE3() && (IsSingleInput || NumEvenDrops != 1)) {
14343 bool V1InUse = false;
14344 bool V2InUse = false;
14345
14346 SDValue PSHUFB = lowerShuffleAsBlendOfPSHUFBs(
14347 DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);
14348
14349 // If both V1 and V2 are in use and we can use a direct blend or an unpack,
14350 // do so. This avoids using them to handle blends-with-zero which is
14351 // important as a single pshufb is significantly faster for that.
14352 if (V1InUse && V2InUse) {
14353 if (Subtarget.hasSSE41())
14354 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
14355 Zeroable, Subtarget, DAG))
14356 return Blend;
14357
14358 // We can use an unpack to do the blending rather than an or in some
14359 // cases. Even though the or may be (very minorly) more efficient, we
14360 // preference this lowering because there are common cases where part of
14361 // the complexity of the shuffles goes away when we do the final blend as
14362 // an unpack.
14363 // FIXME: It might be worth trying to detect if the unpack-feeding
14364 // shuffles will both be pshufb, in which case we shouldn't bother with
14365 // this.
14366 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(
14367 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14368 return Unpack;
14369
14370 // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8).
14371 if (Subtarget.hasVBMI())
14372 return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, Subtarget,
14373 DAG);
14374
14375 // If we have XOP we can use one VPPERM instead of multiple PSHUFBs.
14376 if (Subtarget.hasXOP()) {
14377 SDValue MaskNode = getConstVector(Mask, MVT::v16i8, DAG, DL, true);
14378 return DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, V1, V2, MaskNode);
14379 }
14380
14381 // Use PALIGNR+Permute if possible - permute might become PSHUFB but the
14382 // PALIGNR will be cheaper than the second PSHUFB+OR.
14383 if (SDValue V = lowerShuffleAsByteRotateAndPermute(
14384 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14385 return V;
14386 }
14387
14388 return PSHUFB;
14389 }
14390
14391 // There are special ways we can lower some single-element blends.
14392 if (NumV2Elements == 1)
14393 if (SDValue V = lowerShuffleAsElementInsertion(
14394 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
14395 return V;
14396
14397 if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
14398 return Blend;
14399
14400 // Check whether a compaction lowering can be done. This handles shuffles
14401 // which take every Nth element for some even N. See the helper function for
14402 // details.
14403 //
14404 // We special case these as they can be particularly efficiently handled with
14405 // the PACKUSB instruction on x86 and they show up in common patterns of
14406 // rearranging bytes to truncate wide elements.
14407 if (NumEvenDrops) {
14408 // NumEvenDrops is the power of two stride of the elements. Another way of
14409 // thinking about it is that we need to drop the even elements this many
14410 // times to get the original input.
14411
14412 // First we need to zero all the dropped bytes.
14413 assert(NumEvenDrops <= 3 &&
14414 "No support for dropping even elements more than 3 times.");
14415 SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16));
14416 for (unsigned i = 0; i != 8; i += 1 << (NumEvenDrops - 1))
14417 WordClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i16);
14418 SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps);
14419 V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V1),
14420 WordClearMask);
14421 if (!IsSingleInput)
14422 V2 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V2),
14423 WordClearMask);
14424
14425 // Now pack things back together.
14426 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
14427 IsSingleInput ? V1 : V2);
14428 for (int i = 1; i < NumEvenDrops; ++i) {
14429 Result = DAG.getBitcast(MVT::v8i16, Result);
14430 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
14431 }
14432 return Result;
14433 }
14434
14435 int NumOddDrops = canLowerByDroppingElements(Mask, false, IsSingleInput);
14436 if (NumOddDrops == 1) {
14437 V1 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,
14438 DAG.getBitcast(MVT::v8i16, V1),
14439 DAG.getTargetConstant(8, DL, MVT::i8));
14440 if (!IsSingleInput)
14441 V2 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,
14442 DAG.getBitcast(MVT::v8i16, V2),
14443 DAG.getTargetConstant(8, DL, MVT::i8));
14444 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,
14445 IsSingleInput ? V1 : V2);
14446 }
14447
14448 // Handle multi-input cases by blending/unpacking single-input shuffles.
14449 if (NumV2Elements > 0)
14450 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v16i8, V1, V2, Mask,
14451 Zeroable, Subtarget, DAG);
14452
14453 // The fallback path for single-input shuffles widens this into two v8i16
14454 // vectors with unpacks, shuffles those, and then pulls them back together
14455 // with a pack.
14456 SDValue V = V1;
14457
14458 std::array<int, 8> LoBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}};
14459 std::array<int, 8> HiBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}};
14460 for (int i = 0; i < 16; ++i)
14461 if (Mask[i] >= 0)
14462 (i < 8 ? LoBlendMask[i] : HiBlendMask[i % 8]) = Mask[i];
14463
14464 SDValue VLoHalf, VHiHalf;
14465 // Check if any of the odd lanes in the v16i8 are used. If not, we can mask
14466 // them out and avoid using UNPCK{L,H} to extract the elements of V as
14467 // i16s.
14468 if (none_of(LoBlendMask, [](int M) { return M >= 0 && M % 2 == 1; }) &&
14469 none_of(HiBlendMask, [](int M) { return M >= 0 && M % 2 == 1; })) {
14470 // Use a mask to drop the high bytes.
14471 VLoHalf = DAG.getBitcast(MVT::v8i16, V);
14472 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf,
14473 DAG.getConstant(0x00FF, DL, MVT::v8i16));
14474
14475 // This will be a single vector shuffle instead of a blend so nuke VHiHalf.
14476 VHiHalf = DAG.getUNDEF(MVT::v8i16);
14477
14478 // Squash the masks to point directly into VLoHalf.
14479 for (int &M : LoBlendMask)
14480 if (M >= 0)
14481 M /= 2;
14482 for (int &M : HiBlendMask)
14483 if (M >= 0)
14484 M /= 2;
14485 } else {
14486 // Otherwise just unpack the low half of V into VLoHalf and the high half into
14487 // VHiHalf so that we can blend them as i16s.
14488 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
14489
14490 VLoHalf = DAG.getBitcast(
14491 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
14492 VHiHalf = DAG.getBitcast(
14493 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
14494 }
14495
14496 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
14497 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
14498
14499 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
14500}
14501
14502/// Dispatching routine to lower various 128-bit x86 vector shuffles.
14503///
14504/// This routine breaks down the specific type of 128-bit shuffle and
14505/// dispatches to the lowering routines accordingly.
14506static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
14507 MVT VT, SDValue V1, SDValue V2,
14508 const APInt &Zeroable,
14509 const X86Subtarget &Subtarget,
14510 SelectionDAG &DAG) {
14511 if (VT == MVT::v8bf16) {
14512 V1 = DAG.getBitcast(MVT::v8i16, V1);
14513 V2 = DAG.getBitcast(MVT::v8i16, V2);
14514 return DAG.getBitcast(VT,
14515 DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask));
14516 }
14517
14518 switch (VT.SimpleTy) {
14519 case MVT::v2i64:
14520 return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14521 case MVT::v2f64:
14522 return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14523 case MVT::v4i32:
14524 return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14525 case MVT::v4f32:
14526 return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14527 case MVT::v8i16:
14528 return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14529 case MVT::v8f16:
14530 return lowerV8F16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14531 case MVT::v16i8:
14532 return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14533
14534 default:
14535 llvm_unreachable("Unimplemented!");
14536 }
14537}
14538
14539/// Generic routine to split vector shuffle into half-sized shuffles.
14540///
14541/// This routine just extracts two subvectors, shuffles them independently, and
14542/// then concatenates them back together. This should work effectively with all
14543/// AVX vector shuffle types.
14544static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
14545 SDValue V2, ArrayRef<int> Mask,
14546 SelectionDAG &DAG, bool SimpleOnly) {
14547 assert(VT.getSizeInBits() >= 256 &&
14548 "Only for 256-bit or wider vector shuffles!");
14549 assert(V1.getSimpleValueType() == VT && "Bad operand type!");
14550 assert(V2.getSimpleValueType() == VT && "Bad operand type!");
14551
14552 ArrayRef<int> LoMask = Mask.slice(0, Mask.size() / 2);
14553 ArrayRef<int> HiMask = Mask.slice(Mask.size() / 2);
14554
14555 int NumElements = VT.getVectorNumElements();
14556 int SplitNumElements = NumElements / 2;
14557 MVT ScalarVT = VT.getVectorElementType();
14558 MVT SplitVT = MVT::getVectorVT(ScalarVT, SplitNumElements);
14559
14560 // Use splitVector/extractSubVector so that split build-vectors just build two
14561 // narrower build vectors. This helps shuffling with splats and zeros.
14562 auto SplitVector = [&](SDValue V) {
14563 SDValue LoV, HiV;
14564 std::tie(LoV, HiV) = splitVector(peekThroughBitcasts(V), DAG, DL);
14565 return std::make_pair(DAG.getBitcast(SplitVT, LoV),
14566 DAG.getBitcast(SplitVT, HiV));
14567 };
14568
14569 SDValue LoV1, HiV1, LoV2, HiV2;
14570 std::tie(LoV1, HiV1) = SplitVector(V1);
14571 std::tie(LoV2, HiV2) = SplitVector(V2);
14572
14573 // Now create two 4-way blends of these half-width vectors.
14574 auto GetHalfBlendPiecesReq = [&](const ArrayRef<int> &HalfMask, bool &UseLoV1,
14575 bool &UseHiV1, bool &UseLoV2,
14576 bool &UseHiV2) {
14577 UseLoV1 = UseHiV1 = UseLoV2 = UseHiV2 = false;
14578 for (int i = 0; i < SplitNumElements; ++i) {
14579 int M = HalfMask[i];
14580 if (M >= NumElements) {
14581 if (M >= NumElements + SplitNumElements)
14582 UseHiV2 = true;
14583 else
14584 UseLoV2 = true;
14585 } else if (M >= 0) {
14586 if (M >= SplitNumElements)
14587 UseHiV1 = true;
14588 else
14589 UseLoV1 = true;
14590 }
14591 }
14592 };
14593
14594 auto CheckHalfBlendUsable = [&](const ArrayRef<int> &HalfMask) -> bool {
14595 if (!SimpleOnly)
14596 return true;
14597
14598 bool UseLoV1, UseHiV1, UseLoV2, UseHiV2;
14599 GetHalfBlendPiecesReq(HalfMask, UseLoV1, UseHiV1, UseLoV2, UseHiV2);
14600
14601 return !(UseHiV1 || UseHiV2);
14602 };
14603
14604 auto HalfBlend = [&](ArrayRef<int> HalfMask) {
14605 SmallVector<int, 32> V1BlendMask((unsigned)SplitNumElements, -1);
14606 SmallVector<int, 32> V2BlendMask((unsigned)SplitNumElements, -1);
14607 SmallVector<int, 32> BlendMask((unsigned)SplitNumElements, -1);
14608 for (int i = 0; i < SplitNumElements; ++i) {
14609 int M = HalfMask[i];
14610 if (M >= NumElements) {
14611 V2BlendMask[i] = M - NumElements;
14612 BlendMask[i] = SplitNumElements + i;
14613 } else if (M >= 0) {
14614 V1BlendMask[i] = M;
14615 BlendMask[i] = i;
14616 }
14617 }
14618
14619 bool UseLoV1, UseHiV1, UseLoV2, UseHiV2;
14620 GetHalfBlendPiecesReq(HalfMask, UseLoV1, UseHiV1, UseLoV2, UseHiV2);
14621
14622 // Because the lowering happens after all combining takes place, we need to
14623 // manually combine these blend masks as much as possible so that we create
14624 // a minimal number of high-level vector shuffle nodes.
14625 assert((!SimpleOnly || (!UseHiV1 && !UseHiV2)) && "Shuffle isn't simple");
14626
14627 // First try just blending the halves of V1 or V2.
14628 if (!UseLoV1 && !UseHiV1 && !UseLoV2 && !UseHiV2)
14629 return DAG.getUNDEF(SplitVT);
14630 if (!UseLoV2 && !UseHiV2)
14631 return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14632 if (!UseLoV1 && !UseHiV1)
14633 return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14634
14635 SDValue V1Blend, V2Blend;
14636 if (UseLoV1 && UseHiV1) {
14637 V1Blend = DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14638 } else {
14639 // We only use half of V1 so map the usage down into the final blend mask.
14640 V1Blend = UseLoV1 ? LoV1 : HiV1;
14641 for (int i = 0; i < SplitNumElements; ++i)
14642 if (BlendMask[i] >= 0 && BlendMask[i] < SplitNumElements)
14643 BlendMask[i] = V1BlendMask[i] - (UseLoV1 ? 0 : SplitNumElements);
14644 }
14645 if (UseLoV2 && UseHiV2) {
14646 V2Blend = DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14647 } else {
14648 // We only use half of V2 so map the usage down into the final blend mask.
14649 V2Blend = UseLoV2 ? LoV2 : HiV2;
14650 for (int i = 0; i < SplitNumElements; ++i)
14651 if (BlendMask[i] >= SplitNumElements)
14652 BlendMask[i] = V2BlendMask[i] + (UseLoV2 ? SplitNumElements : 0);
14653 }
14654 return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask);
14655 };
14656
14657 if (!CheckHalfBlendUsable(LoMask) || !CheckHalfBlendUsable(HiMask))
14658 return SDValue();
14659
14660 SDValue Lo = HalfBlend(LoMask);
14661 SDValue Hi = HalfBlend(HiMask);
14662 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
14663}
14664
14665/// Either split a vector in halves or decompose the shuffles and the
14666/// blend/unpack.
14667///
14668/// This is provided as a good fallback for many lowerings of non-single-input
14669/// shuffles with more than one 128-bit lane. In those cases, we want to select
14670/// between splitting the shuffle into 128-bit components and stitching those
14671/// back together vs. extracting the single-input shuffles and blending those
14672/// results.
14673static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
14674 SDValue V2, ArrayRef<int> Mask,
14675 const APInt &Zeroable,
14676 const X86Subtarget &Subtarget,
14677 SelectionDAG &DAG) {
14678 assert(!V2.isUndef() && "This routine must not be used to lower single-input "
14679 "shuffles as it could then recurse on itself.");
14680 int Size = Mask.size();
14681
14682 // If this can be modeled as a broadcast of two elements followed by a blend,
14683 // prefer that lowering. This is especially important because broadcasts can
14684 // often fold with memory operands.
14685 auto DoBothBroadcast = [&] {
14686 int V1BroadcastIdx = -1, V2BroadcastIdx = -1;
14687 for (int M : Mask)
14688 if (M >= Size) {
14689 if (V2BroadcastIdx < 0)
14690 V2BroadcastIdx = M - Size;
14691 else if (M - Size != V2BroadcastIdx)
14692 return false;
14693 } else if (M >= 0) {
14694 if (V1BroadcastIdx < 0)
14695 V1BroadcastIdx = M;
14696 else if (M != V1BroadcastIdx)
14697 return false;
14698 }
14699 return true;
14700 };
14701 if (DoBothBroadcast())
14702 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Zeroable,
14703 Subtarget, DAG);
14704
14705 // If the inputs all stem from a single 128-bit lane of each input, then we
14706 // split them rather than blending because the split will decompose to
14707 // unusually few instructions.
14708 int LaneCount = VT.getSizeInBits() / 128;
14709 int LaneSize = Size / LaneCount;
14710 SmallBitVector LaneInputs[2];
14711 LaneInputs[0].resize(LaneCount, false);
14712 LaneInputs[1].resize(LaneCount, false);
14713 for (int i = 0; i < Size; ++i)
14714 if (Mask[i] >= 0)
14715 LaneInputs[Mask[i] / Size][(Mask[i] % Size) / LaneSize] = true;
14716 if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1)
14717 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
14718 /*SimpleOnly*/ false);
14719
14720 // Otherwise, just fall back to decomposed shuffles and a blend/unpack. This
14721 // requires that the decomposed single-input shuffles don't end up here.
14722 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Zeroable,
14723 Subtarget, DAG);
14724}
14725
14726// Lower as SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
14727// TODO: Extend to support v8f32 (+ 512-bit shuffles).
14728static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT,
14729 SDValue V1, SDValue V2,
14730 ArrayRef<int> Mask,
14731 SelectionDAG &DAG) {
14732 assert(VT == MVT::v4f64 && "Only for v4f64 shuffles");
14733
14734 int LHSMask[4] = {-1, -1, -1, -1};
14735 int RHSMask[4] = {-1, -1, -1, -1};
14736 unsigned SHUFPMask = 0;
14737
14738 // As SHUFPD uses a single LHS/RHS element per lane, we can always
14739 // perform the shuffle once the lanes have been shuffled in place.
14740 for (int i = 0; i != 4; ++i) {
14741 int M = Mask[i];
14742 if (M < 0)
14743 continue;
14744 int LaneBase = i & ~1;
14745 auto &LaneMask = (i & 1) ? RHSMask : LHSMask;
14746 LaneMask[LaneBase + (M & 1)] = M;
14747 SHUFPMask |= (M & 1) << i;
14748 }
14749
14750 SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask);
14751 SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask);
14752 return DAG.getNode(X86ISD::SHUFP, DL, VT, LHS, RHS,
14753 DAG.getTargetConstant(SHUFPMask, DL, MVT::i8));
14754}
14755
14756/// Lower a vector shuffle crossing multiple 128-bit lanes as
14757/// a lane permutation followed by a per-lane permutation.
14758///
14759/// This is mainly for cases where we can have non-repeating permutes
14760/// in each lane.
14761///
14762/// TODO: This is very similar to lowerShuffleAsLanePermuteAndRepeatedMask,
14763/// we should investigate merging them.
14764static SDValue lowerShuffleAsLanePermuteAndPermute(
14765 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14766 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
14767 int NumElts = VT.getVectorNumElements();
14768 int NumLanes = VT.getSizeInBits() / 128;
14769 int NumEltsPerLane = NumElts / NumLanes;
14770 bool CanUseSublanes = Subtarget.hasAVX2() && V2.isUndef();
14771
14772 /// Attempts to find a sublane permute with the given size
14773 /// that gets all elements into their target lanes.
14774 ///
14775 /// If successful, fills CrossLaneMask and InLaneMask and returns true.
14776 /// If unsuccessful, returns false and may overwrite InLaneMask.
14777 auto getSublanePermute = [&](int NumSublanes) -> SDValue {
14778 int NumSublanesPerLane = NumSublanes / NumLanes;
14779 int NumEltsPerSublane = NumElts / NumSublanes;
14780
14781 SmallVector<int, 16> CrossLaneMask;
14782 SmallVector<int, 16> InLaneMask(NumElts, SM_SentinelUndef);
14783 // CrossLaneMask but one entry == one sublane.
14784 SmallVector<int, 16> CrossLaneMaskLarge(NumSublanes, SM_SentinelUndef);
14785
14786 for (int i = 0; i != NumElts; ++i) {
14787 int M = Mask[i];
14788 if (M < 0)
14789 continue;
14790
14791 int SrcSublane = M / NumEltsPerSublane;
14792 int DstLane = i / NumEltsPerLane;
14793
14794 // We only need to get the elements into the right lane, not sublane.
14795 // So search all sublanes that make up the destination lane.
14796 bool Found = false;
14797 int DstSubStart = DstLane * NumSublanesPerLane;
14798 int DstSubEnd = DstSubStart + NumSublanesPerLane;
14799 for (int DstSublane = DstSubStart; DstSublane < DstSubEnd; ++DstSublane) {
14800 if (!isUndefOrEqual(CrossLaneMaskLarge[DstSublane], SrcSublane))
14801 continue;
14802
14803 Found = true;
14804 CrossLaneMaskLarge[DstSublane] = SrcSublane;
14805 int DstSublaneOffset = DstSublane * NumEltsPerSublane;
14806 InLaneMask[i] = DstSublaneOffset + M % NumEltsPerSublane;
14807 break;
14808 }
14809 if (!Found)
14810 return SDValue();
14811 }
14812
14813 // Fill CrossLaneMask using CrossLaneMaskLarge.
14814 narrowShuffleMaskElts(NumEltsPerSublane, CrossLaneMaskLarge, CrossLaneMask);
14815
14816 if (!CanUseSublanes) {
14817 // If we're only shuffling a single lowest lane and the rest are identity
14818 // then don't bother.
14819 // TODO - isShuffleMaskInputInPlace could be extended to something like
14820 // this.
14821 int NumIdentityLanes = 0;
14822 bool OnlyShuffleLowestLane = true;
14823 for (int i = 0; i != NumLanes; ++i) {
14824 int LaneOffset = i * NumEltsPerLane;
14825 if (isSequentialOrUndefInRange(InLaneMask, LaneOffset, NumEltsPerLane,
14826 i * NumEltsPerLane))
14827 NumIdentityLanes++;
14828 else if (CrossLaneMask[LaneOffset] != 0)
14829 OnlyShuffleLowestLane = false;
14830 }
14831 if (OnlyShuffleLowestLane && NumIdentityLanes == (NumLanes - 1))
14832 return SDValue();
14833 }
14834
14835 // Avoid returning the same shuffle operation. For example,
14836 // t7: v16i16 = vector_shuffle<8,9,10,11,4,5,6,7,0,1,2,3,12,13,14,15> t5,
14837 // undef:v16i16
14838 if (CrossLaneMask == Mask || InLaneMask == Mask)
14839 return SDValue();
14840
14841 SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask);
14842 return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT),
14843 InLaneMask);
14844 };
14845
14846 // First attempt a solution with full lanes.
14847 if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes))
14848 return V;
14849
14850 // The rest of the solutions use sublanes.
14851 if (!CanUseSublanes)
14852 return SDValue();
14853
14854 // Then attempt a solution with 64-bit sublanes (vpermq).
14855 if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes * 2))
14856 return V;
14857
14858 // If that doesn't work and we have fast variable cross-lane shuffle,
14859 // attempt 32-bit sublanes (vpermd).
14860 if (!Subtarget.hasFastVariableCrossLaneShuffle())
14861 return SDValue();
14862
14863 return getSublanePermute(/*NumSublanes=*/NumLanes * 4);
14864}
14865
14866/// Helper to get compute inlane shuffle mask for a complete shuffle mask.
14867static void computeInLaneShuffleMask(const ArrayRef<int> &Mask, int LaneSize,
14868 SmallVector<int> &InLaneMask) {
14869 int Size = Mask.size();
14870 InLaneMask.assign(Mask.begin(), Mask.end());
14871 for (int i = 0; i < Size; ++i) {
14872 int &M = InLaneMask[i];
14873 if (M < 0)
14874 continue;
14875 if (((M % Size) / LaneSize) != (i / LaneSize))
14876 M = (M % LaneSize) + ((i / LaneSize) * LaneSize) + Size;
14877 }
14878}
14879
14880/// Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one
14881/// source with a lane permutation.
14882///
14883/// This lowering strategy results in four instructions in the worst case for a
14884/// single-input cross lane shuffle which is lower than any other fully general
14885/// cross-lane shuffle strategy I'm aware of. Special cases for each particular
14886/// shuffle pattern should be handled prior to trying this lowering.
14887static SDValue lowerShuffleAsLanePermuteAndShuffle(
14888 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14889 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
14890 // FIXME: This should probably be generalized for 512-bit vectors as well.
14891 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
14892 int Size = Mask.size();
14893 int LaneSize = Size / 2;
14894
14895 // Fold to SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
14896 // Only do this if the elements aren't all from the lower lane,
14897 // otherwise we're (probably) better off doing a split.
14898 if (VT == MVT::v4f64 &&
14899 !all_of(Mask, [LaneSize](int M) { return M < LaneSize; }))
14900 return lowerShuffleAsLanePermuteAndSHUFP(DL, VT, V1, V2, Mask, DAG);
14901
14902 // If there are only inputs from one 128-bit lane, splitting will in fact be
14903 // less expensive. The flags track whether the given lane contains an element
14904 // that crosses to another lane.
14905 bool AllLanes;
14906 if (!Subtarget.hasAVX2()) {
14907 bool LaneCrossing[2] = {false, false};
14908 for (int i = 0; i < Size; ++i)
14909 if (Mask[i] >= 0 && ((Mask[i] % Size) / LaneSize) != (i / LaneSize))
14910 LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
14911 AllLanes = LaneCrossing[0] && LaneCrossing[1];
14912 } else {
14913 bool LaneUsed[2] = {false, false};
14914 for (int i = 0; i < Size; ++i)
14915 if (Mask[i] >= 0)
14916 LaneUsed[(Mask[i] % Size) / LaneSize] = true;
14917 AllLanes = LaneUsed[0] && LaneUsed[1];
14918 }
14919
14920 // TODO - we could support shuffling V2 in the Flipped input.
14921 assert(V2.isUndef() &&
14922 "This last part of this routine only works on single input shuffles");
14923
14924 SmallVector<int> InLaneMask;
14925 computeInLaneShuffleMask(Mask, Mask.size() / 2, InLaneMask);
14926
14927 assert(!is128BitLaneCrossingShuffleMask(VT, InLaneMask) &&
14928 "In-lane shuffle mask expected");
14929
14930 // If we're not using both lanes in each lane and the inlane mask is not
14931 // repeating, then we're better off splitting.
14932 if (!AllLanes && !is128BitLaneRepeatedShuffleMask(VT, InLaneMask))
14933 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
14934 /*SimpleOnly*/ false);
14935
14936 // Flip the lanes, and shuffle the results which should now be in-lane.
14937 MVT PVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;
14938 SDValue Flipped = DAG.getBitcast(PVT, V1);
14939 Flipped =
14940 DAG.getVectorShuffle(PVT, DL, Flipped, DAG.getUNDEF(PVT), {2, 3, 0, 1});
14941 Flipped = DAG.getBitcast(VT, Flipped);
14942 return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask);
14943}
14944
14945/// Handle lowering 2-lane 128-bit shuffles.
14946static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
14947 SDValue V2, ArrayRef<int> Mask,
14948 const APInt &Zeroable,
14949 const X86Subtarget &Subtarget,
14950 SelectionDAG &DAG) {
14951 if (V2.isUndef()) {
14952 // Attempt to match VBROADCAST*128 subvector broadcast load.
14953 bool SplatLo = isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1);
14954 bool SplatHi = isShuffleEquivalent(Mask, {2, 3, 2, 3}, V1);
14955 if ((SplatLo || SplatHi) && !Subtarget.hasAVX512() && V1.hasOneUse() &&
14956 X86::mayFoldLoad(peekThroughOneUseBitcasts(V1), Subtarget)) {
14957 MVT MemVT = VT.getHalfNumVectorElementsVT();
14958 unsigned Ofs = SplatLo ? 0 : MemVT.getStoreSize();
14959 auto *Ld = cast<LoadSDNode>(peekThroughOneUseBitcasts(V1));
14960 if (SDValue BcstLd = getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, DL,
14961 VT, MemVT, Ld, Ofs, DAG))
14962 return BcstLd;
14963 }
14964
14965 // With AVX2, use VPERMQ/VPERMPD for unary shuffles to allow memory folding.
14966 if (Subtarget.hasAVX2())
14967 return SDValue();
14968 }
14969
14970 bool V2IsZero = !V2.isUndef() && ISD::isBuildVectorAllZeros(V2.getNode());
14971
14972 SmallVector<int, 4> WidenedMask;
14973 if (!canWidenShuffleElements(Mask, Zeroable, V2IsZero, WidenedMask))
14974 return SDValue();
14975
14976 bool IsLowZero = (Zeroable & 0x3) == 0x3;
14977 bool IsHighZero = (Zeroable & 0xc) == 0xc;
14978
14979 // Try to use an insert into a zero vector.
14980 if (WidenedMask[0] == 0 && IsHighZero) {
14981 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
14982 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
14983 DAG.getIntPtrConstant(0, DL));
14984 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
14985 getZeroVector(VT, Subtarget, DAG, DL), LoV,
14986 DAG.getIntPtrConstant(0, DL));
14987 }
14988
14989 // TODO: If minimizing size and one of the inputs is a zero vector and the
14990 // the zero vector has only one use, we could use a VPERM2X128 to save the
14991 // instruction bytes needed to explicitly generate the zero vector.
14992
14993 // Blends are faster and handle all the non-lane-crossing cases.
14994 if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
14995 Subtarget, DAG))
14996 return Blend;
14997
14998 // If either input operand is a zero vector, use VPERM2X128 because its mask
14999 // allows us to replace the zero input with an implicit zero.
15000 if (!IsLowZero && !IsHighZero) {
15001 // Check for patterns which can be matched with a single insert of a 128-bit
15002 // subvector.
15003 bool OnlyUsesV1 = isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1, V2);
15004 if (OnlyUsesV1 || isShuffleEquivalent(Mask, {0, 1, 4, 5}, V1, V2)) {
15005
15006 // With AVX1, use vperm2f128 (below) to allow load folding. Otherwise,
15007 // this will likely become vinsertf128 which can't fold a 256-bit memop.
15008 if (!isa<LoadSDNode>(peekThroughBitcasts(V1))) {
15009 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
15010 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
15011 OnlyUsesV1 ? V1 : V2,
15012 DAG.getIntPtrConstant(0, DL));
15013 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
15014 DAG.getIntPtrConstant(2, DL));
15015 }
15016 }
15017
15018 // Try to use SHUF128 if possible.
15019 if (Subtarget.hasVLX()) {
15020 if (WidenedMask[0] < 2 && WidenedMask[1] >= 2) {
15021 unsigned PermMask = ((WidenedMask[0] % 2) << 0) |
15022 ((WidenedMask[1] % 2) << 1);
15023 return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
15024 DAG.getTargetConstant(PermMask, DL, MVT::i8));
15025 }
15026 }
15027 }
15028
15029 // Otherwise form a 128-bit permutation. After accounting for undefs,
15030 // convert the 64-bit shuffle mask selection values into 128-bit
15031 // selection bits by dividing the indexes by 2 and shifting into positions
15032 // defined by a vperm2*128 instruction's immediate control byte.
15033
15034 // The immediate permute control byte looks like this:
15035 // [1:0] - select 128 bits from sources for low half of destination
15036 // [2] - ignore
15037 // [3] - zero low half of destination
15038 // [5:4] - select 128 bits from sources for high half of destination
15039 // [6] - ignore
15040 // [7] - zero high half of destination
15041
15042 assert((WidenedMask[0] >= 0 || IsLowZero) &&
15043 (WidenedMask[1] >= 0 || IsHighZero) && "Undef half?");
15044
15045 unsigned PermMask = 0;
15046 PermMask |= IsLowZero ? 0x08 : (WidenedMask[0] << 0);
15047 PermMask |= IsHighZero ? 0x80 : (WidenedMask[1] << 4);
15048
15049 // Check the immediate mask and replace unused sources with undef.
15050 if ((PermMask & 0x0a) != 0x00 && (PermMask & 0xa0) != 0x00)
15051 V1 = DAG.getUNDEF(VT);
15052 if ((PermMask & 0x0a) != 0x02 && (PermMask & 0xa0) != 0x20)
15053 V2 = DAG.getUNDEF(VT);
15054
15055 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
15056 DAG.getTargetConstant(PermMask, DL, MVT::i8));
15057}
15058
15059/// Lower a vector shuffle by first fixing the 128-bit lanes and then
15060/// shuffling each lane.
15061///
15062/// This attempts to create a repeated lane shuffle where each lane uses one
15063/// or two of the lanes of the inputs. The lanes of the input vectors are
15064/// shuffled in one or two independent shuffles to get the lanes into the
15065/// position needed by the final shuffle.
15066static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(
15067 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15068 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
15069 assert(!V2.isUndef() && "This is only useful with multiple inputs.");
15070
15071 if (is128BitLaneRepeatedShuffleMask(VT, Mask))
15072 return SDValue();
15073
15074 int NumElts = Mask.size();
15075 int NumLanes = VT.getSizeInBits() / 128;
15076 int NumLaneElts = 128 / VT.getScalarSizeInBits();
15077 SmallVector<int, 16> RepeatMask(NumLaneElts, -1);
15078 SmallVector<std::array<int, 2>, 2> LaneSrcs(NumLanes, {{-1, -1}});
15079
15080 // First pass will try to fill in the RepeatMask from lanes that need two
15081 // sources.
15082 for (int Lane = 0; Lane != NumLanes; ++Lane) {
15083 int Srcs[2] = {-1, -1};
15084 SmallVector<int, 16> InLaneMask(NumLaneElts, -1);
15085 for (int i = 0; i != NumLaneElts; ++i) {
15086 int M = Mask[(Lane * NumLaneElts) + i];
15087 if (M < 0)
15088 continue;
15089 // Determine which of the possible input lanes (NumLanes from each source)
15090 // this element comes from. Assign that as one of the sources for this
15091 // lane. We can assign up to 2 sources for this lane. If we run out
15092 // sources we can't do anything.
15093 int LaneSrc = M / NumLaneElts;
15094 int Src;
15095 if (Srcs[0] < 0 || Srcs[0] == LaneSrc)
15096 Src = 0;
15097 else if (Srcs[1] < 0 || Srcs[1] == LaneSrc)
15098 Src = 1;
15099 else
15100 return SDValue();
15101
15102 Srcs[Src] = LaneSrc;
15103 InLaneMask[i] = (M % NumLaneElts) + Src * NumElts;
15104 }
15105
15106 // If this lane has two sources, see if it fits with the repeat mask so far.
15107 if (Srcs[1] < 0)
15108 continue;
15109
15110 LaneSrcs[Lane][0] = Srcs[0];
15111 LaneSrcs[Lane][1] = Srcs[1];
15112
15113 auto MatchMasks = [](ArrayRef<int> M1, ArrayRef<int> M2) {
15114 assert(M1.size() == M2.size() && "Unexpected mask size");
15115 for (int i = 0, e = M1.size(); i != e; ++i)
15116 if (M1[i] >= 0 && M2[i] >= 0 && M1[i] != M2[i])
15117 return false;
15118 return true;
15119 };
15120
15121 auto MergeMasks = [](ArrayRef<int> Mask, MutableArrayRef<int> MergedMask) {
15122 assert(Mask.size() == MergedMask.size() && "Unexpected mask size");
15123 for (int i = 0, e = MergedMask.size(); i != e; ++i) {
15124 int M = Mask[i];
15125 if (M < 0)
15126 continue;
15127 assert((MergedMask[i] < 0 || MergedMask[i] == M) &&
15128 "Unexpected mask element");
15129 MergedMask[i] = M;
15130 }
15131 };
15132
15133 if (MatchMasks(InLaneMask, RepeatMask)) {
15134 // Merge this lane mask into the final repeat mask.
15135 MergeMasks(InLaneMask, RepeatMask);
15136 continue;
15137 }
15138
15139 // Didn't find a match. Swap the operands and try again.
15140 std::swap(LaneSrcs[Lane][0], LaneSrcs[Lane][1]);
15141 ShuffleVectorSDNode::commuteMask(InLaneMask);
15142
15143 if (MatchMasks(InLaneMask, RepeatMask)) {
15144 // Merge this lane mask into the final repeat mask.
15145 MergeMasks(InLaneMask, RepeatMask);
15146 continue;
15147 }
15148
15149 // Couldn't find a match with the operands in either order.
15150 return SDValue();
15151 }
15152
15153 // Now handle any lanes with only one source.
15154 for (int Lane = 0; Lane != NumLanes; ++Lane) {
15155 // If this lane has already been processed, skip it.
15156 if (LaneSrcs[Lane][0] >= 0)
15157 continue;
15158
15159 for (int i = 0; i != NumLaneElts; ++i) {
15160 int M = Mask[(Lane * NumLaneElts) + i];
15161 if (M < 0)
15162 continue;
15163
15164 // If RepeatMask isn't defined yet we can define it ourself.
15165 if (RepeatMask[i] < 0)
15166 RepeatMask[i] = M % NumLaneElts;
15167
15168 if (RepeatMask[i] < NumElts) {
15169 if (RepeatMask[i] != M % NumLaneElts)
15170 return SDValue();
15171 LaneSrcs[Lane][0] = M / NumLaneElts;
15172 } else {
15173 if (RepeatMask[i] != ((M % NumLaneElts) + NumElts))
15174 return SDValue();
15175 LaneSrcs[Lane][1] = M / NumLaneElts;
15176 }
15177 }
15178
15179 if (LaneSrcs[Lane][0] < 0 && LaneSrcs[Lane][1] < 0)
15180 return SDValue();
15181 }
15182
15183 SmallVector<int, 16> NewMask(NumElts, -1);
15184 for (int Lane = 0; Lane != NumLanes; ++Lane) {
15185 int Src = LaneSrcs[Lane][0];
15186 for (int i = 0; i != NumLaneElts; ++i) {
15187 int M = -1;
15188 if (Src >= 0)
15189 M = Src * NumLaneElts + i;
15190 NewMask[Lane * NumLaneElts + i] = M;
15191 }
15192 }
15193 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15194 // Ensure we didn't get back the shuffle we started with.
15195 // FIXME: This is a hack to make up for some splat handling code in
15196 // getVectorShuffle.
15197 if (isa<ShuffleVectorSDNode>(NewV1) &&
15198 cast<ShuffleVectorSDNode>(NewV1)->getMask() == Mask)
15199 return SDValue();
15200
15201 for (int Lane = 0; Lane != NumLanes; ++Lane) {
15202 int Src = LaneSrcs[Lane][1];
15203 for (int i = 0; i != NumLaneElts; ++i) {
15204 int M = -1;
15205 if (Src >= 0)
15206 M = Src * NumLaneElts + i;
15207 NewMask[Lane * NumLaneElts + i] = M;
15208 }
15209 }
15210 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15211 // Ensure we didn't get back the shuffle we started with.
15212 // FIXME: This is a hack to make up for some splat handling code in
15213 // getVectorShuffle.
15214 if (isa<ShuffleVectorSDNode>(NewV2) &&
15215 cast<ShuffleVectorSDNode>(NewV2)->getMask() == Mask)
15216 return SDValue();
15217
15218 for (int i = 0; i != NumElts; ++i) {
15219 if (Mask[i] < 0) {
15220 NewMask[i] = -1;
15221 continue;
15222 }
15223 NewMask[i] = RepeatMask[i % NumLaneElts];
15224 if (NewMask[i] < 0)
15225 continue;
15226
15227 NewMask[i] += (i / NumLaneElts) * NumLaneElts;
15228 }
15229 return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask);
15230}
15231
15232/// If the input shuffle mask results in a vector that is undefined in all upper
15233/// or lower half elements and that mask accesses only 2 halves of the
15234/// shuffle's operands, return true. A mask of half the width with mask indexes
15235/// adjusted to access the extracted halves of the original shuffle operands is
15236/// returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or
15237/// lower half of each input operand is accessed.
15238static bool
15239getHalfShuffleMask(ArrayRef<int> Mask, MutableArrayRef<int> HalfMask,
15240 int &HalfIdx1, int &HalfIdx2) {
15241 assert((Mask.size() == HalfMask.size() * 2) &&
15242 "Expected input mask to be twice as long as output");
15243
15244 // Exactly one half of the result must be undef to allow narrowing.
15245 bool UndefLower = isUndefLowerHalf(Mask);
15246 bool UndefUpper = isUndefUpperHalf(Mask);
15247 if (UndefLower == UndefUpper)
15248 return false;
15249
15250 unsigned HalfNumElts = HalfMask.size();
15251 unsigned MaskIndexOffset = UndefLower ? HalfNumElts : 0;
15252 HalfIdx1 = -1;
15253 HalfIdx2 = -1;
15254 for (unsigned i = 0; i != HalfNumElts; ++i) {
15255 int M = Mask[i + MaskIndexOffset];
15256 if (M < 0) {
15257 HalfMask[i] = M;
15258 continue;
15259 }
15260
15261 // Determine which of the 4 half vectors this element is from.
15262 // i.e. 0 = Lower V1, 1 = Upper V1, 2 = Lower V2, 3 = Upper V2.
15263 int HalfIdx = M / HalfNumElts;
15264
15265 // Determine the element index into its half vector source.
15266 int HalfElt = M % HalfNumElts;
15267
15268 // We can shuffle with up to 2 half vectors, set the new 'half'
15269 // shuffle mask accordingly.
15270 if (HalfIdx1 < 0 || HalfIdx1 == HalfIdx) {
15271 HalfMask[i] = HalfElt;
15272 HalfIdx1 = HalfIdx;
15273 continue;
15274 }
15275 if (HalfIdx2 < 0 || HalfIdx2 == HalfIdx) {
15276 HalfMask[i] = HalfElt + HalfNumElts;
15277 HalfIdx2 = HalfIdx;
15278 continue;
15279 }
15280
15281 // Too many half vectors referenced.
15282 return false;
15283 }
15284
15285 return true;
15286}
15287
15288/// Given the output values from getHalfShuffleMask(), create a half width
15289/// shuffle of extracted vectors followed by an insert back to full width.
15290static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15291 ArrayRef<int> HalfMask, int HalfIdx1,
15292 int HalfIdx2, bool UndefLower,
15293 SelectionDAG &DAG, bool UseConcat = false) {
15294 assert(V1.getValueType() == V2.getValueType() && "Different sized vectors?");
15295 assert(V1.getValueType().isSimple() && "Expecting only simple types");
15296
15297 MVT VT = V1.getSimpleValueType();
15298 MVT HalfVT = VT.getHalfNumVectorElementsVT();
15299 unsigned HalfNumElts = HalfVT.getVectorNumElements();
15300
15301 auto getHalfVector = [&](int HalfIdx) {
15302 if (HalfIdx < 0)
15303 return DAG.getUNDEF(HalfVT);
15304 SDValue V = (HalfIdx < 2 ? V1 : V2);
15305 HalfIdx = (HalfIdx % 2) * HalfNumElts;
15306 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V,
15307 DAG.getIntPtrConstant(HalfIdx, DL));
15308 };
15309
15310 // ins undef, (shuf (ext V1, HalfIdx1), (ext V2, HalfIdx2), HalfMask), Offset
15311 SDValue Half1 = getHalfVector(HalfIdx1);
15312 SDValue Half2 = getHalfVector(HalfIdx2);
15313 SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
15314 if (UseConcat) {
15315 SDValue Op0 = V;
15316 SDValue Op1 = DAG.getUNDEF(HalfVT);
15317 if (UndefLower)
15318 std::swap(Op0, Op1);
15319 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1);
15320 }
15321
15322 unsigned Offset = UndefLower ? HalfNumElts : 0;
15323 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V,
15324 DAG.getIntPtrConstant(Offset, DL));
15325}
15326
15327/// Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
15328/// This allows for fast cases such as subvector extraction/insertion
15329/// or shuffling smaller vector types which can lower more efficiently.
15330static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
15331 SDValue V2, ArrayRef<int> Mask,
15332 const X86Subtarget &Subtarget,
15333 SelectionDAG &DAG) {
15334 assert((VT.is256BitVector() || VT.is512BitVector()) &&
15335 "Expected 256-bit or 512-bit vector");
15336
15337 bool UndefLower = isUndefLowerHalf(Mask);
15338 if (!UndefLower && !isUndefUpperHalf(Mask))
15339 return SDValue();
15340
15341 assert((!UndefLower || !isUndefUpperHalf(Mask)) &&
15342 "Completely undef shuffle mask should have been simplified already");
15343
15344 // Upper half is undef and lower half is whole upper subvector.
15345 // e.g. vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
15346 MVT HalfVT = VT.getHalfNumVectorElementsVT();
15347 unsigned HalfNumElts = HalfVT.getVectorNumElements();
15348 if (!UndefLower &&
15349 isSequentialOrUndefInRange(Mask, 0, HalfNumElts, HalfNumElts)) {
15350 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15351 DAG.getIntPtrConstant(HalfNumElts, DL));
15352 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15353 DAG.getIntPtrConstant(0, DL));
15354 }
15355
15356 // Lower half is undef and upper half is whole lower subvector.
15357 // e.g. vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
15358 if (UndefLower &&
15359 isSequentialOrUndefInRange(Mask, HalfNumElts, HalfNumElts, 0)) {
15360 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15361 DAG.getIntPtrConstant(0, DL));
15362 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15363 DAG.getIntPtrConstant(HalfNumElts, DL));
15364 }
15365
15366 int HalfIdx1, HalfIdx2;
15367 SmallVector<int, 8> HalfMask(HalfNumElts);
15368 if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2))
15369 return SDValue();
15370
15371 assert(HalfMask.size() == HalfNumElts && "Unexpected shuffle mask length");
15372
15373 // Only shuffle the halves of the inputs when useful.
15374 unsigned NumLowerHalves =
15375 (HalfIdx1 == 0 || HalfIdx1 == 2) + (HalfIdx2 == 0 || HalfIdx2 == 2);
15376 unsigned NumUpperHalves =
15377 (HalfIdx1 == 1 || HalfIdx1 == 3) + (HalfIdx2 == 1 || HalfIdx2 == 3);
15378 assert(NumLowerHalves + NumUpperHalves <= 2 && "Only 1 or 2 halves allowed");
15379
15380 // Determine the larger pattern of undef/halves, then decide if it's worth
15381 // splitting the shuffle based on subtarget capabilities and types.
15382 unsigned EltWidth = VT.getVectorElementType().getSizeInBits();
15383 if (!UndefLower) {
15384 // XXXXuuuu: no insert is needed.
15385 // Always extract lowers when setting lower - these are all free subreg ops.
15386 if (NumUpperHalves == 0)
15387 return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15388 UndefLower, DAG);
15389
15390 if (NumUpperHalves == 1) {
15391 // AVX2 has efficient 32/64-bit element cross-lane shuffles.
15392 if (Subtarget.hasAVX2()) {
15393 // extract128 + vunpckhps/vshufps, is better than vblend + vpermps.
15394 if (EltWidth == 32 && NumLowerHalves && HalfVT.is128BitVector() &&
15395 !is128BitUnpackShuffleMask(HalfMask, DAG) &&
15396 (!isSingleSHUFPSMask(HalfMask) ||
15397 Subtarget.hasFastVariableCrossLaneShuffle()))
15398 return SDValue();
15399 // If this is a unary shuffle (assume that the 2nd operand is
15400 // canonicalized to undef), then we can use vpermpd. Otherwise, we
15401 // are better off extracting the upper half of 1 operand and using a
15402 // narrow shuffle.
15403 if (EltWidth == 64 && V2.isUndef())
15404 return SDValue();
15405 }
15406 // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
15407 if (Subtarget.hasAVX512() && VT.is512BitVector())
15408 return SDValue();
15409 // Extract + narrow shuffle is better than the wide alternative.
15410 return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15411 UndefLower, DAG);
15412 }
15413
15414 // Don't extract both uppers, instead shuffle and then extract.
15415 assert(NumUpperHalves == 2 && "Half vector count went wrong");
15416 return SDValue();
15417 }
15418
15419 // UndefLower - uuuuXXXX: an insert to high half is required if we split this.
15420 if (NumUpperHalves == 0) {
15421 // AVX2 has efficient 64-bit element cross-lane shuffles.
15422 // TODO: Refine to account for unary shuffle, splat, and other masks?
15423 if (Subtarget.hasAVX2() && EltWidth == 64)
15424 return SDValue();
15425 // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
15426 if (Subtarget.hasAVX512() && VT.is512BitVector())
15427 return SDValue();
15428 // Narrow shuffle + insert is better than the wide alternative.
15429 return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15430 UndefLower, DAG);
15431 }
15432
15433 // NumUpperHalves != 0: don't bother with extract, shuffle, and then insert.
15434 return SDValue();
15435}
15436
15437/// Handle case where shuffle sources are coming from the same 128-bit lane and
15438/// every lane can be represented as the same repeating mask - allowing us to
15439/// shuffle the sources with the repeating shuffle and then permute the result
15440/// to the destination lanes.
15441static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(
15442 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15443 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
15444 int NumElts = VT.getVectorNumElements();
15445 int NumLanes = VT.getSizeInBits() / 128;
15446 int NumLaneElts = NumElts / NumLanes;
15447
15448 // On AVX2 we may be able to just shuffle the lowest elements and then
15449 // broadcast the result.
15450 if (Subtarget.hasAVX2()) {
15451 for (unsigned BroadcastSize : {16, 32, 64}) {
15452 if (BroadcastSize <= VT.getScalarSizeInBits())
15453 continue;
15454 int NumBroadcastElts = BroadcastSize / VT.getScalarSizeInBits();
15455
15456 // Attempt to match a repeating pattern every NumBroadcastElts,
15457 // accounting for UNDEFs but only references the lowest 128-bit
15458 // lane of the inputs.
15459 auto FindRepeatingBroadcastMask = [&](SmallVectorImpl<int> &RepeatMask) {
15460 for (int i = 0; i != NumElts; i += NumBroadcastElts)
15461 for (int j = 0; j != NumBroadcastElts; ++j) {
15462 int M = Mask[i + j];
15463 if (M < 0)
15464 continue;
15465 int &R = RepeatMask[j];
15466 if (0 != ((M % NumElts) / NumLaneElts))
15467 return false;
15468 if (0 <= R && R != M)
15469 return false;
15470 R = M;
15471 }
15472 return true;
15473 };
15474
15475 SmallVector<int, 8> RepeatMask((unsigned)NumElts, -1);
15476 if (!FindRepeatingBroadcastMask(RepeatMask))
15477 continue;
15478
15479 // Shuffle the (lowest) repeated elements in place for broadcast.
15480 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
15481
15482 // Shuffle the actual broadcast.
15483 SmallVector<int, 8> BroadcastMask((unsigned)NumElts, -1);
15484 for (int i = 0; i != NumElts; i += NumBroadcastElts)
15485 for (int j = 0; j != NumBroadcastElts; ++j)
15486 BroadcastMask[i + j] = j;
15487
15488 // Avoid returning the same shuffle operation. For example,
15489 // v8i32 = vector_shuffle<0,1,0,1,0,1,0,1> t5, undef:v8i32
15490 if (BroadcastMask == Mask)
15491 return SDValue();
15492
15493 return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT),
15494 BroadcastMask);
15495 }
15496 }
15497
15498 // Bail if the shuffle mask doesn't cross 128-bit lanes.
15499 if (!is128BitLaneCrossingShuffleMask(VT, Mask))
15500 return SDValue();
15501
15502 // Bail if we already have a repeated lane shuffle mask.
15503 if (is128BitLaneRepeatedShuffleMask(VT, Mask))
15504 return SDValue();
15505
15506 // Helper to look for repeated mask in each split sublane, and that those
15507 // sublanes can then be permuted into place.
15508 auto ShuffleSubLanes = [&](int SubLaneScale) {
15509 int NumSubLanes = NumLanes * SubLaneScale;
15510 int NumSubLaneElts = NumLaneElts / SubLaneScale;
15511
15512 // Check that all the sources are coming from the same lane and see if we
15513 // can form a repeating shuffle mask (local to each sub-lane). At the same
15514 // time, determine the source sub-lane for each destination sub-lane.
15515 int TopSrcSubLane = -1;
15516 SmallVector<int, 8> Dst2SrcSubLanes((unsigned)NumSubLanes, -1);
15517 SmallVector<SmallVector<int, 8>> RepeatedSubLaneMasks(
15518 SubLaneScale,
15519 SmallVector<int, 8>((unsigned)NumSubLaneElts, SM_SentinelUndef));
15520
15521 for (int DstSubLane = 0; DstSubLane != NumSubLanes; ++DstSubLane) {
15522 // Extract the sub-lane mask, check that it all comes from the same lane
15523 // and normalize the mask entries to come from the first lane.
15524 int SrcLane = -1;
15525 SmallVector<int, 8> SubLaneMask((unsigned)NumSubLaneElts, -1);
15526 for (int Elt = 0; Elt != NumSubLaneElts; ++Elt) {
15527 int M = Mask[(DstSubLane * NumSubLaneElts) + Elt];
15528 if (M < 0)
15529 continue;
15530 int Lane = (M % NumElts) / NumLaneElts;
15531 if ((0 <= SrcLane) && (SrcLane != Lane))
15532 return SDValue();
15533 SrcLane = Lane;
15534 int LocalM = (M % NumLaneElts) + (M < NumElts ? 0 : NumElts);
15535 SubLaneMask[Elt] = LocalM;
15536 }
15537
15538 // Whole sub-lane is UNDEF.
15539 if (SrcLane < 0)
15540 continue;
15541
15542 // Attempt to match against the candidate repeated sub-lane masks.
15543 for (int SubLane = 0; SubLane != SubLaneScale; ++SubLane) {
15544 auto MatchMasks = [NumSubLaneElts](ArrayRef<int> M1, ArrayRef<int> M2) {
15545 for (int i = 0; i != NumSubLaneElts; ++i) {
15546 if (M1[i] < 0 || M2[i] < 0)
15547 continue;
15548 if (M1[i] != M2[i])
15549 return false;
15550 }
15551 return true;
15552 };
15553
15554 auto &RepeatedSubLaneMask = RepeatedSubLaneMasks[SubLane];
15555 if (!MatchMasks(SubLaneMask, RepeatedSubLaneMask))
15556 continue;
15557
15558 // Merge the sub-lane mask into the matching repeated sub-lane mask.
15559 for (int i = 0; i != NumSubLaneElts; ++i) {
15560 int M = SubLaneMask[i];
15561 if (M < 0)
15562 continue;
15563 assert((RepeatedSubLaneMask[i] < 0 || RepeatedSubLaneMask[i] == M) &&
15564 "Unexpected mask element");
15565 RepeatedSubLaneMask[i] = M;
15566 }
15567
15568 // Track the top most source sub-lane - by setting the remaining to
15569 // UNDEF we can greatly simplify shuffle matching.
15570 int SrcSubLane = (SrcLane * SubLaneScale) + SubLane;
15571 TopSrcSubLane = std::max(TopSrcSubLane, SrcSubLane);
15572 Dst2SrcSubLanes[DstSubLane] = SrcSubLane;
15573 break;
15574 }
15575
15576 // Bail if we failed to find a matching repeated sub-lane mask.
15577 if (Dst2SrcSubLanes[DstSubLane] < 0)
15578 return SDValue();
15579 }
15580 assert(0 <= TopSrcSubLane && TopSrcSubLane < NumSubLanes &&
15581 "Unexpected source lane");
15582
15583 // Create a repeating shuffle mask for the entire vector.
15584 SmallVector<int, 8> RepeatedMask((unsigned)NumElts, -1);
15585 for (int SubLane = 0; SubLane <= TopSrcSubLane; ++SubLane) {
15586 int Lane = SubLane / SubLaneScale;
15587 auto &RepeatedSubLaneMask = RepeatedSubLaneMasks[SubLane % SubLaneScale];
15588 for (int Elt = 0; Elt != NumSubLaneElts; ++Elt) {
15589 int M = RepeatedSubLaneMask[Elt];
15590 if (M < 0)
15591 continue;
15592 int Idx = (SubLane * NumSubLaneElts) + Elt;
15593 RepeatedMask[Idx] = M + (Lane * NumLaneElts);
15594 }
15595 }
15596
15597 // Shuffle each source sub-lane to its destination.
15598 SmallVector<int, 8> SubLaneMask((unsigned)NumElts, -1);
15599 for (int i = 0; i != NumElts; i += NumSubLaneElts) {
15600 int SrcSubLane = Dst2SrcSubLanes[i / NumSubLaneElts];
15601 if (SrcSubLane < 0)
15602 continue;
15603 for (int j = 0; j != NumSubLaneElts; ++j)
15604 SubLaneMask[i + j] = j + (SrcSubLane * NumSubLaneElts);
15605 }
15606
15607 // Avoid returning the same shuffle operation.
15608 // v8i32 = vector_shuffle<0,1,4,5,2,3,6,7> t5, undef:v8i32
15609 if (RepeatedMask == Mask || SubLaneMask == Mask)
15610 return SDValue();
15611
15612 SDValue RepeatedShuffle =
15613 DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);
15614
15615 return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT),
15616 SubLaneMask);
15617 };
15618
15619 // On AVX2 targets we can permute 256-bit vectors as 64-bit sub-lanes
15620 // (with PERMQ/PERMPD). On AVX2/AVX512BW targets, permuting 32-bit sub-lanes,
15621 // even with a variable shuffle, can be worth it for v32i8/v64i8 vectors.
15622 // Otherwise we can only permute whole 128-bit lanes.
15623 int MinSubLaneScale = 1, MaxSubLaneScale = 1;
15624 if (Subtarget.hasAVX2() && VT.is256BitVector()) {
15625 bool OnlyLowestElts = isUndefOrInRange(Mask, 0, NumLaneElts);
15626 MinSubLaneScale = 2;
15627 MaxSubLaneScale =
15628 (!OnlyLowestElts && V2.isUndef() && VT == MVT::v32i8) ? 4 : 2;
15629 }
15630 if (Subtarget.hasBWI() && VT == MVT::v64i8)
15631 MinSubLaneScale = MaxSubLaneScale = 4;
15632
15633 for (int Scale = MinSubLaneScale; Scale <= MaxSubLaneScale; Scale *= 2)
15634 if (SDValue Shuffle = ShuffleSubLanes(Scale))
15635 return Shuffle;
15636
15637 return SDValue();
15638}
15639
15640static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
15641 bool &ForceV1Zero, bool &ForceV2Zero,
15642 unsigned &ShuffleImm, ArrayRef<int> Mask,
15643 const APInt &Zeroable) {
15644 int NumElts = VT.getVectorNumElements();
15645 assert(VT.getScalarSizeInBits() == 64 &&
15646 (NumElts == 2 || NumElts == 4 || NumElts == 8) &&
15647 "Unexpected data type for VSHUFPD");
15648 assert(isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) &&
15649 "Illegal shuffle mask");
15650
15651 bool ZeroLane[2] = { true, true };
15652 for (int i = 0; i < NumElts; ++i)
15653 ZeroLane[i & 1] &= Zeroable[i];
15654
15655 // Mask for V8F64: 0/1, 8/9, 2/3, 10/11, 4/5, ..
15656 // Mask for V4F64; 0/1, 4/5, 2/3, 6/7..
15657 ShuffleImm = 0;
15658 bool ShufpdMask = true;
15659 bool CommutableMask = true;
15660 for (int i = 0; i < NumElts; ++i) {
15661 if (Mask[i] == SM_SentinelUndef || ZeroLane[i & 1])
15662 continue;
15663 if (Mask[i] < 0)
15664 return false;
15665 int Val = (i & 6) + NumElts * (i & 1);
15666 int CommutVal = (i & 0xe) + NumElts * ((i & 1) ^ 1);
15667 if (Mask[i] < Val || Mask[i] > Val + 1)
15668 ShufpdMask = false;
15669 if (Mask[i] < CommutVal || Mask[i] > CommutVal + 1)
15670 CommutableMask = false;
15671 ShuffleImm |= (Mask[i] % 2) << i;
15672 }
15673
15674 if (!ShufpdMask && !CommutableMask)
15675 return false;
15676
15677 if (!ShufpdMask && CommutableMask)
15678 std::swap(V1, V2);
15679
15680 ForceV1Zero = ZeroLane[0];
15681 ForceV2Zero = ZeroLane[1];
15682 return true;
15683}
15684
15685static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
15686 SDValue V2, ArrayRef<int> Mask,
15687 const APInt &Zeroable,
15688 const X86Subtarget &Subtarget,
15689 SelectionDAG &DAG) {
15690 assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) &&
15691 "Unexpected data type for VSHUFPD");
15692
15693 unsigned Immediate = 0;
15694 bool ForceV1Zero = false, ForceV2Zero = false;
15695 if (!matchShuffleWithSHUFPD(VT, V1, V2, ForceV1Zero, ForceV2Zero, Immediate,
15696 Mask, Zeroable))
15697 return SDValue();
15698
15699 // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs.
15700 if (ForceV1Zero)
15701 V1 = getZeroVector(VT, Subtarget, DAG, DL);
15702 if (ForceV2Zero)
15703 V2 = getZeroVector(VT, Subtarget, DAG, DL);
15704
15705 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
15706 DAG.getTargetConstant(Immediate, DL, MVT::i8));
15707}
15708
15709// Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
15710// by zeroable elements in the remaining 24 elements. Turn this into two
15711// vmovqb instructions shuffled together.
15712static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
15713 SDValue V1, SDValue V2,
15714 ArrayRef<int> Mask,
15715 const APInt &Zeroable,
15716 SelectionDAG &DAG) {
15717 assert(VT == MVT::v32i8 && "Unexpected type!");
15718
15719 // The first 8 indices should be every 8th element.
15720 if (!isSequentialOrUndefInRange(Mask, 0, 8, 0, 8))
15721 return SDValue();
15722
15723 // Remaining elements need to be zeroable.
15724 if (Zeroable.countl_one() < (Mask.size() - 8))
15725 return SDValue();
15726
15727 V1 = DAG.getBitcast(MVT::v4i64, V1);
15728 V2 = DAG.getBitcast(MVT::v4i64, V2);
15729
15730 V1 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V1);
15731 V2 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V2);
15732
15733 // The VTRUNCs will put 0s in the upper 12 bytes. Use them to put zeroes in
15734 // the upper bits of the result using an unpckldq.
15735 SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
15736 { 0, 1, 2, 3, 16, 17, 18, 19,
15737 4, 5, 6, 7, 20, 21, 22, 23 });
15738 // Insert the unpckldq into a zero vector to widen to v32i8.
15739 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8,
15740 DAG.getConstant(0, DL, MVT::v32i8), Unpack,
15741 DAG.getIntPtrConstant(0, DL));
15742}
15743
15744// a = shuffle v1, v2, mask1 ; interleaving lower lanes of v1 and v2
15745// b = shuffle v1, v2, mask2 ; interleaving higher lanes of v1 and v2
15746// =>
15747// ul = unpckl v1, v2
15748// uh = unpckh v1, v2
15749// a = vperm ul, uh
15750// b = vperm ul, uh
15751//
15752// Pattern-match interleave(256b v1, 256b v2) -> 512b v3 and lower it into unpck
15753// and permute. We cannot directly match v3 because it is split into two
15754// 256-bit vectors in earlier isel stages. Therefore, this function matches a
15755// pair of 256-bit shuffles and makes sure the masks are consecutive.
15756//
15757// Once unpck and permute nodes are created, the permute corresponding to this
15758// shuffle is returned, while the other permute replaces the other half of the
15759// shuffle in the selection dag.
15760static SDValue lowerShufflePairAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
15761 SDValue V1, SDValue V2,
15762 ArrayRef<int> Mask,
15763 SelectionDAG &DAG) {
15764 if (VT != MVT::v8f32 && VT != MVT::v8i32 && VT != MVT::v16i16 &&
15765 VT != MVT::v32i8)
15766 return SDValue();
15767 // <B0, B1, B0+1, B1+1, ..., >
15768 auto IsInterleavingPattern = [&](ArrayRef<int> Mask, unsigned Begin0,
15769 unsigned Begin1) {
15770 size_t Size = Mask.size();
15771 assert(Size % 2 == 0 && "Expected even mask size");
15772 for (unsigned I = 0; I < Size; I += 2) {
15773 if (Mask[I] != (int)(Begin0 + I / 2) ||
15774 Mask[I + 1] != (int)(Begin1 + I / 2))
15775 return false;
15776 }
15777 return true;
15778 };
15779 // Check which half is this shuffle node
15780 int NumElts = VT.getVectorNumElements();
15781 size_t FirstQtr = NumElts / 2;
15782 size_t ThirdQtr = NumElts + NumElts / 2;
15783 bool IsFirstHalf = IsInterleavingPattern(Mask, 0, NumElts);
15784 bool IsSecondHalf = IsInterleavingPattern(Mask, FirstQtr, ThirdQtr);
15785 if (!IsFirstHalf && !IsSecondHalf)
15786 return SDValue();
15787
15788 // Find the intersection between shuffle users of V1 and V2.
15789 SmallVector<SDNode *, 2> Shuffles;
15790 for (SDNode *User : V1->uses())
15791 if (User->getOpcode() == ISD::VECTOR_SHUFFLE && User->getOperand(0) == V1 &&
15792 User->getOperand(1) == V2)
15793 Shuffles.push_back(User);
15794 // Limit user size to two for now.
15795 if (Shuffles.size() != 2)
15796 return SDValue();
15797 // Find out which half of the 512-bit shuffles is each smaller shuffle
15798 auto *SVN1 = cast<ShuffleVectorSDNode>(Shuffles[0]);
15799 auto *SVN2 = cast<ShuffleVectorSDNode>(Shuffles[1]);
15800 SDNode *FirstHalf;
15801 SDNode *SecondHalf;
15802 if (IsInterleavingPattern(SVN1->getMask(), 0, NumElts) &&
15803 IsInterleavingPattern(SVN2->getMask(), FirstQtr, ThirdQtr)) {
15804 FirstHalf = Shuffles[0];
15805 SecondHalf = Shuffles[1];
15806 } else if (IsInterleavingPattern(SVN1->getMask(), FirstQtr, ThirdQtr) &&
15807 IsInterleavingPattern(SVN2->getMask(), 0, NumElts)) {
15808 FirstHalf = Shuffles[1];
15809 SecondHalf = Shuffles[0];
15810 } else {
15811 return SDValue();
15812 }
15813 // Lower into unpck and perm. Return the perm of this shuffle and replace
15814 // the other.
15815 SDValue Unpckl = DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
15816 SDValue Unpckh = DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
15817 SDValue Perm1 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
15818 DAG.getTargetConstant(0x20, DL, MVT::i8));
15819 SDValue Perm2 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh,
15820 DAG.getTargetConstant(0x31, DL, MVT::i8));
15821 if (IsFirstHalf) {
15822 DAG.ReplaceAllUsesWith(SecondHalf, &Perm2);
15823 return Perm1;
15824 }
15825 DAG.ReplaceAllUsesWith(FirstHalf, &Perm1);
15826 return Perm2;
15827}
15828
15829/// Handle lowering of 4-lane 64-bit floating point shuffles.
15830///
15831/// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2
15832/// isn't available.
15833static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15834 const APInt &Zeroable, SDValue V1, SDValue V2,
15835 const X86Subtarget &Subtarget,
15836 SelectionDAG &DAG) {
15837 assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
15838 assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
15839 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
15840
15841 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
15842 Subtarget, DAG))
15843 return V;
15844
15845 if (V2.isUndef()) {
15846 // Check for being able to broadcast a single element.
15847 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,
15848 Mask, Subtarget, DAG))
15849 return Broadcast;
15850
15851 // Use low duplicate instructions for masks that match their pattern.
15852 if (isShuffleEquivalent(Mask, {0, 0, 2, 2}, V1, V2))
15853 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
15854
15855 if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) {
15856 // Non-half-crossing single input shuffles can be lowered with an
15857 // interleaved permutation.
15858 unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
15859 ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3);
15860 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
15861 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
15862 }
15863
15864 // With AVX2 we have direct support for this permutation.
15865 if (Subtarget.hasAVX2())
15866 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
15867 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
15868
15869 // Try to create an in-lane repeating shuffle mask and then shuffle the
15870 // results into the target lanes.
15871 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15872 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15873 return V;
15874
15875 // Try to permute the lanes and then use a per-lane permute.
15876 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,
15877 Mask, DAG, Subtarget))
15878 return V;
15879
15880 // Otherwise, fall back.
15881 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v4f64, V1, V2, Mask,
15882 DAG, Subtarget);
15883 }
15884
15885 // Use dedicated unpack instructions for masks that match their pattern.
15886 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, V1, V2, Mask, DAG))
15887 return V;
15888
15889 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
15890 Zeroable, Subtarget, DAG))
15891 return Blend;
15892
15893 // Check if the blend happens to exactly fit that of SHUFPD.
15894 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,
15895 Zeroable, Subtarget, DAG))
15896 return Op;
15897
15898 bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
15899 bool V2IsInPlace = isShuffleMaskInputInPlace(1, Mask);
15900
15901 // If we have lane crossing shuffles AND they don't all come from the lower
15902 // lane elements, lower to SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
15903 // TODO: Handle BUILD_VECTOR sources which getVectorShuffle currently
15904 // canonicalize to a blend of splat which isn't necessary for this combine.
15905 if (is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask) &&
15906 !all_of(Mask, [](int M) { return M < 2 || (4 <= M && M < 6); }) &&
15907 (V1.getOpcode() != ISD::BUILD_VECTOR) &&
15908 (V2.getOpcode() != ISD::BUILD_VECTOR))
15909 return lowerShuffleAsLanePermuteAndSHUFP(DL, MVT::v4f64, V1, V2, Mask, DAG);
15910
15911 // If we have one input in place, then we can permute the other input and
15912 // blend the result.
15913 if (V1IsInPlace || V2IsInPlace)
15914 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
15915 Zeroable, Subtarget, DAG);
15916
15917 // Try to create an in-lane repeating shuffle mask and then shuffle the
15918 // results into the target lanes.
15919 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15920 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15921 return V;
15922
15923 // Try to simplify this by merging 128-bit lanes to enable a lane-based
15924 // shuffle. However, if we have AVX2 and either inputs are already in place,
15925 // we will be able to shuffle even across lanes the other input in a single
15926 // instruction so skip this pattern.
15927 if (!(Subtarget.hasAVX2() && (V1IsInPlace || V2IsInPlace)))
15928 if (SDValue V = lowerShuffleAsLanePermuteAndRepeatedMask(
15929 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15930 return V;
15931
15932 // If we have VLX support, we can use VEXPAND.
15933 if (Subtarget.hasVLX())
15934 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, V1, V2,
15935 DAG, Subtarget))
15936 return V;
15937
15938 // If we have AVX2 then we always want to lower with a blend because an v4 we
15939 // can fully permute the elements.
15940 if (Subtarget.hasAVX2())
15941 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
15942 Zeroable, Subtarget, DAG);
15943
15944 // Otherwise fall back on generic lowering.
15945 return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
15946 Subtarget, DAG);
15947}
15948
15949/// Handle lowering of 4-lane 64-bit integer shuffles.
15950///
15951/// This routine is only called when we have AVX2 and thus a reasonable
15952/// instruction set for v4i64 shuffling..
15953static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15954 const APInt &Zeroable, SDValue V1, SDValue V2,
15955 const X86Subtarget &Subtarget,
15956 SelectionDAG &DAG) {
15957 assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
15958 assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
15959 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
15960 assert(Subtarget.hasAVX2() && "We can only lower v4i64 with AVX2!");
15961
15962 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
15963 Subtarget, DAG))
15964 return V;
15965
15966 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
15967 Zeroable, Subtarget, DAG))
15968 return Blend;
15969
15970 // Check for being able to broadcast a single element.
15971 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
15972 Subtarget, DAG))
15973 return Broadcast;
15974
15975 // Try to use shift instructions if fast.
15976 if (Subtarget.preferLowerShuffleAsShift())
15977 if (SDValue Shift =
15978 lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
15979 Subtarget, DAG, /*BitwiseOnly*/ true))
15980 return Shift;
15981
15982 if (V2.isUndef()) {
15983 // When the shuffle is mirrored between the 128-bit lanes of the unit, we
15984 // can use lower latency instructions that will operate on both lanes.
15985 SmallVector<int, 2> RepeatedMask;
15986 if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) {
15987 SmallVector<int, 4> PSHUFDMask;
15988 narrowShuffleMaskElts(2, RepeatedMask, PSHUFDMask);
15989 return DAG.getBitcast(
15990 MVT::v4i64,
15991 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
15992 DAG.getBitcast(MVT::v8i32, V1),
15993 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
15994 }
15995
15996 // AVX2 provides a direct instruction for permuting a single input across
15997 // lanes.
15998 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
15999 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
16000 }
16001
16002 // Try to use shift instructions.
16003 if (SDValue Shift =
16004 lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, Zeroable, Subtarget,
16005 DAG, /*BitwiseOnly*/ false))
16006 return Shift;
16007
16008 // If we have VLX support, we can use VALIGN or VEXPAND.
16009 if (Subtarget.hasVLX()) {
16010 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i64, V1, V2, Mask,
16011 Zeroable, Subtarget, DAG))
16012 return Rotate;
16013
16014 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, V1, V2,
16015 DAG, Subtarget))
16016 return V;
16017 }
16018
16019 // Try to use PALIGNR.
16020 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,
16021 Subtarget, DAG))
16022 return Rotate;
16023
16024 // Use dedicated unpack instructions for masks that match their pattern.
16025 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, V1, V2, Mask, DAG))
16026 return V;
16027
16028 bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
16029 bool V2IsInPlace = isShuffleMaskInputInPlace(1, Mask);
16030
16031 // If we have one input in place, then we can permute the other input and
16032 // blend the result.
16033 if (V1IsInPlace || V2IsInPlace)
16034 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
16035 Zeroable, Subtarget, DAG);
16036
16037 // Try to create an in-lane repeating shuffle mask and then shuffle the
16038 // results into the target lanes.
16039 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16040 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
16041 return V;
16042
16043 // Try to lower to PERMQ(BLENDD(V1,V2)).
16044 if (SDValue V =
16045 lowerShuffleAsBlendAndPermute(DL, MVT::v4i64, V1, V2, Mask, DAG))
16046 return V;
16047
16048 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16049 // shuffle. However, if we have AVX2 and either inputs are already in place,
16050 // we will be able to shuffle even across lanes the other input in a single
16051 // instruction so skip this pattern.
16052 if (!V1IsInPlace && !V2IsInPlace)
16053 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16054 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
16055 return Result;
16056
16057 // Otherwise fall back on generic blend lowering.
16058 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
16059 Zeroable, Subtarget, DAG);
16060}
16061
16062/// Handle lowering of 8-lane 32-bit floating point shuffles.
16063///
16064/// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2
16065/// isn't available.
16066static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16067 const APInt &Zeroable, SDValue V1, SDValue V2,
16068 const X86Subtarget &Subtarget,
16069 SelectionDAG &DAG) {
16070 assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
16071 assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
16072 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16073
16074 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
16075 Zeroable, Subtarget, DAG))
16076 return Blend;
16077
16078 // Check for being able to broadcast a single element.
16079 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
16080 Subtarget, DAG))
16081 return Broadcast;
16082
16083 if (!Subtarget.hasAVX2()) {
16084 SmallVector<int> InLaneMask;
16085 computeInLaneShuffleMask(Mask, Mask.size() / 2, InLaneMask);
16086
16087 if (!is128BitLaneRepeatedShuffleMask(MVT::v8f32, InLaneMask))
16088 if (SDValue R = splitAndLowerShuffle(DL, MVT::v8f32, V1, V2, Mask, DAG,
16089 /*SimpleOnly*/ true))
16090 return R;
16091 }
16092 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
16093 Zeroable, Subtarget, DAG))
16094 return DAG.getBitcast(MVT::v8f32, ZExt);
16095
16096 // If the shuffle mask is repeated in each 128-bit lane, we have many more
16097 // options to efficiently lower the shuffle.
16098 SmallVector<int, 4> RepeatedMask;
16099 if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) {
16100 assert(RepeatedMask.size() == 4 &&
16101 "Repeated masks must be half the mask width!");
16102
16103 // Use even/odd duplicate instructions for masks that match their pattern.
16104 if (isShuffleEquivalent(RepeatedMask, {0, 0, 2, 2}, V1, V2))
16105 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
16106 if (isShuffleEquivalent(RepeatedMask, {1, 1, 3, 3}, V1, V2))
16107 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
16108
16109 if (V2.isUndef())
16110 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
16111 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16112
16113 // Use dedicated unpack instructions for masks that match their pattern.
16114 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, V1, V2, Mask, DAG))
16115 return V;
16116
16117 // Otherwise, fall back to a SHUFPS sequence. Here it is important that we
16118 // have already handled any direct blends.
16119 return lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
16120 }
16121
16122 // Try to create an in-lane repeating shuffle mask and then shuffle the
16123 // results into the target lanes.
16124 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16125 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16126 return V;
16127
16128 // If we have a single input shuffle with different shuffle patterns in the
16129 // two 128-bit lanes use the variable mask to VPERMILPS.
16130 if (V2.isUndef()) {
16131 if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask)) {
16132 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16133 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask);
16134 }
16135 if (Subtarget.hasAVX2()) {
16136 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16137 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1);
16138 }
16139 // Otherwise, fall back.
16140 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v8f32, V1, V2, Mask,
16141 DAG, Subtarget);
16142 }
16143
16144 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16145 // shuffle.
16146 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16147 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16148 return Result;
16149
16150 // If we have VLX support, we can use VEXPAND.
16151 if (Subtarget.hasVLX())
16152 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, V1, V2,
16153 DAG, Subtarget))
16154 return V;
16155
16156 // Try to match an interleave of two v8f32s and lower them as unpck and
16157 // permutes using ymms. This needs to go before we try to split the vectors.
16158 //
16159 // TODO: Expand this to AVX1. Currently v8i32 is casted to v8f32 and hits
16160 // this path inadvertently.
16161 if (Subtarget.hasAVX2() && !Subtarget.hasAVX512())
16162 if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v8f32, V1, V2,
16163 Mask, DAG))
16164 return V;
16165
16166 // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
16167 // since after split we get a more efficient code using vpunpcklwd and
16168 // vpunpckhwd instrs than vblend.
16169 if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32, DAG))
16170 return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Zeroable,
16171 Subtarget, DAG);
16172
16173 // If we have AVX2 then we always want to lower with a blend because at v8 we
16174 // can fully permute the elements.
16175 if (Subtarget.hasAVX2())
16176 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8f32, V1, V2, Mask,
16177 Zeroable, Subtarget, DAG);
16178
16179 // Otherwise fall back on generic lowering.
16180 return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Zeroable,
16181 Subtarget, DAG);
16182}
16183
16184/// Handle lowering of 8-lane 32-bit integer shuffles.
16185///
16186/// This routine is only called when we have AVX2 and thus a reasonable
16187/// instruction set for v8i32 shuffling..
16188static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16189 const APInt &Zeroable, SDValue V1, SDValue V2,
16190 const X86Subtarget &Subtarget,
16191 SelectionDAG &DAG) {
16192 assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
16193 assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
16194 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16195 assert(Subtarget.hasAVX2() && "We can only lower v8i32 with AVX2!");
16196
16197 int NumV2Elements = count_if(Mask, [](int M) { return M >= 8; });
16198
16199 // Whenever we can lower this as a zext, that instruction is strictly faster
16200 // than any alternative. It also allows us to fold memory operands into the
16201 // shuffle in many cases.
16202 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
16203 Zeroable, Subtarget, DAG))
16204 return ZExt;
16205
16206 // Try to match an interleave of two v8i32s and lower them as unpck and
16207 // permutes using ymms. This needs to go before we try to split the vectors.
16208 if (!Subtarget.hasAVX512())
16209 if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v8i32, V1, V2,
16210 Mask, DAG))
16211 return V;
16212
16213 // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
16214 // since after split we get a more efficient code than vblend by using
16215 // vpunpcklwd and vpunpckhwd instrs.
16216 if (isUnpackWdShuffleMask(Mask, MVT::v8i32, DAG) && !V2.isUndef() &&
16217 !Subtarget.hasAVX512())
16218 return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Zeroable,
16219 Subtarget, DAG);
16220
16221 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
16222 Zeroable, Subtarget, DAG))
16223 return Blend;
16224
16225 // Check for being able to broadcast a single element.
16226 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
16227 Subtarget, DAG))
16228 return Broadcast;
16229
16230 // Try to use shift instructions if fast.
16231 if (Subtarget.preferLowerShuffleAsShift()) {
16232 if (SDValue Shift =
16233 lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, Zeroable,
16234 Subtarget, DAG, /*BitwiseOnly*/ true))
16235 return Shift;
16236 if (NumV2Elements == 0)
16237 if (SDValue Rotate =
16238 lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG))
16239 return Rotate;
16240 }
16241
16242 // If the shuffle mask is repeated in each 128-bit lane we can use more
16243 // efficient instructions that mirror the shuffles across the two 128-bit
16244 // lanes.
16245 SmallVector<int, 4> RepeatedMask;
16246 bool Is128BitLaneRepeatedShuffle =
16247 is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask);
16248 if (Is128BitLaneRepeatedShuffle) {
16249 assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
16250 if (V2.isUndef())
16251 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
16252 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16253
16254 // Use dedicated unpack instructions for masks that match their pattern.
16255 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, V1, V2, Mask, DAG))
16256 return V;
16257 }
16258
16259 // Try to use shift instructions.
16260 if (SDValue Shift =
16261 lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, Zeroable, Subtarget,
16262 DAG, /*BitwiseOnly*/ false))
16263 return Shift;
16264
16265 if (!Subtarget.preferLowerShuffleAsShift() && NumV2Elements == 0)
16266 if (SDValue Rotate =
16267 lowerShuffleAsBitRotate(DL, MVT::v8i32, V1, Mask, Subtarget, DAG))
16268 return Rotate;
16269
16270 // If we have VLX support, we can use VALIGN or EXPAND.
16271 if (Subtarget.hasVLX()) {
16272 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i32, V1, V2, Mask,
16273 Zeroable, Subtarget, DAG))
16274 return Rotate;
16275
16276 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, V1, V2,
16277 DAG, Subtarget))
16278 return V;
16279 }
16280
16281 // Try to use byte rotation instructions.
16282 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,
16283 Subtarget, DAG))
16284 return Rotate;
16285
16286 // Try to create an in-lane repeating shuffle mask and then shuffle the
16287 // results into the target lanes.
16288 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16289 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16290 return V;
16291
16292 if (V2.isUndef()) {
16293 // Try to produce a fixed cross-128-bit lane permute followed by unpack
16294 // because that should be faster than the variable permute alternatives.
16295 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, V1, V2, Mask, DAG))
16296 return V;
16297
16298 // If the shuffle patterns aren't repeated but it's a single input, directly
16299 // generate a cross-lane VPERMD instruction.
16300 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16301 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1);
16302 }
16303
16304 // Assume that a single SHUFPS is faster than an alternative sequence of
16305 // multiple instructions (even if the CPU has a domain penalty).
16306 // If some CPU is harmed by the domain switch, we can fix it in a later pass.
16307 if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
16308 SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);
16309 SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);
16310 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
16311 CastV1, CastV2, DAG);
16312 return DAG.getBitcast(MVT::v8i32, ShufPS);
16313 }
16314
16315 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16316 // shuffle.
16317 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16318 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16319 return Result;
16320
16321 // Otherwise fall back on generic blend lowering.
16322 return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i32, V1, V2, Mask,
16323 Zeroable, Subtarget, DAG);
16324}
16325
16326/// Handle lowering of 16-lane 16-bit integer shuffles.
16327///
16328/// This routine is only called when we have AVX2 and thus a reasonable
16329/// instruction set for v16i16 shuffling..
16330static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16331 const APInt &Zeroable, SDValue V1, SDValue V2,
16332 const X86Subtarget &Subtarget,
16333 SelectionDAG &DAG) {
16334 assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
16335 assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
16336 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
16337 assert(Subtarget.hasAVX2() && "We can only lower v16i16 with AVX2!");
16338
16339 // Whenever we can lower this as a zext, that instruction is strictly faster
16340 // than any alternative. It also allows us to fold memory operands into the
16341 // shuffle in many cases.
16342 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16343 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16344 return ZExt;
16345
16346 // Check for being able to broadcast a single element.
16347 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,
16348 Subtarget, DAG))
16349 return Broadcast;
16350
16351 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
16352 Zeroable, Subtarget, DAG))
16353 return Blend;
16354
16355 // Use dedicated unpack instructions for masks that match their pattern.
16356 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, V1, V2, Mask, DAG))
16357 return V;
16358
16359 // Use dedicated pack instructions for masks that match their pattern.
16360 if (SDValue V =
16361 lowerShuffleWithPACK(DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16362 return V;
16363
16364 // Try to use lower using a truncation.
16365 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16366 Subtarget, DAG))
16367 return V;
16368
16369 // Try to use shift instructions.
16370 if (SDValue Shift =
16371 lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16372 Subtarget, DAG, /*BitwiseOnly*/ false))
16373 return Shift;
16374
16375 // Try to use byte rotation instructions.
16376 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,
16377 Subtarget, DAG))
16378 return Rotate;
16379
16380 // Try to create an in-lane repeating shuffle mask and then shuffle the
16381 // results into the target lanes.
16382 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16383 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16384 return V;
16385
16386 if (V2.isUndef()) {
16387 // Try to use bit rotation instructions.
16388 if (SDValue Rotate =
16389 lowerShuffleAsBitRotate(DL, MVT::v16i16, V1, Mask, Subtarget, DAG))
16390 return Rotate;
16391
16392 // Try to produce a fixed cross-128-bit lane permute followed by unpack
16393 // because that should be faster than the variable permute alternatives.
16394 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask, DAG))
16395 return V;
16396
16397 // There are no generalized cross-lane shuffle operations available on i16
16398 // element types.
16399 if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) {
16400 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16401 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16402 return V;
16403
16404 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v16i16, V1, V2, Mask,
16405 DAG, Subtarget);
16406 }
16407
16408 SmallVector<int, 8> RepeatedMask;
16409 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
16410 // As this is a single-input shuffle, the repeated mask should be
16411 // a strictly valid v8i16 mask that we can pass through to the v8i16
16412 // lowering to handle even the v16 case.
16413 return lowerV8I16GeneralSingleInputShuffle(
16414 DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);
16415 }
16416 }
16417
16418 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,
16419 Zeroable, Subtarget, DAG))
16420 return PSHUFB;
16421
16422 // AVX512BW can lower to VPERMW (non-VLX will pad to v32i16).
16423 if (Subtarget.hasBWI())
16424 return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, Subtarget, DAG);
16425
16426 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16427 // shuffle.
16428 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16429 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16430 return Result;
16431
16432 // Try to permute the lanes and then use a per-lane permute.
16433 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16434 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16435 return V;
16436
16437 // Try to match an interleave of two v16i16s and lower them as unpck and
16438 // permutes using ymms.
16439 if (!Subtarget.hasAVX512())
16440 if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v16i16, V1, V2,
16441 Mask, DAG))
16442 return V;
16443
16444 // Otherwise fall back on generic lowering.
16445 return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
16446 Subtarget, DAG);
16447}
16448
16449/// Handle lowering of 32-lane 8-bit integer shuffles.
16450///
16451/// This routine is only called when we have AVX2 and thus a reasonable
16452/// instruction set for v32i8 shuffling..
16453static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16454 const APInt &Zeroable, SDValue V1, SDValue V2,
16455 const X86Subtarget &Subtarget,
16456 SelectionDAG &DAG) {
16457 assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
16458 assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
16459 assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
16460 assert(Subtarget.hasAVX2() && "We can only lower v32i8 with AVX2!");
16461
16462 // Whenever we can lower this as a zext, that instruction is strictly faster
16463 // than any alternative. It also allows us to fold memory operands into the
16464 // shuffle in many cases.
16465 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,
16466 Zeroable, Subtarget, DAG))
16467 return ZExt;
16468
16469 // Check for being able to broadcast a single element.
16470 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
16471 Subtarget, DAG))
16472 return Broadcast;
16473
16474 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
16475 Zeroable, Subtarget, DAG))
16476 return Blend;
16477
16478 // Use dedicated unpack instructions for masks that match their pattern.
16479 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, V1, V2, Mask, DAG))
16480 return V;
16481
16482 // Use dedicated pack instructions for masks that match their pattern.
16483 if (SDValue V =
16484 lowerShuffleWithPACK(DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16485 return V;
16486
16487 // Try to use lower using a truncation.
16488 if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v32i8, V1, V2, Mask, Zeroable,
16489 Subtarget, DAG))
16490 return V;
16491
16492 // Try to use shift instructions.
16493 if (SDValue Shift =
16494 lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, Zeroable, Subtarget,
16495 DAG, /*BitwiseOnly*/ false))
16496 return Shift;
16497
16498 // Try to use byte rotation instructions.
16499 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,
16500 Subtarget, DAG))
16501 return Rotate;
16502
16503 // Try to use bit rotation instructions.
16504 if (V2.isUndef())
16505 if (SDValue Rotate =
16506 lowerShuffleAsBitRotate(DL, MVT::v32i8, V1, Mask, Subtarget, DAG))
16507 return Rotate;
16508
16509 // Try to create an in-lane repeating shuffle mask and then shuffle the
16510 // results into the target lanes.
16511 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16512 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16513 return V;
16514
16515 // There are no generalized cross-lane shuffle operations available on i8
16516 // element types.
16517 if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) {
16518 // Try to produce a fixed cross-128-bit lane permute followed by unpack
16519 // because that should be faster than the variable permute alternatives.
16520 if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, V1, V2, Mask, DAG))
16521 return V;
16522
16523 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16524 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16525 return V;
16526
16527 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v32i8, V1, V2, Mask,
16528 DAG, Subtarget);
16529 }
16530
16531 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,
16532 Zeroable, Subtarget, DAG))
16533 return PSHUFB;
16534
16535 // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8).
16536 if (Subtarget.hasVBMI())
16537 return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, Subtarget, DAG);
16538
16539 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16540 // shuffle.
16541 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16542 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16543 return Result;
16544
16545 // Try to permute the lanes and then use a per-lane permute.
16546 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16547 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16548 return V;
16549
16550 // Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
16551 // by zeroable elements in the remaining 24 elements. Turn this into two
16552 // vmovqb instructions shuffled together.
16553 if (Subtarget.hasVLX())
16554 if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
16555 Mask, Zeroable, DAG))
16556 return V;
16557
16558 // Try to match an interleave of two v32i8s and lower them as unpck and
16559 // permutes using ymms.
16560 if (!Subtarget.hasAVX512())
16561 if (SDValue V = lowerShufflePairAsUNPCKAndPermute(DL, MVT::v32i8, V1, V2,
16562 Mask, DAG))
16563 return V;
16564
16565 // Otherwise fall back on generic lowering.
16566 return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, Zeroable,
16567 Subtarget, DAG);
16568}
16569
16570/// High-level routine to lower various 256-bit x86 vector shuffles.
16571///
16572/// This routine either breaks down the specific type of a 256-bit x86 vector
16573/// shuffle or splits it into two 128-bit shuffles and fuses the results back
16574/// together based on the available instructions.
16575static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
16576 SDValue V1, SDValue V2, const APInt &Zeroable,
16577 const X86Subtarget &Subtarget,
16578 SelectionDAG &DAG) {
16579 // If we have a single input to the zero element, insert that into V1 if we
16580 // can do so cheaply.
16581 int NumElts = VT.getVectorNumElements();
16582 int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
16583
16584 if (NumV2Elements == 1 && Mask[0] >= NumElts)
16585 if (SDValue Insertion = lowerShuffleAsElementInsertion(
16586 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
16587 return Insertion;
16588
16589 // Handle special cases where the lower or upper half is UNDEF.
16590 if (SDValue V =
16591 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
16592 return V;
16593
16594 // There is a really nice hard cut-over between AVX1 and AVX2 that means we
16595 // can check for those subtargets here and avoid much of the subtarget
16596 // querying in the per-vector-type lowering routines. With AVX1 we have
16597 // essentially *zero* ability to manipulate a 256-bit vector with integer
16598 // types. Since we'll use floating point types there eventually, just
16599 // immediately cast everything to a float and operate entirely in that domain.
16600 if (VT.isInteger() && !Subtarget.hasAVX2()) {
16601 int ElementBits = VT.getScalarSizeInBits();
16602 if (ElementBits < 32) {
16603 // No floating point type available, if we can't use the bit operations
16604 // for masking/blending then decompose into 128-bit vectors.
16605 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
16606 Subtarget, DAG))
16607 return V;
16608 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
16609 return V;
16610 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
16611 }
16612
16613 MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
16614 VT.getVectorNumElements());
16615 V1 = DAG.getBitcast(FpVT, V1);
16616 V2 = DAG.getBitcast(FpVT, V2);
16617 return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
16618 }
16619
16620 if (VT == MVT::v16f16 || VT == MVT::v16bf16) {
16621 V1 = DAG.getBitcast(MVT::v16i16, V1);
16622 V2 = DAG.getBitcast(MVT::v16i16, V2);
16623 return DAG.getBitcast(VT,
16624 DAG.getVectorShuffle(MVT::v16i16, DL, V1, V2, Mask));
16625 }
16626
16627 switch (VT.SimpleTy) {
16628 case MVT::v4f64:
16629 return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16630 case MVT::v4i64:
16631 return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16632 case MVT::v8f32:
16633 return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16634 case MVT::v8i32:
16635 return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16636 case MVT::v16i16:
16637 return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16638 case MVT::v32i8:
16639 return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16640
16641 default:
16642 llvm_unreachable("Not a valid 256-bit x86 vector type!");
16643 }
16644}
16645
16646/// Try to lower a vector shuffle as a 128-bit shuffles.
16647static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
16648 const APInt &Zeroable, SDValue V1, SDValue V2,
16649 const X86Subtarget &Subtarget,
16650 SelectionDAG &DAG) {
16651 assert(VT.getScalarSizeInBits() == 64 &&
16652 "Unexpected element type size for 128bit shuffle.");
16653
16654 // To handle 256 bit vector requires VLX and most probably
16655 // function lowerV2X128VectorShuffle() is better solution.
16656 assert(VT.is512BitVector() && "Unexpected vector size for 512bit shuffle.");
16657
16658 // TODO - use Zeroable like we do for lowerV2X128VectorShuffle?
16659 SmallVector<int, 4> Widened128Mask;
16660 if (!canWidenShuffleElements(Mask, Widened128Mask))
16661 return SDValue();
16662 assert(Widened128Mask.size() == 4 && "Shuffle widening mismatch");
16663
16664 // Try to use an insert into a zero vector.
16665 if (Widened128Mask[0] == 0 && (Zeroable & 0xf0) == 0xf0 &&
16666 (Widened128Mask[1] == 1 || (Zeroable & 0x0c) == 0x0c)) {
16667 unsigned NumElts = ((Zeroable & 0x0c) == 0x0c) ? 2 : 4;
16668 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
16669 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
16670 DAG.getIntPtrConstant(0, DL));
16671 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
16672 getZeroVector(VT, Subtarget, DAG, DL), LoV,
16673 DAG.getIntPtrConstant(0, DL));
16674 }
16675
16676 // Check for patterns which can be matched with a single insert of a 256-bit
16677 // subvector.
16678 bool OnlyUsesV1 = isShuffleEquivalent(Mask, {0, 1, 2, 3, 0, 1, 2, 3}, V1, V2);
16679 if (OnlyUsesV1 ||
16680 isShuffleEquivalent(Mask, {0, 1, 2, 3, 8, 9, 10, 11}, V1, V2)) {
16681 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4);
16682 SDValue SubVec =
16683 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2,
16684 DAG.getIntPtrConstant(0, DL));
16685 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
16686 DAG.getIntPtrConstant(4, DL));
16687 }
16688
16689 // See if this is an insertion of the lower 128-bits of V2 into V1.
16690 bool IsInsert = true;
16691 int V2Index = -1;
16692 for (int i = 0; i < 4; ++i) {
16693 assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value");
16694 if (Widened128Mask[i] < 0)
16695 continue;
16696
16697 // Make sure all V1 subvectors are in place.
16698 if (Widened128Mask[i] < 4) {
16699 if (Widened128Mask[i] != i) {
16700 IsInsert = false;
16701 break;
16702 }
16703 } else {
16704 // Make sure we only have a single V2 index and its the lowest 128-bits.
16705 if (V2Index >= 0 || Widened128Mask[i] != 4) {
16706 IsInsert = false;
16707 break;
16708 }
16709 V2Index = i;
16710 }
16711 }
16712 if (IsInsert && V2Index >= 0) {
16713 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
16714 SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
16715 DAG.getIntPtrConstant(0, DL));
16716 return insert128BitVector(V1, Subvec, V2Index * 2, DAG, DL);
16717 }
16718
16719 // See if we can widen to a 256-bit lane shuffle, we're going to lose 128-lane
16720 // UNDEF info by lowering to X86ISD::SHUF128 anyway, so by widening where
16721 // possible we at least ensure the lanes stay sequential to help later
16722 // combines.
16723 SmallVector<int, 2> Widened256Mask;
16724 if (canWidenShuffleElements(Widened128Mask, Widened256Mask)) {
16725 Widened128Mask.clear();
16726 narrowShuffleMaskElts(2, Widened256Mask, Widened128Mask);
16727 }
16728
16729 // Try to lower to vshuf64x2/vshuf32x4.
16730 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
16731 int PermMask[4] = {-1, -1, -1, -1};
16732 // Ensure elements came from the same Op.
16733 for (int i = 0; i < 4; ++i) {
16734 assert(Widened128Mask[i] >= -1 && "Illegal shuffle sentinel value");
16735 if (Widened128Mask[i] < 0)
16736 continue;
16737
16738 SDValue Op = Widened128Mask[i] >= 4 ? V2 : V1;
16739 unsigned OpIndex = i / 2;
16740 if (Ops[OpIndex].isUndef())
16741 Ops[OpIndex] = Op;
16742 else if (Ops[OpIndex] != Op)
16743 return SDValue();
16744
16745 PermMask[i] = Widened128Mask[i] % 4;
16746 }
16747
16748 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
16749 getV4X86ShuffleImm8ForMask(PermMask, DL, DAG));
16750}
16751
16752/// Handle lowering of 8-lane 64-bit floating point shuffles.
16753static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16754 const APInt &Zeroable, SDValue V1, SDValue V2,
16755 const X86Subtarget &Subtarget,
16756 SelectionDAG &DAG) {
16757 assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
16758 assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
16759 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16760
16761 if (V2.isUndef()) {
16762 // Use low duplicate instructions for masks that match their pattern.
16763 if (isShuffleEquivalent(Mask, {0, 0, 2, 2, 4, 4, 6, 6}, V1, V2))
16764 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1);
16765
16766 if (!is128BitLaneCrossingShuffleMask(MVT::v8f64, Mask)) {
16767 // Non-half-crossing single input shuffles can be lowered with an
16768 // interleaved permutation.
16769 unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
16770 ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3) |
16771 ((Mask[4] == 5) << 4) | ((Mask[5] == 5) << 5) |
16772 ((Mask[6] == 7) << 6) | ((Mask[7] == 7) << 7);
16773 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,
16774 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
16775 }
16776
16777 SmallVector<int, 4> RepeatedMask;
16778 if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask))
16779 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1,
16780 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16781 }
16782
16783 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
16784 V2, Subtarget, DAG))
16785 return Shuf128;
16786
16787 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, V1, V2, Mask, DAG))
16788 return Unpck;
16789
16790 // Check if the blend happens to exactly fit that of SHUFPD.
16791 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,
16792 Zeroable, Subtarget, DAG))
16793 return Op;
16794
16795 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,
16796 DAG, Subtarget))
16797 return V;
16798
16799 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
16800 Zeroable, Subtarget, DAG))
16801 return Blend;
16802
16803 return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, Subtarget, DAG);
16804}
16805
16806/// Handle lowering of 16-lane 32-bit floating point shuffles.
16807static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16808 const APInt &Zeroable, SDValue V1, SDValue V2,
16809 const X86Subtarget &Subtarget,
16810 SelectionDAG &DAG) {
16811 assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
16812 assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
16813 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
16814
16815 // If the shuffle mask is repeated in each 128-bit lane, we have many more
16816 // options to efficiently lower the shuffle.
16817 SmallVector<int, 4> RepeatedMask;
16818 if (is128BitLaneRepeatedShuffleMask(MVT::v16f32, Mask, RepeatedMask)) {
16819 assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
16820
16821 // Use even/odd duplicate instructions for masks that match their pattern.
16822 if (isShuffleEquivalent(RepeatedMask, {0, 0, 2, 2}, V1, V2))
16823 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1);
16824 if (isShuffleEquivalent(RepeatedMask, {1, 1, 3, 3}, V1, V2))
16825 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1);
16826
16827 if (V2.isUndef())
16828 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,
16829 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16830
16831 // Use dedicated unpack instructions for masks that match their pattern.
16832 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, V1, V2, Mask, DAG))
16833 return V;
16834
16835 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
16836 Zeroable, Subtarget, DAG))
16837 return Blend;
16838
16839 // Otherwise, fall back to a SHUFPS sequence.
16840 return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
16841 }
16842
16843 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
16844 Zeroable, Subtarget, DAG))
16845 return Blend;
16846
16847 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16848 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
16849 return DAG.getBitcast(MVT::v16f32, ZExt);
16850
16851 // Try to create an in-lane repeating shuffle mask and then shuffle the
16852 // results into the target lanes.
16853 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16854 DL, MVT::v16f32, V1, V2, Mask, Subtarget, DAG))
16855 return V;
16856
16857 // If we have a single input shuffle with different shuffle patterns in the
16858 // 128-bit lanes and don't lane cross, use variable mask VPERMILPS.
16859 if (V2.isUndef() &&
16860 !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) {
16861 SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
16862 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);
16863 }
16864
16865 // If we have AVX512F support, we can use VEXPAND.
16866 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
16867 V1, V2, DAG, Subtarget))
16868 return V;
16869
16870 return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, Subtarget, DAG);
16871}
16872
16873/// Handle lowering of 8-lane 64-bit integer shuffles.
16874static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16875 const APInt &Zeroable, SDValue V1, SDValue V2,
16876 const X86Subtarget &Subtarget,
16877 SelectionDAG &DAG) {
16878 assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
16879 assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
16880 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
16881
16882 // Try to use shift instructions if fast.
16883 if (Subtarget.preferLowerShuffleAsShift())
16884 if (SDValue Shift =
16885 lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask, Zeroable,
16886 Subtarget, DAG, /*BitwiseOnly*/ true))
16887 return Shift;
16888
16889 if (V2.isUndef()) {
16890 // When the shuffle is mirrored between the 128-bit lanes of the unit, we
16891 // can use lower latency instructions that will operate on all four
16892 // 128-bit lanes.
16893 SmallVector<int, 2> Repeated128Mask;
16894 if (is128BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated128Mask)) {
16895 SmallVector<int, 4> PSHUFDMask;
16896 narrowShuffleMaskElts(2, Repeated128Mask, PSHUFDMask);
16897 return DAG.getBitcast(
16898 MVT::v8i64,
16899 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,
16900 DAG.getBitcast(MVT::v16i32, V1),
16901 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
16902 }
16903
16904 SmallVector<int, 4> Repeated256Mask;
16905 if (is256BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated256Mask))
16906 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1,
16907 getV4X86ShuffleImm8ForMask(Repeated256Mask, DL, DAG));
16908 }
16909
16910 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
16911 V2, Subtarget, DAG))
16912 return Shuf128;
16913
16914 // Try to use shift instructions.
16915 if (SDValue Shift =
16916 lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask, Zeroable, Subtarget,
16917 DAG, /*BitwiseOnly*/ false))
16918 return Shift;
16919
16920 // Try to use VALIGN.
16921 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i64, V1, V2, Mask,
16922 Zeroable, Subtarget, DAG))
16923 return Rotate;
16924
16925 // Try to use PALIGNR.
16926 if (Subtarget.hasBWI())
16927 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
16928 Subtarget, DAG))
16929 return Rotate;
16930
16931 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, V1, V2, Mask, DAG))
16932 return Unpck;
16933
16934 // If we have AVX512F support, we can use VEXPAND.
16935 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,
16936 DAG, Subtarget))
16937 return V;
16938
16939 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
16940 Zeroable, Subtarget, DAG))
16941 return Blend;
16942
16943 return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, Subtarget, DAG);
16944}
16945
16946/// Handle lowering of 16-lane 32-bit integer shuffles.
16947static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16948 const APInt &Zeroable, SDValue V1, SDValue V2,
16949 const X86Subtarget &Subtarget,
16950 SelectionDAG &DAG) {
16951 assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
16952 assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
16953 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
16954
16955 int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
16956
16957 // Whenever we can lower this as a zext, that instruction is strictly faster
16958 // than any alternative. It also allows us to fold memory operands into the
16959 // shuffle in many cases.
16960 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16961 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
16962 return ZExt;
16963
16964 // Try to use shift instructions if fast.
16965 if (Subtarget.preferLowerShuffleAsShift()) {
16966 if (SDValue Shift =
16967 lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask, Zeroable,
16968 Subtarget, DAG, /*BitwiseOnly*/ true))
16969 return Shift;
16970 if (NumV2Elements == 0)
16971 if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i32, V1, Mask,
16972 Subtarget, DAG))
16973 return Rotate;
16974 }
16975
16976 // If the shuffle mask is repeated in each 128-bit lane we can use more
16977 // efficient instructions that mirror the shuffles across the four 128-bit
16978 // lanes.
16979 SmallVector<int, 4> RepeatedMask;
16980 bool Is128BitLaneRepeatedShuffle =
16981 is128BitLaneRepeatedShuffleMask(MVT::v16i32, Mask, RepeatedMask);
16982 if (Is128BitLaneRepeatedShuffle) {
16983 assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
16984 if (V2.isUndef())
16985 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1,
16986 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16987
16988 // Use dedicated unpack instructions for masks that match their pattern.
16989 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, V1, V2, Mask, DAG))
16990 return V;
16991 }
16992
16993 // Try to use shift instructions.
16994 if (SDValue Shift =
16995 lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask, Zeroable,
16996 Subtarget, DAG, /*BitwiseOnly*/ false))
16997 return Shift;
16998
16999 if (!Subtarget.preferLowerShuffleAsShift() && NumV2Elements != 0)
17000 if (SDValue Rotate =
17001 lowerShuffleAsBitRotate(DL, MVT::v16i32, V1, Mask, Subtarget, DAG))
17002 return Rotate;
17003
17004 // Try to use VALIGN.
17005 if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v16i32, V1, V2, Mask,
17006 Zeroable, Subtarget, DAG))
17007 return Rotate;
17008
17009 // Try to use byte rotation instructions.
17010 if (Subtarget.hasBWI())
17011 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,
17012 Subtarget, DAG))
17013 return Rotate;
17014
17015 // Assume that a single SHUFPS is faster than using a permv shuffle.
17016 // If some CPU is harmed by the domain switch, we can fix it in a later pass.
17017 if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
17018 SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);
17019 SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);
17020 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
17021 CastV1, CastV2, DAG);
17022 return DAG.getBitcast(MVT::v16i32, ShufPS);
17023 }
17024
17025 // Try to create an in-lane repeating shuffle mask and then shuffle the
17026 // results into the target lanes.
17027 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17028 DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG))
17029 return V;
17030
17031 // If we have AVX512F support, we can use VEXPAND.
17032 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,
17033 DAG, Subtarget))
17034 return V;
17035
17036 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
17037 Zeroable, Subtarget, DAG))
17038 return Blend;
17039
17040 return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, Subtarget, DAG);
17041}
17042
17043/// Handle lowering of 32-lane 16-bit integer shuffles.
17044static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17045 const APInt &Zeroable, SDValue V1, SDValue V2,
17046 const X86Subtarget &Subtarget,
17047 SelectionDAG &DAG) {
17048 assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
17049 assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
17050 assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
17051 assert(Subtarget.hasBWI() && "We can only lower v32i16 with AVX-512-BWI!");
17052
17053 // Whenever we can lower this as a zext, that instruction is strictly faster
17054 // than any alternative. It also allows us to fold memory operands into the
17055 // shuffle in many cases.
17056 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
17057 DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
17058 return ZExt;
17059
17060 // Use dedicated unpack instructions for masks that match their pattern.
17061 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, V1, V2, Mask, DAG))
17062 return V;
17063
17064 // Use dedicated pack instructions for masks that match their pattern.
17065 if (SDValue V =
17066 lowerShuffleWithPACK(DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG))
17067 return V;
17068
17069 // Try to use shift instructions.
17070 if (SDValue Shift =
17071 lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask, Zeroable,
17072 Subtarget, DAG, /*BitwiseOnly*/ false))
17073 return Shift;
17074
17075 // Try to use byte rotation instructions.
17076 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,
17077 Subtarget, DAG))
17078 return Rotate;
17079
17080 if (V2.isUndef()) {
17081 // Try to use bit rotation instructions.
17082 if (SDValue Rotate =
17083 lowerShuffleAsBitRotate(DL, MVT::v32i16, V1, Mask, Subtarget, DAG))
17084 return Rotate;
17085
17086 SmallVector<int, 8> RepeatedMask;
17087 if (is128BitLaneRepeatedShuffleMask(MVT::v32i16, Mask, RepeatedMask)) {
17088 // As this is a single-input shuffle, the repeated mask should be
17089 // a strictly valid v8i16 mask that we can pass through to the v8i16
17090 // lowering to handle even the v32 case.
17091 return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v32i16, V1,
17092 RepeatedMask, Subtarget, DAG);
17093 }
17094 }
17095
17096 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
17097 Zeroable, Subtarget, DAG))
17098 return Blend;
17099
17100 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
17101 Zeroable, Subtarget, DAG))
17102 return PSHUFB;
17103
17104 return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG);
17105}
17106
17107/// Handle lowering of 64-lane 8-bit integer shuffles.
17108static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
17109 const APInt &Zeroable, SDValue V1, SDValue V2,
17110 const X86Subtarget &Subtarget,
17111 SelectionDAG &DAG) {
17112 assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
17113 assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
17114 assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!");
17115 assert(Subtarget.hasBWI() && "We can only lower v64i8 with AVX-512-BWI!");
17116
17117 // Whenever we can lower this as a zext, that instruction is strictly faster
17118 // than any alternative. It also allows us to fold memory operands into the
17119 // shuffle in many cases.
17120 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
17121 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
17122 return ZExt;
17123
17124 // Use dedicated unpack instructions for masks that match their pattern.
17125 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, V1, V2, Mask, DAG))
17126 return V;
17127
17128 // Use dedicated pack instructions for masks that match their pattern.
17129 if (SDValue V =
17130 lowerShuffleWithPACK(DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17131 return V;
17132
17133 // Try to use shift instructions.
17134 if (SDValue Shift =
17135 lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget,
17136 DAG, /*BitwiseOnly*/ false))
17137 return Shift;
17138
17139 // Try to use byte rotation instructions.
17140 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,
17141 Subtarget, DAG))
17142 return Rotate;
17143
17144 // Try to use bit rotation instructions.
17145 if (V2.isUndef())
17146 if (SDValue Rotate =
17147 lowerShuffleAsBitRotate(DL, MVT::v64i8, V1, Mask, Subtarget, DAG))
17148 return Rotate;
17149
17150 // Lower as AND if possible.
17151 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v64i8, V1, V2, Mask,
17152 Zeroable, Subtarget, DAG))
17153 return Masked;
17154
17155 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
17156 Zeroable, Subtarget, DAG))
17157 return PSHUFB;
17158
17159 // Try to create an in-lane repeating shuffle mask and then shuffle the
17160 // results into the target lanes.
17161 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
17162 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17163 return V;
17164
17165 if (SDValue Result = lowerShuffleAsLanePermuteAndPermute(
17166 DL, MVT::v64i8, V1, V2, Mask, DAG, Subtarget))
17167 return Result;
17168
17169 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
17170 Zeroable, Subtarget, DAG))
17171 return Blend;
17172
17173 if (!is128BitLaneCrossingShuffleMask(MVT::v64i8, Mask)) {
17174 // Use PALIGNR+Permute if possible - permute might become PSHUFB but the
17175 // PALIGNR will be cheaper than the second PSHUFB+OR.
17176 if (SDValue V = lowerShuffleAsByteRotateAndPermute(DL, MVT::v64i8, V1, V2,
17177 Mask, Subtarget, DAG))
17178 return V;
17179
17180 // If we can't directly blend but can use PSHUFB, that will be better as it
17181 // can both shuffle and set up the inefficient blend.
17182 bool V1InUse, V2InUse;
17183 return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v64i8, V1, V2, Mask, Zeroable,
17184 DAG, V1InUse, V2InUse);
17185 }
17186
17187 // Try to simplify this by merging 128-bit lanes to enable a lane-based
17188 // shuffle.
17189 if (!V2.isUndef())
17190 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
17191 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
17192 return Result;
17193
17194 // VBMI can use VPERMV/VPERMV3 byte shuffles.
17195 if (Subtarget.hasVBMI())
17196 return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, Subtarget, DAG);
17197
17198 return splitAndLowerShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
17199}
17200
17201/// High-level routine to lower various 512-bit x86 vector shuffles.
17202///
17203/// This routine either breaks down the specific type of a 512-bit x86 vector
17204/// shuffle or splits it into two 256-bit shuffles and fuses the results back
17205/// together based on the available instructions.
17206static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17207 MVT VT, SDValue V1, SDValue V2,
17208 const APInt &Zeroable,
17209 const X86Subtarget &Subtarget,
17210 SelectionDAG &DAG) {
17211 assert(Subtarget.hasAVX512() &&
17212 "Cannot lower 512-bit vectors w/ basic ISA!");
17213
17214 // If we have a single input to the zero element, insert that into V1 if we
17215 // can do so cheaply.
17216 int NumElts = Mask.size();
17217 int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
17218
17219 if (NumV2Elements == 1 && Mask[0] >= NumElts)
17220 if (SDValue Insertion = lowerShuffleAsElementInsertion(
17221 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
17222 return Insertion;
17223
17224 // Handle special cases where the lower or upper half is UNDEF.
17225 if (SDValue V =
17226 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
17227 return V;
17228
17229 // Check for being able to broadcast a single element.
17230 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
17231 Subtarget, DAG))
17232 return Broadcast;
17233
17234 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) {
17235 // Try using bit ops for masking and blending before falling back to
17236 // splitting.
17237 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
17238 Subtarget, DAG))
17239 return V;
17240 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
17241 return V;
17242
17243 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false);
17244 }
17245
17246 if (VT == MVT::v32f16 || VT == MVT::v32bf16) {
17247 if (!Subtarget.hasBWI())
17248 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG,
17249 /*SimpleOnly*/ false);
17250
17251 V1 = DAG.getBitcast(MVT::v32i16, V1);
17252 V2 = DAG.getBitcast(MVT::v32i16, V2);
17253 return DAG.getBitcast(VT,
17254 DAG.getVectorShuffle(MVT::v32i16, DL, V1, V2, Mask));
17255 }
17256
17257 // Dispatch to each element type for lowering. If we don't have support for
17258 // specific element type shuffles at 512 bits, immediately split them and
17259 // lower them. Each lowering routine of a given type is allowed to assume that
17260 // the requisite ISA extensions for that element type are available.
17261 switch (VT.SimpleTy) {
17262 case MVT::v8f64:
17263 return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17264 case MVT::v16f32:
17265 return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17266 case MVT::v8i64:
17267 return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17268 case MVT::v16i32:
17269 return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17270 case MVT::v32i16:
17271 return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17272 case MVT::v64i8:
17273 return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
17274
17275 default:
17276 llvm_unreachable("Not a valid 512-bit x86 vector type!");
17277 }
17278}
17279
17280static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef<int> Mask,
17281 MVT VT, SDValue V1, SDValue V2,
17282 const X86Subtarget &Subtarget,
17283 SelectionDAG &DAG) {
17284 // Shuffle should be unary.
17285 if (!V2.isUndef())
17286 return SDValue();
17287
17288 int ShiftAmt = -1;
17289 int NumElts = Mask.size();
17290 for (int i = 0; i != NumElts; ++i) {
17291 int M = Mask[i];
17292 assert((M == SM_SentinelUndef || (0 <= M && M < NumElts)) &&
17293 "Unexpected mask index.");
17294 if (M < 0)
17295 continue;
17296
17297 // The first non-undef element determines our shift amount.
17298 if (ShiftAmt < 0) {
17299 ShiftAmt = M - i;
17300 // Need to be shifting right.
17301 if (ShiftAmt <= 0)
17302 return SDValue();
17303 }
17304 // All non-undef elements must shift by the same amount.
17305 if (ShiftAmt != M - i)
17306 return SDValue();
17307 }
17308 assert(ShiftAmt >= 0 && "All undef?");
17309
17310 // Great we found a shift right.
17311 SDValue Res = widenMaskVector(V1, false, Subtarget, DAG, DL);
17312 Res = DAG.getNode(X86ISD::KSHIFTR, DL, Res.getValueType(), Res,
17313 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17314 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17315 DAG.getIntPtrConstant(0, DL));
17316}
17317
17318// Determine if this shuffle can be implemented with a KSHIFT instruction.
17319// Returns the shift amount if possible or -1 if not. This is a simplified
17320// version of matchShuffleAsShift.
17321static int match1BitShuffleAsKSHIFT(unsigned &Opcode, ArrayRef<int> Mask,
17322 int MaskOffset, const APInt &Zeroable) {
17323 int Size = Mask.size();
17324
17325 auto CheckZeros = [&](int Shift, bool Left) {
17326 for (int j = 0; j < Shift; ++j)
17327 if (!Zeroable[j + (Left ? 0 : (Size - Shift))])
17328 return false;
17329
17330 return true;
17331 };
17332
17333 auto MatchShift = [&](int Shift, bool Left) {
17334 unsigned Pos = Left ? Shift : 0;
17335 unsigned Low = Left ? 0 : Shift;
17336 unsigned Len = Size - Shift;
17337 return isSequentialOrUndefInRange(Mask, Pos, Len, Low + MaskOffset);
17338 };
17339
17340 for (int Shift = 1; Shift != Size; ++Shift)
17341 for (bool Left : {true, false})
17342 if (CheckZeros(Shift, Left) && MatchShift(Shift, Left)) {
17343 Opcode = Left ? X86ISD::KSHIFTL : X86ISD::KSHIFTR;
17344 return Shift;
17345 }
17346
17347 return -1;
17348}
17349
17350
17351// Lower vXi1 vector shuffles.
17352// There is no a dedicated instruction on AVX-512 that shuffles the masks.
17353// The only way to shuffle bits is to sign-extend the mask vector to SIMD
17354// vector, shuffle and then truncate it back.
17355static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17356 MVT VT, SDValue V1, SDValue V2,
17357 const APInt &Zeroable,
17358 const X86Subtarget &Subtarget,
17359 SelectionDAG &DAG) {
17360 assert(Subtarget.hasAVX512() &&
17361 "Cannot lower 512-bit vectors w/o basic ISA!");
17362
17363 int NumElts = Mask.size();
17364 int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
17365
17366 // Try to recognize shuffles that are just padding a subvector with zeros.
17367 int SubvecElts = 0;
17368 int Src = -1;
17369 for (int i = 0; i != NumElts; ++i) {
17370 if (Mask[i] >= 0) {
17371 // Grab the source from the first valid mask. All subsequent elements need
17372 // to use this same source.
17373 if (Src < 0)
17374 Src = Mask[i] / NumElts;
17375 if (Src != (Mask[i] / NumElts) || (Mask[i] % NumElts) != i)
17376 break;
17377 }
17378
17379 ++SubvecElts;
17380 }
17381 assert(SubvecElts != NumElts && "Identity shuffle?");
17382
17383 // Clip to a power 2.
17384 SubvecElts = llvm::bit_floor<uint32_t>(SubvecElts);
17385
17386 // Make sure the number of zeroable bits in the top at least covers the bits
17387 // not covered by the subvector.
17388 if ((int)Zeroable.countl_one() >= (NumElts - SubvecElts)) {
17389 assert(Src >= 0 && "Expected a source!");
17390 MVT ExtractVT = MVT::getVectorVT(MVT::i1, SubvecElts);
17391 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT,
17392 Src == 0 ? V1 : V2,
17393 DAG.getIntPtrConstant(0, DL));
17394 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
17395 DAG.getConstant(0, DL, VT),
17396 Extract, DAG.getIntPtrConstant(0, DL));
17397 }
17398
17399 // Try a simple shift right with undef elements. Later we'll try with zeros.
17400 if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget,
17401 DAG))
17402 return Shift;
17403
17404 // Try to match KSHIFTs.
17405 unsigned Offset = 0;
17406 for (SDValue V : { V1, V2 }) {
17407 unsigned Opcode;
17408 int ShiftAmt = match1BitShuffleAsKSHIFT(Opcode, Mask, Offset, Zeroable);
17409 if (ShiftAmt >= 0) {
17410 SDValue Res = widenMaskVector(V, false, Subtarget, DAG, DL);
17411 MVT WideVT = Res.getSimpleValueType();
17412 // Widened right shifts need two shifts to ensure we shift in zeroes.
17413 if (Opcode == X86ISD::KSHIFTR && WideVT != VT) {
17414 int WideElts = WideVT.getVectorNumElements();
17415 // Shift left to put the original vector in the MSBs of the new size.
17416 Res = DAG.getNode(X86ISD::KSHIFTL, DL, WideVT, Res,
17417 DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8));
17418 // Increase the shift amount to account for the left shift.
17419 ShiftAmt += WideElts - NumElts;
17420 }
17421
17422 Res = DAG.getNode(Opcode, DL, WideVT, Res,
17423 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17424 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17425 DAG.getIntPtrConstant(0, DL));
17426 }
17427 Offset += NumElts; // Increment for next iteration.
17428 }
17429
17430 // If we're performing an unary shuffle on a SETCC result, try to shuffle the
17431 // ops instead.
17432 // TODO: What other unary shuffles would benefit from this?
17433 if (NumV2Elements == 0 && V1.getOpcode() == ISD::SETCC && V1->hasOneUse()) {
17434 SDValue Op0 = V1.getOperand(0);
17435 SDValue Op1 = V1.getOperand(1);
17436 ISD::CondCode CC = cast<CondCodeSDNode>(V1.getOperand(2))->get();
17437 EVT OpVT = Op0.getValueType();
17438 if (OpVT.getScalarSizeInBits() >= 32 || isBroadcastShuffleMask(Mask))
17439 return DAG.getSetCC(
17440 DL, VT, DAG.getVectorShuffle(OpVT, DL, Op0, DAG.getUNDEF(OpVT), Mask),
17441 DAG.getVectorShuffle(OpVT, DL, Op1, DAG.getUNDEF(OpVT), Mask), CC);
17442 }
17443
17444 MVT ExtVT;
17445 switch (VT.SimpleTy) {
17446 default:
17447 llvm_unreachable("Expected a vector of i1 elements");
17448 case MVT::v2i1:
17449 ExtVT = MVT::v2i64;
17450 break;
17451 case MVT::v4i1:
17452 ExtVT = MVT::v4i32;
17453 break;
17454 case MVT::v8i1:
17455 // Take 512-bit type, more shuffles on KNL. If we have VLX use a 256-bit
17456 // shuffle.
17457 ExtVT = Subtarget.hasVLX() ? MVT::v8i32 : MVT::v8i64;
17458 break;
17459 case MVT::v16i1:
17460 // Take 512-bit type, unless we are avoiding 512-bit types and have the
17461 // 256-bit operation available.
17462 ExtVT = Subtarget.canExtendTo512DQ() ? MVT::v16i32 : MVT::v16i16;
17463 break;
17464 case MVT::v32i1:
17465 // Take 512-bit type, unless we are avoiding 512-bit types and have the
17466 // 256-bit operation available.
17467 assert(Subtarget.hasBWI() && "Expected AVX512BW support");
17468 ExtVT = Subtarget.canExtendTo512BW() ? MVT::v32i16 : MVT::v32i8;
17469 break;
17470 case MVT::v64i1:
17471 // Fall back to scalarization. FIXME: We can do better if the shuffle
17472 // can be partitioned cleanly.
17473 if (!Subtarget.useBWIRegs())
17474 return SDValue();
17475 ExtVT = MVT::v64i8;
17476 break;
17477 }
17478
17479 V1 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V1);
17480 V2 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V2);
17481
17482 SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask);
17483 // i1 was sign extended we can use X86ISD::CVT2MASK.
17484 int NumElems = VT.getVectorNumElements();
17485 if ((Subtarget.hasBWI() && (NumElems >= 32)) ||
17486 (Subtarget.hasDQI() && (NumElems < 32)))
17487 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT),
17488 Shuffle, ISD::SETGT);
17489
17490 return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle);
17491}
17492
17493/// Helper function that returns true if the shuffle mask should be
17494/// commuted to improve canonicalization.
17495static bool canonicalizeShuffleMaskWithCommute(ArrayRef<int> Mask) {
17496 int NumElements = Mask.size();
17497
17498 int NumV1Elements = 0, NumV2Elements = 0;
17499 for (int M : Mask)
17500 if (M < 0)
17501 continue;
17502 else if (M < NumElements)
17503 ++NumV1Elements;
17504 else
17505 ++NumV2Elements;
17506
17507 // Commute the shuffle as needed such that more elements come from V1 than
17508 // V2. This allows us to match the shuffle pattern strictly on how many
17509 // elements come from V1 without handling the symmetric cases.
17510 if (NumV2Elements > NumV1Elements)
17511 return true;
17512
17513 assert(NumV1Elements > 0 && "No V1 indices");
17514
17515 if (NumV2Elements == 0)
17516 return false;
17517
17518 // When the number of V1 and V2 elements are the same, try to minimize the
17519 // number of uses of V2 in the low half of the vector. When that is tied,
17520 // ensure that the sum of indices for V1 is equal to or lower than the sum
17521 // indices for V2. When those are equal, try to ensure that the number of odd
17522 // indices for V1 is lower than the number of odd indices for V2.
17523 if (NumV1Elements == NumV2Elements) {
17524 int LowV1Elements = 0, LowV2Elements = 0;
17525 for (int M : Mask.slice(0, NumElements / 2))
17526 if (M >= NumElements)
17527 ++LowV2Elements;
17528 else if (M >= 0)
17529 ++LowV1Elements;
17530 if (LowV2Elements > LowV1Elements)
17531 return true;
17532 if (LowV2Elements == LowV1Elements) {
17533 int SumV1Indices = 0, SumV2Indices = 0;
17534 for (int i = 0, Size = Mask.size(); i < Size; ++i)
17535 if (Mask[i] >= NumElements)
17536 SumV2Indices += i;
17537 else if (Mask[i] >= 0)
17538 SumV1Indices += i;
17539 if (SumV2Indices < SumV1Indices)
17540 return true;
17541 if (SumV2Indices == SumV1Indices) {
17542 int NumV1OddIndices = 0, NumV2OddIndices = 0;
17543 for (int i = 0, Size = Mask.size(); i < Size; ++i)
17544 if (Mask[i] >= NumElements)
17545 NumV2OddIndices += i % 2;
17546 else if (Mask[i] >= 0)
17547 NumV1OddIndices += i % 2;
17548 if (NumV2OddIndices < NumV1OddIndices)
17549 return true;
17550 }
17551 }
17552 }
17553
17554 return false;
17555}
17556
17557static bool canCombineAsMaskOperation(SDValue V,
17558 const X86Subtarget &Subtarget) {
17559 if (!Subtarget.hasAVX512())
17560 return false;
17561
17562 if (!V.getValueType().isSimple())
17563 return false;
17564
17565 MVT VT = V.getSimpleValueType().getScalarType();
17566 if ((VT == MVT::i16 || VT == MVT::i8) && !Subtarget.hasBWI())
17567 return false;
17568
17569 // If vec width < 512, widen i8/i16 even with BWI as blendd/blendps/blendpd
17570 // are preferable to blendw/blendvb/masked-mov.
17571 if ((VT == MVT::i16 || VT == MVT::i8) &&
17572 V.getSimpleValueType().getSizeInBits() < 512)
17573 return false;
17574
17575 auto HasMaskOperation = [&](SDValue V) {
17576 // TODO: Currently we only check limited opcode. We probably extend
17577 // it to all binary operation by checking TLI.isBinOp().
17578 switch (V->getOpcode()) {
17579 default:
17580 return false;
17581 case ISD::ADD:
17582 case ISD::SUB:
17583 case ISD::AND:
17584 case ISD::XOR:
17585 case ISD::OR:
17586 case ISD::SMAX:
17587 case ISD::SMIN:
17588 case ISD::UMAX:
17589 case ISD::UMIN:
17590 case ISD::ABS:
17591 case ISD::SHL:
17592 case ISD::SRL:
17593 case ISD::SRA:
17594 case ISD::MUL:
17595 break;
17596 }
17597 if (!V->hasOneUse())
17598 return false;
17599
17600 return true;
17601 };
17602
17603 if (HasMaskOperation(V))
17604 return true;
17605
17606 return false;
17607}
17608
17609// Forward declaration.
17610static SDValue canonicalizeShuffleMaskWithHorizOp(
17611 MutableArrayRef<SDValue> Ops, MutableArrayRef<int> Mask,
17612 unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
17613 const X86Subtarget &Subtarget);
17614
17615 /// Top-level lowering for x86 vector shuffles.
17616///
17617/// This handles decomposition, canonicalization, and lowering of all x86
17618/// vector shuffles. Most of the specific lowering strategies are encapsulated
17619/// above in helper routines. The canonicalization attempts to widen shuffles
17620/// to involve fewer lanes of wider elements, consolidate symmetric patterns
17621/// s.t. only one of the two inputs needs to be tested, etc.
17622static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
17623 SelectionDAG &DAG) {
17624 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
17625 ArrayRef<int> OrigMask = SVOp->getMask();
17626 SDValue V1 = Op.getOperand(0);
17627 SDValue V2 = Op.getOperand(1);
17628 MVT VT = Op.getSimpleValueType();
17629 int NumElements = VT.getVectorNumElements();
17630 SDLoc DL(Op);
17631 bool Is1BitVector = (VT.getVectorElementType() == MVT::i1);
17632
17633 assert((VT.getSizeInBits() != 64 || Is1BitVector) &&
17634 "Can't lower MMX shuffles");
17635
17636 bool V1IsUndef = V1.isUndef();
17637 bool V2IsUndef = V2.isUndef();
17638 if (V1IsUndef && V2IsUndef)
17639 return DAG.getUNDEF(VT);
17640
17641 // When we create a shuffle node we put the UNDEF node to second operand,
17642 // but in some cases the first operand may be transformed to UNDEF.
17643 // In this case we should just commute the node.
17644 if (V1IsUndef)
17645 return DAG.getCommutedVectorShuffle(*SVOp);
17646
17647 // Check for non-undef masks pointing at an undef vector and make the masks
17648 // undef as well. This makes it easier to match the shuffle based solely on
17649 // the mask.
17650 if (V2IsUndef &&
17651 any_of(OrigMask, [NumElements](int M) { return M >= NumElements; })) {
17652 SmallVector<int, 8> NewMask(OrigMask);
17653 for (int &M : NewMask)
17654 if (M >= NumElements)
17655 M = -1;
17656 return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
17657 }
17658
17659 // Check for illegal shuffle mask element index values.
17660 int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);
17661 (void)MaskUpperLimit;
17662 assert(llvm::all_of(OrigMask,
17663 [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&
17664 "Out of bounds shuffle index");
17665
17666 // We actually see shuffles that are entirely re-arrangements of a set of
17667 // zero inputs. This mostly happens while decomposing complex shuffles into
17668 // simple ones. Directly lower these as a buildvector of zeros.
17669 APInt KnownUndef, KnownZero;
17670 computeZeroableShuffleElements(OrigMask, V1, V2, KnownUndef, KnownZero);
17671
17672 APInt Zeroable = KnownUndef | KnownZero;
17673 if (Zeroable.isAllOnes())
17674 return getZeroVector(VT, Subtarget, DAG, DL);
17675
17676 bool V2IsZero = !V2IsUndef && ISD::isBuildVectorAllZeros(V2.getNode());
17677
17678 // Try to collapse shuffles into using a vector type with fewer elements but
17679 // wider element types. We cap this to not form integers or floating point
17680 // elements wider than 64 bits. It does not seem beneficial to form i128
17681 // integers to handle flipping the low and high halves of AVX 256-bit vectors.
17682 SmallVector<int, 16> WidenedMask;
17683 if (VT.getScalarSizeInBits() < 64 && !Is1BitVector &&
17684 !canCombineAsMaskOperation(V1, Subtarget) &&
17685 !canCombineAsMaskOperation(V2, Subtarget) &&
17686 canWidenShuffleElements(OrigMask, Zeroable, V2IsZero, WidenedMask)) {
17687 // Shuffle mask widening should not interfere with a broadcast opportunity
17688 // by obfuscating the operands with bitcasts.
17689 // TODO: Avoid lowering directly from this top-level function: make this
17690 // a query (canLowerAsBroadcast) and defer lowering to the type-based calls.
17691 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
17692 Subtarget, DAG))
17693 return Broadcast;
17694
17695 MVT NewEltVT = VT.isFloatingPoint()
17696 ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2)
17697 : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2);
17698 int NewNumElts = NumElements / 2;
17699 MVT NewVT = MVT::getVectorVT(NewEltVT, NewNumElts);
17700 // Make sure that the new vector type is legal. For example, v2f64 isn't
17701 // legal on SSE1.
17702 if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {
17703 if (V2IsZero) {
17704 // Modify the new Mask to take all zeros from the all-zero vector.
17705 // Choose indices that are blend-friendly.
17706 bool UsedZeroVector = false;
17707 assert(is_contained(WidenedMask, SM_SentinelZero) &&
17708 "V2's non-undef elements are used?!");
17709 for (int i = 0; i != NewNumElts; ++i)
17710 if (WidenedMask[i] == SM_SentinelZero) {
17711 WidenedMask[i] = i + NewNumElts;
17712 UsedZeroVector = true;
17713 }
17714 // Ensure all elements of V2 are zero - isBuildVectorAllZeros permits
17715 // some elements to be undef.
17716 if (UsedZeroVector)
17717 V2 = getZeroVector(NewVT, Subtarget, DAG, DL);
17718 }
17719 V1 = DAG.getBitcast(NewVT, V1);
17720 V2 = DAG.getBitcast(NewVT, V2);
17721 return DAG.getBitcast(
17722 VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask));
17723 }
17724 }
17725
17726 SmallVector<SDValue> Ops = {V1, V2};
17727 SmallVector<int> Mask(OrigMask);
17728
17729 // Canonicalize the shuffle with any horizontal ops inputs.
17730 // NOTE: This may update Ops and Mask.
17731 if (SDValue HOp = canonicalizeShuffleMaskWithHorizOp(
17732 Ops, Mask, VT.getSizeInBits(), DL, DAG, Subtarget))
17733 return DAG.getBitcast(VT, HOp);
17734
17735 V1 = DAG.getBitcast(VT, Ops[0]);
17736 V2 = DAG.getBitcast(VT, Ops[1]);
17737 assert(NumElements == (int)Mask.size() &&
17738 "canonicalizeShuffleMaskWithHorizOp "
17739 "shouldn't alter the shuffle mask size");
17740
17741 // Commute the shuffle if it will improve canonicalization.
17742 if (canonicalizeShuffleMaskWithCommute(Mask)) {
17743 ShuffleVectorSDNode::commuteMask(Mask);
17744 std::swap(V1, V2);
17745 }
17746
17747 // For each vector width, delegate to a specialized lowering routine.
17748 if (VT.is128BitVector())
17749 return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17750
17751 if (VT.is256BitVector())
17752 return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17753
17754 if (VT.is512BitVector())
17755 return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17756
17757 if (Is1BitVector)
17758 return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17759
17760 llvm_unreachable("Unimplemented!");
17761}
17762
17763/// Try to lower a VSELECT instruction to a vector shuffle.
17764static SDValue lowerVSELECTtoVectorShuffle(SDValue Op,
17765 const X86Subtarget &Subtarget,
17766 SelectionDAG &DAG) {
17767 SDValue Cond = Op.getOperand(0);
17768 SDValue LHS = Op.getOperand(1);
17769 SDValue RHS = Op.getOperand(2);
17770 MVT VT = Op.getSimpleValueType();
17771
17772 // Only non-legal VSELECTs reach this lowering, convert those into generic
17773 // shuffles and re-use the shuffle lowering path for blends.
17774 if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) {
17775 SmallVector<int, 32> Mask;
17776 if (createShuffleMaskFromVSELECT(Mask, Cond))
17777 return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask);
17778 }
17779
17780 return SDValue();
17781}
17782
17783SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
17784 SDValue Cond = Op.getOperand(0);
17785 SDValue LHS = Op.getOperand(1);
17786 SDValue RHS = Op.getOperand(2);
17787
17788 SDLoc dl(Op);
17789 MVT VT = Op.getSimpleValueType();
17790 if (isSoftF16(VT, Subtarget)) {
17791 MVT NVT = VT.changeVectorElementTypeToInteger();
17792 return DAG.getBitcast(VT, DAG.getNode(ISD::VSELECT, dl, NVT, Cond,
17793 DAG.getBitcast(NVT, LHS),
17794 DAG.getBitcast(NVT, RHS)));
17795 }
17796
17797 // A vselect where all conditions and data are constants can be optimized into
17798 // a single vector load by SelectionDAGLegalize::ExpandBUILD_VECTOR().
17799 if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()) &&
17800 ISD::isBuildVectorOfConstantSDNodes(LHS.getNode()) &&
17801 ISD::isBuildVectorOfConstantSDNodes(RHS.getNode()))
17802 return SDValue();
17803
17804 // Try to lower this to a blend-style vector shuffle. This can handle all
17805 // constant condition cases.
17806 if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG))
17807 return BlendOp;
17808
17809 // If this VSELECT has a vector if i1 as a mask, it will be directly matched
17810 // with patterns on the mask registers on AVX-512.
17811 MVT CondVT = Cond.getSimpleValueType();
17812 unsigned CondEltSize = Cond.getScalarValueSizeInBits();
17813 if (CondEltSize == 1)
17814 return Op;
17815
17816 // Variable blends are only legal from SSE4.1 onward.
17817 if (!Subtarget.hasSSE41())
17818 return SDValue();
17819
17820 unsigned EltSize = VT.getScalarSizeInBits();
17821 unsigned NumElts = VT.getVectorNumElements();
17822
17823 // Expand v32i16/v64i8 without BWI.
17824 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
17825 return SDValue();
17826
17827 // If the VSELECT is on a 512-bit type, we have to convert a non-i1 condition
17828 // into an i1 condition so that we can use the mask-based 512-bit blend
17829 // instructions.
17830 if (VT.getSizeInBits() == 512) {
17831 // Build a mask by testing the condition against zero.
17832 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
17833 SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
17834 DAG.getConstant(0, dl, CondVT),
17835 ISD::SETNE);
17836 // Now return a new VSELECT using the mask.
17837 return DAG.getSelect(dl, VT, Mask, LHS, RHS);
17838 }
17839
17840 // SEXT/TRUNC cases where the mask doesn't match the destination size.
17841 if (CondEltSize != EltSize) {
17842 // If we don't have a sign splat, rely on the expansion.
17843 if (CondEltSize != DAG.ComputeNumSignBits(Cond))
17844 return SDValue();
17845
17846 MVT NewCondSVT = MVT::getIntegerVT(EltSize);
17847 MVT NewCondVT = MVT::getVectorVT(NewCondSVT, NumElts);
17848 Cond = DAG.getSExtOrTrunc(Cond, dl, NewCondVT);
17849 return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS);
17850 }
17851
17852 // v16i16/v32i8 selects without AVX2, if the condition and another operand
17853 // are free to split, then better to split before expanding the
17854 // select. Don't bother with XOP as it has the fast VPCMOV instruction.
17855 // TODO: This is very similar to narrowVectorSelect.
17856 // TODO: Add Load splitting to isFreeToSplitVector ?
17857 if (EltSize < 32 && VT.is256BitVector() && !Subtarget.hasAVX2() &&
17858 !Subtarget.hasXOP()) {
17859 bool FreeCond = isFreeToSplitVector(Cond.getNode(), DAG);
17860 bool FreeLHS = isFreeToSplitVector(LHS.getNode(), DAG) ||
17861 (ISD::isNormalLoad(LHS.getNode()) && LHS.hasOneUse());
17862 bool FreeRHS = isFreeToSplitVector(RHS.getNode(), DAG) ||
17863 (ISD::isNormalLoad(RHS.getNode()) && RHS.hasOneUse());
17864 if (FreeCond && (FreeLHS || FreeRHS))
17865 return splitVectorOp(Op, DAG, dl);
17866 }
17867
17868 // Only some types will be legal on some subtargets. If we can emit a legal
17869 // VSELECT-matching blend, return Op, and but if we need to expand, return
17870 // a null value.
17871 switch (VT.SimpleTy) {
17872 default:
17873 // Most of the vector types have blends past SSE4.1.
17874 return Op;
17875
17876 case MVT::v32i8:
17877 // The byte blends for AVX vectors were introduced only in AVX2.
17878 if (Subtarget.hasAVX2())
17879 return Op;
17880
17881 return SDValue();
17882
17883 case MVT::v8i16:
17884 case MVT::v16i16: {
17885 // Bitcast everything to the vXi8 type and use a vXi8 vselect.
17886 MVT CastVT = MVT::getVectorVT(MVT::i8, NumElts * 2);
17887 Cond = DAG.getBitcast(CastVT, Cond);
17888 LHS = DAG.getBitcast(CastVT, LHS);
17889 RHS = DAG.getBitcast(CastVT, RHS);
17890 SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS);
17891 return DAG.getBitcast(VT, Select);
17892 }
17893 }
17894}
17895
17896static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
17897 MVT VT = Op.getSimpleValueType();
17898 SDValue Vec = Op.getOperand(0);
17899 SDValue Idx = Op.getOperand(1);
17900 assert(isa<ConstantSDNode>(Idx) && "Constant index expected");
17901 SDLoc dl(Op);
17902
17903 if (!Vec.getSimpleValueType().is128BitVector())
17904 return SDValue();
17905
17906 if (VT.getSizeInBits() == 8) {
17907 // If IdxVal is 0, it's cheaper to do a move instead of a pextrb, unless
17908 // we're going to zero extend the register or fold the store.
17909 if (llvm::isNullConstant(Idx) && !X86::mayFoldIntoZeroExtend(Op) &&
17910 !X86::mayFoldIntoStore(Op))
17911 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,
17912 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17913 DAG.getBitcast(MVT::v4i32, Vec), Idx));
17914
17915 unsigned IdxVal = Idx->getAsZExtVal();
17916 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec,
17917 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17918 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
17919 }
17920
17921 if (VT == MVT::f32) {
17922 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy
17923 // the result back to FR32 register. It's only worth matching if the
17924 // result has a single use which is a store or a bitcast to i32. And in
17925 // the case of a store, it's not worth it if the index is a constant 0,
17926 // because a MOVSSmr can be used instead, which is smaller and faster.
17927 if (!Op.hasOneUse())
17928 return SDValue();
17929 SDNode *User = *Op.getNode()->use_begin();
17930 if ((User->getOpcode() != ISD::STORE || isNullConstant(Idx)) &&
17931 (User->getOpcode() != ISD::BITCAST ||
17932 User->getValueType(0) != MVT::i32))
17933 return SDValue();
17934 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17935 DAG.getBitcast(MVT::v4i32, Vec), Idx);
17936 return DAG.getBitcast(MVT::f32, Extract);
17937 }
17938
17939 if (VT == MVT::i32 || VT == MVT::i64)
17940 return Op;
17941
17942 return SDValue();
17943}
17944
17945/// Extract one bit from mask vector, like v16i1 or v8i1.
17946/// AVX-512 feature.
17947static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
17948 const X86Subtarget &Subtarget) {
17949 SDValue Vec = Op.getOperand(0);
17950 SDLoc dl(Vec);
17951 MVT VecVT = Vec.getSimpleValueType();
17952 SDValue Idx = Op.getOperand(1);
17953 auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
17954 MVT EltVT = Op.getSimpleValueType();
17955
17956 assert((VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI()) &&
17957 "Unexpected vector type in ExtractBitFromMaskVector");
17958
17959 // variable index can't be handled in mask registers,
17960 // extend vector to VR512/128
17961 if (!IdxC) {
17962 unsigned NumElts = VecVT.getVectorNumElements();
17963 // Extending v8i1/v16i1 to 512-bit get better performance on KNL
17964 // than extending to 128/256bit.
17965 if (NumElts == 1) {
17966 Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
17967 MVT IntVT = MVT::getIntegerVT(Vec.getValueType().getVectorNumElements());
17968 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, DAG.getBitcast(IntVT, Vec));
17969 }
17970 MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
17971 MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
17972 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec);
17973 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx);
17974 return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
17975 }
17976
17977 unsigned IdxVal = IdxC->getZExtValue();
17978 if (IdxVal == 0) // the operation is legal
17979 return Op;
17980
17981 // Extend to natively supported kshift.
17982 Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
17983
17984 // Use kshiftr instruction to move to the lower element.
17985 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec,
17986 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17987
17988 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
17989 DAG.getIntPtrConstant(0, dl));
17990}
17991
17992// Helper to find all the extracted elements from a vector.
17993static APInt getExtractedDemandedElts(SDNode *N) {
17994 MVT VT = N->getSimpleValueType(0);
17995 unsigned NumElts = VT.getVectorNumElements();
17996 APInt DemandedElts = APInt::getZero(NumElts);
17997 for (SDNode *User : N->uses()) {
17998 switch (User->getOpcode()) {
17999 case X86ISD::PEXTRB:
18000 case X86ISD::PEXTRW:
18001 case ISD::EXTRACT_VECTOR_ELT:
18002 if (!isa<ConstantSDNode>(User->getOperand(1))) {
18003 DemandedElts.setAllBits();
18004 return DemandedElts;
18005 }
18006 DemandedElts.setBit(User->getConstantOperandVal(1));
18007 break;
18008 case ISD::BITCAST: {
18009 if (!User->getValueType(0).isSimple() ||
18010 !User->getValueType(0).isVector()) {
18011 DemandedElts.setAllBits();
18012 return DemandedElts;
18013 }
18014 APInt DemandedSrcElts = getExtractedDemandedElts(User);
18015 DemandedElts |= APIntOps::ScaleBitMask(DemandedSrcElts, NumElts);
18016 break;
18017 }
18018 default:
18019 DemandedElts.setAllBits();
18020 return DemandedElts;
18021 }
18022 }
18023 return DemandedElts;
18024}
18025
18026SDValue
18027X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
18028 SelectionDAG &DAG) const {
18029 SDLoc dl(Op);
18030 SDValue Vec = Op.getOperand(0);
18031 MVT VecVT = Vec.getSimpleValueType();
18032 SDValue Idx = Op.getOperand(1);
18033 auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
18034
18035 if (VecVT.getVectorElementType() == MVT::i1)
18036 return ExtractBitFromMaskVector(Op, DAG, Subtarget);
18037
18038 if (!IdxC) {
18039 // Its more profitable to go through memory (1 cycles throughput)
18040 // than using VMOVD + VPERMV/PSHUFB sequence (2/3 cycles throughput)
18041 // IACA tool was used to get performance estimation
18042 // (https://software.intel.com/en-us/articles/intel-architecture-code-analyzer)
18043 //
18044 // example : extractelement <16 x i8> %a, i32 %i
18045 //
18046 // Block Throughput: 3.00 Cycles
18047 // Throughput Bottleneck: Port5
18048 //
18049 // | Num Of | Ports pressure in cycles | |
18050 // | Uops | 0 - DV | 5 | 6 | 7 | |
18051 // ---------------------------------------------
18052 // | 1 | | 1.0 | | | CP | vmovd xmm1, edi
18053 // | 1 | | 1.0 | | | CP | vpshufb xmm0, xmm0, xmm1
18054 // | 2 | 1.0 | 1.0 | | | CP | vpextrb eax, xmm0, 0x0
18055 // Total Num Of Uops: 4
18056 //
18057 //
18058 // Block Throughput: 1.00 Cycles
18059 // Throughput Bottleneck: PORT2_AGU, PORT3_AGU, Port4
18060 //
18061 // | | Ports pressure in cycles | |
18062 // |Uops| 1 | 2 - D |3 - D | 4 | 5 | |
18063 // ---------------------------------------------------------
18064 // |2^ | | 0.5 | 0.5 |1.0| |CP| vmovaps xmmword ptr [rsp-0x18], xmm0
18065 // |1 |0.5| | | |0.5| | lea rax, ptr [rsp-0x18]
18066 // |1 | |0.5, 0.5|0.5, 0.5| | |CP| mov al, byte ptr [rdi+rax*1]
18067 // Total Num Of Uops: 4
18068
18069 return SDValue();
18070 }
18071
18072 unsigned IdxVal = IdxC->getZExtValue();
18073
18074 // If this is a 256-bit vector result, first extract the 128-bit vector and
18075 // then extract the element from the 128-bit vector.
18076 if (VecVT.is256BitVector() || VecVT.is512BitVector()) {
18077 // Get the 128-bit vector.
18078 Vec = extract128BitVector(Vec, IdxVal, DAG, dl);
18079 MVT EltVT = VecVT.getVectorElementType();
18080
18081 unsigned ElemsPerChunk = 128 / EltVT.getSizeInBits();
18082 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
18083
18084 // Find IdxVal modulo ElemsPerChunk. Since ElemsPerChunk is a power of 2
18085 // this can be done with a mask.
18086 IdxVal &= ElemsPerChunk - 1;
18087 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
18088 DAG.getIntPtrConstant(IdxVal, dl));
18089 }
18090
18091 assert(VecVT.is128BitVector() && "Unexpected vector length");
18092
18093 MVT VT = Op.getSimpleValueType();
18094
18095 if (VT == MVT::i16) {
18096 // If IdxVal is 0, it's cheaper to do a move instead of a pextrw, unless
18097 // we're going to zero extend the register or fold the store (SSE41 only).
18098 if (IdxVal == 0 && !X86::mayFoldIntoZeroExtend(Op) &&
18099 !(Subtarget.hasSSE41() && X86::mayFoldIntoStore(Op))) {
18100 if (Subtarget.hasFP16())
18101 return Op;
18102
18103 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
18104 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18105 DAG.getBitcast(MVT::v4i32, Vec), Idx));
18106 }
18107
18108 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec,
18109 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
18110 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
18111 }
18112
18113 if (Subtarget.hasSSE41())
18114 if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
18115 return Res;
18116
18117 // Only extract a single element from a v16i8 source - determine the common
18118 // DWORD/WORD that all extractions share, and extract the sub-byte.
18119 // TODO: Add QWORD MOVQ extraction?
18120 if (VT == MVT::i8) {
18121 APInt DemandedElts = getExtractedDemandedElts(Vec.getNode());
18122 assert(DemandedElts.getBitWidth() == 16 && "Vector width mismatch");
18123
18124 // Extract either the lowest i32 or any i16, and extract the sub-byte.
18125 int DWordIdx = IdxVal / 4;
18126 if (DWordIdx == 0 && DemandedElts == (DemandedElts & 15)) {
18127 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
18128 DAG.getBitcast(MVT::v4i32, Vec),
18129 DAG.getIntPtrConstant(DWordIdx, dl));
18130 int ShiftVal = (IdxVal % 4) * 8;
18131 if (ShiftVal != 0)
18132 Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res,
18133 DAG.getConstant(ShiftVal, dl, MVT::i8));
18134 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
18135 }
18136
18137 int WordIdx = IdxVal / 2;
18138 if (DemandedElts == (DemandedElts & (3 << (WordIdx * 2)))) {
18139 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
18140 DAG.getBitcast(MVT::v8i16, Vec),
18141 DAG.getIntPtrConstant(WordIdx, dl));
18142 int ShiftVal = (IdxVal % 2) * 8;
18143 if (ShiftVal != 0)
18144 Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res,
18145 DAG.getConstant(ShiftVal, dl, MVT::i8));
18146 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
18147 }
18148 }
18149
18150 if (VT == MVT::f16 || VT.getSizeInBits() == 32) {
18151 if (IdxVal == 0)
18152 return Op;
18153
18154 // Shuffle the element to the lowest element, then movss or movsh.
18155 SmallVector<int, 8> Mask(VecVT.getVectorNumElements(), -1);
18156 Mask[0] = static_cast<int>(IdxVal);
18157 Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
18158 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
18159 DAG.getIntPtrConstant(0, dl));
18160 }
18161
18162 if (VT.getSizeInBits() == 64) {
18163 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b
18164 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught
18165 // to match extract_elt for f64.
18166 if (IdxVal == 0)
18167 return Op;
18168
18169 // UNPCKHPD the element to the lowest double word, then movsd.
18170 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
18171 // to a f64mem, the whole operation is folded into a single MOVHPDmr.
18172 int Mask[2] = { 1, -1 };
18173 Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
18174 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
18175 DAG.getIntPtrConstant(0, dl));
18176 }
18177
18178 return SDValue();
18179}
18180
18181/// Insert one bit to mask vector, like v16i1 or v8i1.
18182/// AVX-512 feature.
18183static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
18184 const X86Subtarget &Subtarget) {
18185 SDLoc dl(Op);
18186 SDValue Vec = Op.getOperand(0);
18187 SDValue Elt = Op.getOperand(1);
18188 SDValue Idx = Op.getOperand(2);
18189 MVT VecVT = Vec.getSimpleValueType();
18190
18191 if (!isa<ConstantSDNode>(Idx)) {
18192 // Non constant index. Extend source and destination,
18193 // insert element and then truncate the result.
18194 unsigned NumElts = VecVT.getVectorNumElements();
18195 MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
18196 MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
18197 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
18198 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec),
18199 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx);
18200 return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp);
18201 }
18202
18203 // Copy into a k-register, extract to v1i1 and insert_subvector.
18204 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt);
18205 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VecVT, Vec, EltInVec, Idx);
18206}
18207
18208SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
18209 SelectionDAG &DAG) const {
18210 MVT VT = Op.getSimpleValueType();
18211 MVT EltVT = VT.getVectorElementType();
18212 unsigned NumElts = VT.getVectorNumElements();
18213 unsigned EltSizeInBits = EltVT.getScalarSizeInBits();
18214
18215 if (EltVT == MVT::i1)
18216 return InsertBitToMaskVector(Op, DAG, Subtarget);
18217
18218 SDLoc dl(Op);
18219 SDValue N0 = Op.getOperand(0);
18220 SDValue N1 = Op.getOperand(1);
18221 SDValue N2 = Op.getOperand(2);
18222 auto *N2C = dyn_cast<ConstantSDNode>(N2);
18223
18224 if (EltVT == MVT::bf16) {
18225 MVT IVT = VT.changeVectorElementTypeToInteger();
18226 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVT,
18227 DAG.getBitcast(IVT, N0),
18228 DAG.getBitcast(MVT::i16, N1), N2);
18229 return DAG.getBitcast(VT, Res);
18230 }
18231
18232 if (!N2C) {
18233 // Variable insertion indices, usually we're better off spilling to stack,
18234 // but AVX512 can use a variable compare+select by comparing against all
18235 // possible vector indices, and FP insertion has less gpr->simd traffic.
18236 if (!(Subtarget.hasBWI() ||
18237 (Subtarget.hasAVX512() && EltSizeInBits >= 32) ||
18238 (Subtarget.hasSSE41() && (EltVT == MVT::f32 || EltVT == MVT::f64))))
18239 return SDValue();
18240
18241 MVT IdxSVT = MVT::getIntegerVT(EltSizeInBits);
18242 MVT IdxVT = MVT::getVectorVT(IdxSVT, NumElts);
18243 if (!isTypeLegal(IdxSVT) || !isTypeLegal(IdxVT))
18244 return SDValue();
18245
18246 SDValue IdxExt = DAG.getZExtOrTrunc(N2, dl, IdxSVT);
18247 SDValue IdxSplat = DAG.getSplatBuildVector(IdxVT, dl, IdxExt);
18248 SDValue EltSplat = DAG.getSplatBuildVector(VT, dl, N1);
18249
18250 SmallVector<SDValue, 16> RawIndices;
18251 for (unsigned I = 0; I != NumElts; ++I)
18252 RawIndices.push_back(DAG.getConstant(I, dl, IdxSVT));
18253 SDValue Indices = DAG.getBuildVector(IdxVT, dl, RawIndices);
18254
18255 // inselt N0, N1, N2 --> select (SplatN2 == {0,1,2...}) ? SplatN1 : N0.
18256 return DAG.getSelectCC(dl, IdxSplat, Indices, EltSplat, N0,
18257 ISD::CondCode::SETEQ);
18258 }
18259
18260 if (N2C->getAPIntValue().uge(NumElts))
18261 return SDValue();
18262 uint64_t IdxVal = N2C->getZExtValue();
18263
18264 bool IsZeroElt = X86::isZeroNode(N1);
18265 bool IsAllOnesElt = VT.isInteger() && llvm::isAllOnesConstant(N1);
18266
18267 if (IsZeroElt || IsAllOnesElt) {
18268 // Lower insertion of v16i8/v32i8/v64i16 -1 elts as an 'OR' blend.
18269 // We don't deal with i8 0 since it appears to be handled elsewhere.
18270 if (IsAllOnesElt &&
18271 ((VT == MVT::v16i8 && !Subtarget.hasSSE41()) ||
18272 ((VT == MVT::v32i8 || VT == MVT::v16i16) && !Subtarget.hasInt256()))) {
18273 SDValue ZeroCst = DAG.getConstant(0, dl, VT.getScalarType());
18274 SDValue OnesCst = DAG.getAllOnesConstant(dl, VT.getScalarType());
18275 SmallVector<SDValue, 8> CstVectorElts(NumElts, ZeroCst);
18276 CstVectorElts[IdxVal] = OnesCst;
18277 SDValue CstVector = DAG.getBuildVector(VT, dl, CstVectorElts);
18278 return DAG.getNode(ISD::OR, dl, VT, N0, CstVector);
18279 }
18280 // See if we can do this more efficiently with a blend shuffle with a
18281 // rematerializable vector.
18282 if (Subtarget.hasSSE41() &&
18283 (EltSizeInBits >= 16 || (IsZeroElt && !VT.is128BitVector()))) {
18284 SmallVector<int, 8> BlendMask;
18285 for (unsigned i = 0; i != NumElts; ++i)
18286 BlendMask.push_back(i == IdxVal ? i + NumElts : i);
18287 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl)
18288 : getOnesVector(VT, DAG, dl);
18289 return DAG.getVectorShuffle(VT, dl, N0, CstVector, BlendMask);
18290 }
18291 }
18292
18293 // If the vector is wider than 128 bits, extract the 128-bit subvector, insert
18294 // into that, and then insert the subvector back into the result.
18295 if (VT.is256BitVector() || VT.is512BitVector()) {
18296 // With a 256-bit vector, we can insert into the zero element efficiently
18297 // using a blend if we have AVX or AVX2 and the right data type.
18298 if (VT.is256BitVector() && IdxVal == 0) {
18299 // TODO: It is worthwhile to cast integer to floating point and back
18300 // and incur a domain crossing penalty if that's what we'll end up
18301 // doing anyway after extracting to a 128-bit vector.
18302 if ((Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) ||
18303 (Subtarget.hasAVX2() && (EltVT == MVT::i32 || EltVT == MVT::i64))) {
18304 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18305 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec,
18306 DAG.getTargetConstant(1, dl, MVT::i8));
18307 }
18308 }
18309
18310 unsigned NumEltsIn128 = 128 / EltSizeInBits;
18311 assert(isPowerOf2_32(NumEltsIn128) &&
18312 "Vectors will always have power-of-two number of elements.");
18313
18314 // If we are not inserting into the low 128-bit vector chunk,
18315 // then prefer the broadcast+blend sequence.
18316 // FIXME: relax the profitability check iff all N1 uses are insertions.
18317 if (IdxVal >= NumEltsIn128 &&
18318 ((Subtarget.hasAVX2() && EltSizeInBits != 8) ||
18319 (Subtarget.hasAVX() && (EltSizeInBits >= 32) &&
18320 X86::mayFoldLoad(N1, Subtarget)))) {
18321 SDValue N1SplatVec = DAG.getSplatBuildVector(VT, dl, N1);
18322 SmallVector<int, 8> BlendMask;
18323 for (unsigned i = 0; i != NumElts; ++i)
18324 BlendMask.push_back(i == IdxVal ? i + NumElts : i);
18325 return DAG.getVectorShuffle(VT, dl, N0, N1SplatVec, BlendMask);
18326 }
18327
18328 // Get the desired 128-bit vector chunk.
18329 SDValue V = extract128BitVector(N0, IdxVal, DAG, dl);
18330
18331 // Insert the element into the desired chunk.
18332 // Since NumEltsIn128 is a power of 2 we can use mask instead of modulo.
18333 unsigned IdxIn128 = IdxVal & (NumEltsIn128 - 1);
18334
18335 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1,
18336 DAG.getIntPtrConstant(IdxIn128, dl));
18337
18338 // Insert the changed part back into the bigger vector
18339 return insert128BitVector(N0, V, IdxVal, DAG, dl);
18340 }
18341 assert(VT.is128BitVector() && "Only 128-bit vector types should be left!");
18342
18343 // This will be just movw/movd/movq/movsh/movss/movsd.
18344 if (IdxVal == 0 && ISD::isBuildVectorAllZeros(N0.getNode())) {
18345 if (EltVT == MVT::i32 || EltVT == MVT::f32 || EltVT == MVT::f64 ||
18346 EltVT == MVT::f16 || EltVT == MVT::i64) {
18347 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
18348 return getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG);
18349 }
18350
18351 // We can't directly insert an i8 or i16 into a vector, so zero extend
18352 // it to i32 first.
18353 if (EltVT == MVT::i16 || EltVT == MVT::i8) {
18354 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, N1);
18355 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
18356 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, N1);
18357 N1 = getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG);
18358 return DAG.getBitcast(VT, N1);
18359 }
18360 }
18361
18362 // Transform it so it match pinsr{b,w} which expects a GR32 as its second
18363 // argument. SSE41 required for pinsrb.
18364 if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) {
18365 unsigned Opc;
18366 if (VT == MVT::v8i16) {
18367 assert(Subtarget.hasSSE2() && "SSE2 required for PINSRW");
18368 Opc = X86ISD::PINSRW;
18369 } else {
18370 assert(VT == MVT::v16i8 && "PINSRB requires v16i8 vector");
18371 assert(Subtarget.hasSSE41() && "SSE41 required for PINSRB");
18372 Opc = X86ISD::PINSRB;
18373 }
18374
18375 assert(N1.getValueType() != MVT::i32 && "Unexpected VT");
18376 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
18377 N2 = DAG.getTargetConstant(IdxVal, dl, MVT::i8);
18378 return DAG.getNode(Opc, dl, VT, N0, N1, N2);
18379 }
18380
18381 if (Subtarget.hasSSE41()) {
18382 if (EltVT == MVT::f32) {
18383 // Bits [7:6] of the constant are the source select. This will always be
18384 // zero here. The DAG Combiner may combine an extract_elt index into
18385 // these bits. For example (insert (extract, 3), 2) could be matched by
18386 // putting the '3' into bits [7:6] of X86ISD::INSERTPS.
18387 // Bits [5:4] of the constant are the destination select. This is the
18388 // value of the incoming immediate.
18389 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may
18390 // combine either bitwise AND or insert of float 0.0 to set these bits.
18391
18392 bool MinSize = DAG.getMachineFunction().getFunction().hasMinSize();
18393 if (IdxVal == 0 && (!MinSize || !X86::mayFoldLoad(N1, Subtarget))) {
18394 // If this is an insertion of 32-bits into the low 32-bits of
18395 // a vector, we prefer to generate a blend with immediate rather
18396 // than an insertps. Blends are simpler operations in hardware and so
18397 // will always have equal or better performance than insertps.
18398 // But if optimizing for size and there's a load folding opportunity,
18399 // generate insertps because blendps does not have a 32-bit memory
18400 // operand form.
18401 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
18402 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1,
18403 DAG.getTargetConstant(1, dl, MVT::i8));
18404 }
18405 // Create this as a scalar to vector..
18406 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
18407 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1,
18408 DAG.getTargetConstant(IdxVal << 4, dl, MVT::i8));
18409 }
18410
18411 // PINSR* works with constant index.
18412 if (EltVT == MVT::i32 || EltVT == MVT::i64)
18413 return Op;
18414 }
18415
18416 return SDValue();
18417}
18418
18419static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget,
18420 SelectionDAG &DAG) {
18421 SDLoc dl(Op);
18422 MVT OpVT = Op.getSimpleValueType();
18423
18424 // It's always cheaper to replace a xor+movd with xorps and simplifies further
18425 // combines.
18426 if (X86::isZeroNode(Op.getOperand(0)))
18427 return getZeroVector(OpVT, Subtarget, DAG, dl);
18428
18429 // If this is a 256-bit vector result, first insert into a 128-bit
18430 // vector and then insert into the 256-bit vector.
18431 if (!OpVT.is128BitVector()) {
18432 // Insert into a 128-bit vector.
18433 unsigned SizeFactor = OpVT.getSizeInBits() / 128;
18434 MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(),
18435 OpVT.getVectorNumElements() / SizeFactor);
18436
18437 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
18438
18439 // Insert the 128-bit vector.
18440 return insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
18441 }
18442 assert(OpVT.is128BitVector() && OpVT.isInteger() && OpVT != MVT::v2i64 &&
18443 "Expected an SSE type!");
18444
18445 // Pass through a v4i32 or V8i16 SCALAR_TO_VECTOR as that's what we use in
18446 // tblgen.
18447 if (OpVT == MVT::v4i32 || (OpVT == MVT::v8i16 && Subtarget.hasFP16()))
18448 return Op;
18449
18450 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
18451 return DAG.getBitcast(
18452 OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt));
18453}
18454
18455// Lower a node with an INSERT_SUBVECTOR opcode. This may result in a
18456// simple superregister reference or explicit instructions to insert
18457// the upper bits of a vector.
18458static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18459 SelectionDAG &DAG) {
18460 assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1);
18461
18462 return insert1BitVector(Op, DAG, Subtarget);
18463}
18464
18465static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
18466 SelectionDAG &DAG) {
18467 assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1 &&
18468 "Only vXi1 extract_subvectors need custom lowering");
18469
18470 SDLoc dl(Op);
18471 SDValue Vec = Op.getOperand(0);
18472 uint64_t IdxVal = Op.getConstantOperandVal(1);
18473
18474 if (IdxVal == 0) // the operation is legal
18475 return Op;
18476
18477 // Extend to natively supported kshift.
18478 Vec = widenMaskVector(Vec, false, Subtarget, DAG, dl);
18479
18480 // Shift to the LSB.
18481 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, Vec.getSimpleValueType(), Vec,
18482 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
18483
18484 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, Op.getValueType(), Vec,
18485 DAG.getIntPtrConstant(0, dl));
18486}
18487
18488// Returns the appropriate wrapper opcode for a global reference.
18489unsigned X86TargetLowering::getGlobalWrapperKind(
18490 const GlobalValue *GV, const unsigned char OpFlags) const {
18491 // References to absolute symbols are never PC-relative.
18492 if (GV && GV->isAbsoluteSymbolRef())
18493 return X86ISD::Wrapper;
18494
18495 // The following OpFlags under RIP-rel PIC use RIP.
18496 if (Subtarget.isPICStyleRIPRel() &&
18497 (OpFlags == X86II::MO_NO_FLAG || OpFlags == X86II::MO_COFFSTUB ||
18498 OpFlags == X86II::MO_DLLIMPORT))
18499 return X86ISD::WrapperRIP;
18500
18501 // GOTPCREL references must always use RIP.
18502 if (OpFlags == X86II::MO_GOTPCREL || OpFlags == X86II::MO_GOTPCREL_NORELAX)
18503 return X86ISD::WrapperRIP;
18504
18505 return X86ISD::Wrapper;
18506}
18507
18508// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
18509// their target counterpart wrapped in the X86ISD::Wrapper node. Suppose N is
18510// one of the above mentioned nodes. It has to be wrapped because otherwise
18511// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
18512// be used to form addressing mode. These wrapped nodes will be selected
18513// into MOV32ri.
18514SDValue
18515X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
18516 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
18517
18518 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18519 // global base reg.
18520 unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
18521
18522 auto PtrVT = getPointerTy(DAG.getDataLayout());
18523 SDValue Result = DAG.getTargetConstantPool(
18524 CP->getConstVal(), PtrVT, CP->getAlign(), CP->getOffset(), OpFlag);
18525 SDLoc DL(CP);
18526 Result =
18527 DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result);
18528 // With PIC, the address is actually $g + Offset.
18529 if (OpFlag) {
18530 Result =
18531 DAG.getNode(ISD::ADD, DL, PtrVT,
18532 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18533 }
18534
18535 return Result;
18536}
18537
18538SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
18539 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
18540
18541 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18542 // global base reg.
18543 unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
18544
18545 auto PtrVT = getPointerTy(DAG.getDataLayout());
18546 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
18547 SDLoc DL(JT);
18548 Result =
18549 DAG.getNode(getGlobalWrapperKind(nullptr, OpFlag), DL, PtrVT, Result);
18550
18551 // With PIC, the address is actually $g + Offset.
18552 if (OpFlag)
18553 Result =
18554 DAG.getNode(ISD::ADD, DL, PtrVT,
18555 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18556
18557 return Result;
18558}
18559
18560SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
18561 SelectionDAG &DAG) const {
18562 return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
18563}
18564
18565SDValue
18566X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
18567 // Create the TargetBlockAddressAddress node.
18568 unsigned char OpFlags =
18569 Subtarget.classifyBlockAddressReference();
18570 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
18571 int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
18572 SDLoc dl(Op);
18573 auto PtrVT = getPointerTy(DAG.getDataLayout());
18574 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
18575 Result =
18576 DAG.getNode(getGlobalWrapperKind(nullptr, OpFlags), dl, PtrVT, Result);
18577
18578 // With PIC, the address is actually $g + Offset.
18579 if (isGlobalRelativeToPICBase(OpFlags)) {
18580 Result = DAG.getNode(ISD::ADD, dl, PtrVT,
18581 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
18582 }
18583
18584 return Result;
18585}
18586
18587/// Creates target global address or external symbol nodes for calls or
18588/// other uses.
18589SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
18590 bool ForCall) const {
18591 // Unpack the global address or external symbol.
18592 SDLoc dl(Op);
18593 const GlobalValue *GV = nullptr;
18594 int64_t Offset = 0;
18595 const char *ExternalSym = nullptr;
18596 if (const auto *G = dyn_cast<GlobalAddressSDNode>(Op)) {
18597 GV = G->getGlobal();
18598 Offset = G->getOffset();
18599 } else {
18600 const auto *ES = cast<ExternalSymbolSDNode>(Op);
18601 ExternalSym = ES->getSymbol();
18602 }
18603
18604 // Calculate some flags for address lowering.
18605 const Module &Mod = *DAG.getMachineFunction().getFunction().getParent();
18606 unsigned char OpFlags;
18607 if (ForCall)
18608 OpFlags = Subtarget.classifyGlobalFunctionReference(GV, Mod);
18609 else
18610 OpFlags = Subtarget.classifyGlobalReference(GV, Mod);
18611 bool HasPICReg = isGlobalRelativeToPICBase(OpFlags);
18612 bool NeedsLoad = isGlobalStubReference(OpFlags);
18613
18614 CodeModel::Model M = DAG.getTarget().getCodeModel();
18615 auto PtrVT = getPointerTy(DAG.getDataLayout());
18616 SDValue Result;
18617
18618 if (GV) {
18619 // Create a target global address if this is a global. If possible, fold the
18620 // offset into the global address reference. Otherwise, ADD it on later.
18621 // Suppress the folding if Offset is negative: movl foo-1, %eax is not
18622 // allowed because if the address of foo is 0, the ELF R_X86_64_32
18623 // relocation will compute to a negative value, which is invalid.
18624 int64_t GlobalOffset = 0;
18625 if (OpFlags == X86II::MO_NO_FLAG && Offset >= 0 &&
18626 X86::isOffsetSuitableForCodeModel(Offset, M, true)) {
18627 std::swap(GlobalOffset, Offset);
18628 }
18629 Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, GlobalOffset, OpFlags);
18630 } else {
18631 // If this is not a global address, this must be an external symbol.
18632 Result = DAG.getTargetExternalSymbol(ExternalSym, PtrVT, OpFlags);
18633 }
18634
18635 // If this is a direct call, avoid the wrapper if we don't need to do any
18636 // loads or adds. This allows SDAG ISel to match direct calls.
18637 if (ForCall && !NeedsLoad && !HasPICReg && Offset == 0)
18638 return Result;
18639
18640 Result = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, PtrVT, Result);
18641
18642 // With PIC, the address is actually $g + Offset.
18643 if (HasPICReg) {
18644 Result = DAG.getNode(ISD::ADD, dl, PtrVT,
18645 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
18646 }
18647
18648 // For globals that require a load from a stub to get the address, emit the
18649 // load.
18650 if (NeedsLoad)
18651 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
18652 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
18653
18654 // If there was a non-zero offset that we didn't fold, create an explicit
18655 // addition for it.
18656 if (Offset != 0)
18657 Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result,
18658 DAG.getSignedConstant(Offset, dl, PtrVT));
18659
18660 return Result;
18661}
18662
18663SDValue
18664X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
18665 return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
18666}
18667
18668static SDValue
18669GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
18670 SDValue *InGlue, const EVT PtrVT, unsigned ReturnReg,
18671 unsigned char OperandFlags, bool LocalDynamic = false) {
18672 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
18673 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
18674 SDLoc dl(GA);
18675 SDValue TGA;
18676 bool UseTLSDESC = DAG.getTarget().useTLSDESC();
18677 if (LocalDynamic && UseTLSDESC) {
18678 TGA = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT, OperandFlags);
18679 auto UI = TGA->use_begin();
18680 // Reuse existing GetTLSADDR node if we can find it.
18681 if (UI != TGA->use_end())
18682 return SDValue(*UI->use_begin()->use_begin(), 0);
18683 } else {
18684 TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
18685 GA->getOffset(), OperandFlags);
18686 }
18687
18688 X86ISD::NodeType CallType = UseTLSDESC ? X86ISD::TLSDESC
18689 : LocalDynamic ? X86ISD::TLSBASEADDR
18690 : X86ISD::TLSADDR;
18691
18692 if (InGlue) {
18693 SDValue Ops[] = { Chain, TGA, *InGlue };
18694 Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
18695 } else {
18696 SDValue Ops[] = { Chain, TGA };
18697 Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
18698 }
18699
18700 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
18701 MFI.setAdjustsStack(true);
18702 MFI.setHasCalls(true);
18703
18704 SDValue Glue = Chain.getValue(1);
18705 SDValue Ret = DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Glue);
18706
18707 if (!UseTLSDESC)
18708 return Ret;
18709
18710 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
18711 unsigned Seg = Subtarget.is64Bit() ? X86AS::FS : X86AS::GS;
18712
18713 Value *Ptr = Constant::getNullValue(PointerType::get(*DAG.getContext(), Seg));
18714 SDValue Offset =
18715 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl),
18716 MachinePointerInfo(Ptr));
18717 return DAG.getNode(ISD::ADD, dl, PtrVT, Ret, Offset);
18718}
18719
18720// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
18721static SDValue
18722LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18723 const EVT PtrVT) {
18724 SDValue InGlue;
18725 SDLoc dl(GA); // ? function entry point might be better
18726 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18727 DAG.getNode(X86ISD::GlobalBaseReg,
18728 SDLoc(), PtrVT), InGlue);
18729 InGlue = Chain.getValue(1);
18730
18731 return GetTLSADDR(DAG, Chain, GA, &InGlue, PtrVT, X86::EAX, X86II::MO_TLSGD);
18732}
18733
18734// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit LP64
18735static SDValue
18736LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18737 const EVT PtrVT) {
18738 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
18739 X86::RAX, X86II::MO_TLSGD);
18740}
18741
18742// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit ILP32
18743static SDValue
18744LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18745 const EVT PtrVT) {
18746 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
18747 X86::EAX, X86II::MO_TLSGD);
18748}
18749
18750static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
18751 SelectionDAG &DAG, const EVT PtrVT,
18752 bool Is64Bit, bool Is64BitLP64) {
18753 SDLoc dl(GA);
18754
18755 // Get the start address of the TLS block for this module.
18756 X86MachineFunctionInfo *MFI = DAG.getMachineFunction()
18757 .getInfo<X86MachineFunctionInfo>();
18758 MFI->incNumLocalDynamicTLSAccesses();
18759
18760 SDValue Base;
18761 if (Is64Bit) {
18762 unsigned ReturnReg = Is64BitLP64 ? X86::RAX : X86::EAX;
18763 Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, ReturnReg,
18764 X86II::MO_TLSLD, /*LocalDynamic=*/true);
18765 } else {
18766 SDValue InGlue;
18767 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18768 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InGlue);
18769 InGlue = Chain.getValue(1);
18770 Base = GetTLSADDR(DAG, Chain, GA, &InGlue, PtrVT, X86::EAX,
18771 X86II::MO_TLSLDM, /*LocalDynamic=*/true);
18772 }
18773
18774 // Note: the CleanupLocalDynamicTLSPass will remove redundant computations
18775 // of Base.
18776
18777 // Build x@dtpoff.
18778 unsigned char OperandFlags = X86II::MO_DTPOFF;
18779 unsigned WrapperKind = X86ISD::Wrapper;
18780 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18781 GA->getValueType(0),
18782 GA->getOffset(), OperandFlags);
18783 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18784
18785 // Add x@dtpoff with the base.
18786 return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base);
18787}
18788
18789// Lower ISD::GlobalTLSAddress using the "initial exec" or "local exec" model.
18790static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18791 const EVT PtrVT, TLSModel::Model model,
18792 bool is64Bit, bool isPIC) {
18793 SDLoc dl(GA);
18794
18795 // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
18796 Value *Ptr = Constant::getNullValue(
18797 PointerType::get(*DAG.getContext(), is64Bit ? 257 : 256));
18798
18799 SDValue ThreadPointer =
18800 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl),
18801 MachinePointerInfo(Ptr));
18802
18803 unsigned char OperandFlags = 0;
18804 // Most TLS accesses are not RIP relative, even on x86-64. One exception is
18805 // initialexec.
18806 unsigned WrapperKind = X86ISD::Wrapper;
18807 if (model == TLSModel::LocalExec) {
18808 OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
18809 } else if (model == TLSModel::InitialExec) {
18810 if (is64Bit) {
18811 OperandFlags = X86II::MO_GOTTPOFF;
18812 WrapperKind = X86ISD::WrapperRIP;
18813 } else {
18814 OperandFlags = isPIC ? X86II::MO_GOTNTPOFF : X86II::MO_INDNTPOFF;
18815 }
18816 } else {
18817 llvm_unreachable("Unexpected model");
18818 }
18819
18820 // emit "addl x@ntpoff,%eax" (local exec)
18821 // or "addl x@indntpoff,%eax" (initial exec)
18822 // or "addl x@gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic)
18823 SDValue TGA =
18824 DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
18825 GA->getOffset(), OperandFlags);
18826 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18827
18828 if (model == TLSModel::InitialExec) {
18829 if (isPIC && !is64Bit) {
18830 Offset = DAG.getNode(ISD::ADD, dl, PtrVT,
18831 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
18832 Offset);
18833 }
18834
18835 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
18836 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
18837 }
18838
18839 // The address of the thread local variable is the add of the thread
18840 // pointer with the offset of the variable.
18841 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
18842}
18843
18844SDValue
18845X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
18846
18847 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
18848
18849 if (DAG.getTarget().useEmulatedTLS())
18850 return LowerToTLSEmulatedModel(GA, DAG);
18851
18852 const GlobalValue *GV = GA->getGlobal();
18853 auto PtrVT = getPointerTy(DAG.getDataLayout());
18854 bool PositionIndependent = isPositionIndependent();
18855
18856 if (Subtarget.isTargetELF()) {
18857 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
18858 switch (model) {
18859 case TLSModel::GeneralDynamic:
18860 if (Subtarget.is64Bit()) {
18861 if (Subtarget.isTarget64BitLP64())
18862 return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
18863 return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT);
18864 }
18865 return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
18866 case TLSModel::LocalDynamic:
18867 return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(),
18868 Subtarget.isTarget64BitLP64());
18869 case TLSModel::InitialExec:
18870 case TLSModel::LocalExec:
18871 return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
18872 PositionIndependent);
18873 }
18874 llvm_unreachable("Unknown TLS model.");
18875 }
18876
18877 if (Subtarget.isTargetDarwin()) {
18878 // Darwin only has one model of TLS. Lower to that.
18879 unsigned char OpFlag = 0;
18880 unsigned WrapperKind = 0;
18881
18882 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18883 // global base reg.
18884 bool PIC32 = PositionIndependent && !Subtarget.is64Bit();
18885 if (PIC32) {
18886 OpFlag = X86II::MO_TLVP_PIC_BASE;
18887 WrapperKind = X86ISD::Wrapper;
18888 } else {
18889 OpFlag = X86II::MO_TLVP;
18890 WrapperKind = X86ISD::WrapperRIP;
18891 }
18892 SDLoc DL(Op);
18893 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
18894 GA->getValueType(0),
18895 GA->getOffset(), OpFlag);
18896 SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
18897
18898 // With PIC32, the address is actually $g + Offset.
18899 if (PIC32)
18900 Offset = DAG.getNode(ISD::ADD, DL, PtrVT,
18901 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
18902 Offset);
18903
18904 // Lowering the machine isd will make sure everything is in the right
18905 // location.
18906 SDValue Chain = DAG.getEntryNode();
18907 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
18908 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
18909 SDValue Args[] = { Chain, Offset };
18910 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
18911 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, Chain.getValue(1), DL);
18912
18913 // TLSCALL will be codegen'ed as call. Inform MFI that function has calls.
18914 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
18915 MFI.setAdjustsStack(true);
18916
18917 // And our return value (tls address) is in the standard call return value
18918 // location.
18919 unsigned Reg = Subtarget.is64Bit() ? X86::RAX : X86::EAX;
18920 return DAG.getCopyFromReg(Chain, DL, Reg, PtrVT, Chain.getValue(1));
18921 }
18922
18923 if (Subtarget.isOSWindows()) {
18924 // Just use the implicit TLS architecture
18925 // Need to generate something similar to:
18926 // mov rdx, qword [gs:abs 58H]; Load pointer to ThreadLocalStorage
18927 // ; from TEB
18928 // mov ecx, dword [rel _tls_index]: Load index (from C runtime)
18929 // mov rcx, qword [rdx+rcx*8]
18930 // mov eax, .tls$:tlsvar
18931 // [rax+rcx] contains the address
18932 // Windows 64bit: gs:0x58
18933 // Windows 32bit: fs:__tls_array
18934
18935 SDLoc dl(GA);
18936 SDValue Chain = DAG.getEntryNode();
18937
18938 // Get the Thread Pointer, which is %fs:__tls_array (32-bit) or
18939 // %gs:0x58 (64-bit). On MinGW, __tls_array is not available, so directly
18940 // use its literal value of 0x2C.
18941 Value *Ptr = Constant::getNullValue(
18942 Subtarget.is64Bit() ? PointerType::get(*DAG.getContext(), 256)
18943 : PointerType::get(*DAG.getContext(), 257));
18944
18945 SDValue TlsArray = Subtarget.is64Bit()
18946 ? DAG.getIntPtrConstant(0x58, dl)
18947 : (Subtarget.isTargetWindowsGNU()
18948 ? DAG.getIntPtrConstant(0x2C, dl)
18949 : DAG.getExternalSymbol("_tls_array", PtrVT));
18950
18951 SDValue ThreadPointer =
18952 DAG.getLoad(PtrVT, dl, Chain, TlsArray, MachinePointerInfo(Ptr));
18953
18954 SDValue res;
18955 if (GV->getThreadLocalMode() == GlobalVariable::LocalExecTLSModel) {
18956 res = ThreadPointer;
18957 } else {
18958 // Load the _tls_index variable
18959 SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT);
18960 if (Subtarget.is64Bit())
18961 IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain, IDX,
18962 MachinePointerInfo(), MVT::i32);
18963 else
18964 IDX = DAG.getLoad(PtrVT, dl, Chain, IDX, MachinePointerInfo());
18965
18966 const DataLayout &DL = DAG.getDataLayout();
18967 SDValue Scale =
18968 DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8);
18969 IDX = DAG.getNode(ISD::SHL, dl, PtrVT, IDX, Scale);
18970
18971 res = DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, IDX);
18972 }
18973
18974 res = DAG.getLoad(PtrVT, dl, Chain, res, MachinePointerInfo());
18975
18976 // Get the offset of start of .tls section
18977 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18978 GA->getValueType(0),
18979 GA->getOffset(), X86II::MO_SECREL);
18980 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
18981
18982 // The address of the thread local variable is the add of the thread
18983 // pointer with the offset of the variable.
18984 return DAG.getNode(ISD::ADD, dl, PtrVT, res, Offset);
18985 }
18986
18987 llvm_unreachable("TLS not implemented for this target.");
18988}
18989
18990bool X86TargetLowering::addressingModeSupportsTLS(const GlobalValue &GV) const {
18991 if (Subtarget.is64Bit() && Subtarget.isTargetELF()) {
18992 const TargetMachine &TM = getTargetMachine();
18993 TLSModel::Model Model = TM.getTLSModel(&GV);
18994 switch (Model) {
18995 case TLSModel::LocalExec:
18996 case TLSModel::InitialExec:
18997 // We can include the %fs segment register in addressing modes.
18998 return true;
18999 case TLSModel::LocalDynamic:
19000 case TLSModel::GeneralDynamic:
19001 // These models do not result in %fs relative addresses unless
19002 // TLS descriptior are used.
19003 //
19004 // Even in the case of TLS descriptors we currently have no way to model
19005 // the difference between %fs access and the computations needed for the
19006 // offset and returning `true` for TLS-desc currently duplicates both
19007 // which is detrimental :-/
19008 return false;
19009 }
19010 }
19011 return false;
19012}
19013
19014/// Lower SRA_PARTS and friends, which return two i32 values
19015/// and take a 2 x i32 value to shift plus a shift amount.
19016/// TODO: Can this be moved to general expansion code?
19017static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
19018 SDValue Lo, Hi;
19019 DAG.getTargetLoweringInfo().expandShiftParts(Op.getNode(), Lo, Hi, DAG);
19020 return DAG.getMergeValues({Lo, Hi}, SDLoc(Op));
19021}
19022
19023// Try to use a packed vector operation to handle i64 on 32-bit targets when
19024// AVX512DQ is enabled.
19025static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, const SDLoc &dl,
19026 SelectionDAG &DAG,
19027 const X86Subtarget &Subtarget) {
19028 assert((Op.getOpcode() == ISD::SINT_TO_FP ||
19029 Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
19030 Op.getOpcode() == ISD::STRICT_UINT_TO_FP ||
19031 Op.getOpcode() == ISD::UINT_TO_FP) &&
19032 "Unexpected opcode!");
19033 bool IsStrict = Op->isStrictFPOpcode();
19034 unsigned OpNo = IsStrict ? 1 : 0;
19035 SDValue Src = Op.getOperand(OpNo);
19036 MVT SrcVT = Src.getSimpleValueType();
19037 MVT VT = Op.getSimpleValueType();
19038
19039 if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() ||
19040 (VT != MVT::f32 && VT != MVT::f64))
19041 return SDValue();
19042
19043 // Pack the i64 into a vector, do the operation and extract.
19044
19045 // Using 256-bit to ensure result is 128-bits for f32 case.
19046 unsigned NumElts = Subtarget.hasVLX() ? 4 : 8;
19047 MVT VecInVT = MVT::getVectorVT(MVT::i64, NumElts);
19048 MVT VecVT = MVT::getVectorVT(VT, NumElts);
19049
19050 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src);
19051 if (IsStrict) {
19052 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other},
19053 {Op.getOperand(0), InVec});
19054 SDValue Chain = CvtVec.getValue(1);
19055 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19056 DAG.getIntPtrConstant(0, dl));
19057 return DAG.getMergeValues({Value, Chain}, dl);
19058 }
19059
19060 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec);
19061
19062 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19063 DAG.getIntPtrConstant(0, dl));
19064}
19065
19066// Try to use a packed vector operation to handle i64 on 32-bit targets.
19067static SDValue LowerI64IntToFP16(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
19068 const X86Subtarget &Subtarget) {
19069 assert((Op.getOpcode() == ISD::SINT_TO_FP ||
19070 Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
19071 Op.getOpcode() == ISD::STRICT_UINT_TO_FP ||
19072 Op.getOpcode() == ISD::UINT_TO_FP) &&
19073 "Unexpected opcode!");
19074 bool IsStrict = Op->isStrictFPOpcode();
19075 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
19076 MVT SrcVT = Src.getSimpleValueType();
19077 MVT VT = Op.getSimpleValueType();
19078
19079 if (SrcVT != MVT::i64 || Subtarget.is64Bit() || VT != MVT::f16)
19080 return SDValue();
19081
19082 // Pack the i64 into a vector, do the operation and extract.
19083
19084 assert(Subtarget.hasFP16() && "Expected FP16");
19085
19086 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
19087 if (IsStrict) {
19088 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {MVT::v2f16, MVT::Other},
19089 {Op.getOperand(0), InVec});
19090 SDValue Chain = CvtVec.getValue(1);
19091 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19092 DAG.getIntPtrConstant(0, dl));
19093 return DAG.getMergeValues({Value, Chain}, dl);
19094 }
19095
19096 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, MVT::v2f16, InVec);
19097
19098 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
19099 DAG.getIntPtrConstant(0, dl));
19100}
19101
19102static bool useVectorCast(unsigned Opcode, MVT FromVT, MVT ToVT,
19103 const X86Subtarget &Subtarget) {
19104 switch (Opcode) {
19105 case ISD::SINT_TO_FP:
19106 // TODO: Handle wider types with AVX/AVX512.
19107 if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)
19108 return false;
19109 // CVTDQ2PS or (V)CVTDQ2PD
19110 return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);
19111
19112 case ISD::UINT_TO_FP:
19113 // TODO: Handle wider types and i64 elements.
19114 if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)
19115 return false;
19116 // VCVTUDQ2PS or VCVTUDQ2PD
19117 return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;
19118
19119 default:
19120 return false;
19121 }
19122}
19123
19124/// Given a scalar cast operation that is extracted from a vector, try to
19125/// vectorize the cast op followed by extraction. This will avoid an expensive
19126/// round-trip between XMM and GPR.
19127static SDValue vectorizeExtractedCast(SDValue Cast, const SDLoc &DL,
19128 SelectionDAG &DAG,
19129 const X86Subtarget &Subtarget) {
19130 // TODO: This could be enhanced to handle smaller integer types by peeking
19131 // through an extend.
19132 SDValue Extract = Cast.getOperand(0);
19133 MVT DestVT = Cast.getSimpleValueType();
19134 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
19135 !isa<ConstantSDNode>(Extract.getOperand(1)))
19136 return SDValue();
19137
19138 // See if we have a 128-bit vector cast op for this type of cast.
19139 SDValue VecOp = Extract.getOperand(0);
19140 MVT FromVT = VecOp.getSimpleValueType();
19141 unsigned NumEltsInXMM = 128 / FromVT.getScalarSizeInBits();
19142 MVT Vec128VT = MVT::getVectorVT(FromVT.getScalarType(), NumEltsInXMM);
19143 MVT ToVT = MVT::getVectorVT(DestVT, NumEltsInXMM);
19144 if (!useVectorCast(Cast.getOpcode(), Vec128VT, ToVT, Subtarget))
19145 return SDValue();
19146
19147 // If we are extracting from a non-zero element, first shuffle the source
19148 // vector to allow extracting from element zero.
19149 if (!isNullConstant(Extract.getOperand(1))) {
19150 SmallVector<int, 16> Mask(FromVT.getVectorNumElements(), -1);
19151 Mask[0] = Extract.getConstantOperandVal(1);
19152 VecOp = DAG.getVectorShuffle(FromVT, DL, VecOp, DAG.getUNDEF(FromVT), Mask);
19153 }
19154 // If the source vector is wider than 128-bits, extract the low part. Do not
19155 // create an unnecessarily wide vector cast op.
19156 if (FromVT != Vec128VT)
19157 VecOp = extract128BitVector(VecOp, 0, DAG, DL);
19158
19159 // cast (extelt V, 0) --> extelt (cast (extract_subv V)), 0
19160 // cast (extelt V, C) --> extelt (cast (extract_subv (shuffle V, [C...]))), 0
19161 SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
19162 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, DestVT, VCast,
19163 DAG.getIntPtrConstant(0, DL));
19164}
19165
19166/// Given a scalar cast to FP with a cast to integer operand (almost an ftrunc),
19167/// try to vectorize the cast ops. This will avoid an expensive round-trip
19168/// between XMM and GPR.
19169static SDValue lowerFPToIntToFP(SDValue CastToFP, const SDLoc &DL,
19170 SelectionDAG &DAG,
19171 const X86Subtarget &Subtarget) {
19172 // TODO: Allow FP_TO_UINT.
19173 SDValue CastToInt = CastToFP.getOperand(0);
19174 MVT VT = CastToFP.getSimpleValueType();
19175 if (CastToInt.getOpcode() != ISD::FP_TO_SINT || VT.isVector())
19176 return SDValue();
19177
19178 MVT IntVT = CastToInt.getSimpleValueType();
19179 SDValue X = CastToInt.getOperand(0);
19180 MVT SrcVT = X.getSimpleValueType();
19181 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
19182 return SDValue();
19183
19184 // See if we have 128-bit vector cast instructions for this type of cast.
19185 // We need cvttps2dq/cvttpd2dq and cvtdq2ps/cvtdq2pd.
19186 if (!Subtarget.hasSSE2() || (VT != MVT::f32 && VT != MVT::f64) ||
19187 IntVT != MVT::i32)
19188 return SDValue();
19189
19190 unsigned SrcSize = SrcVT.getSizeInBits();
19191 unsigned IntSize = IntVT.getSizeInBits();
19192 unsigned VTSize = VT.getSizeInBits();
19193 MVT VecSrcVT = MVT::getVectorVT(SrcVT, 128 / SrcSize);
19194 MVT VecIntVT = MVT::getVectorVT(IntVT, 128 / IntSize);
19195 MVT VecVT = MVT::getVectorVT(VT, 128 / VTSize);
19196
19197 // We need target-specific opcodes if this is v2f64 -> v4i32 -> v2f64.
19198 unsigned ToIntOpcode =
19199 SrcSize != IntSize ? X86ISD::CVTTP2SI : (unsigned)ISD::FP_TO_SINT;
19200 unsigned ToFPOpcode =
19201 IntSize != VTSize ? X86ISD::CVTSI2P : (unsigned)ISD::SINT_TO_FP;
19202
19203 // sint_to_fp (fp_to_sint X) --> extelt (sint_to_fp (fp_to_sint (s2v X))), 0
19204 //
19205 // We are not defining the high elements (for example, zero them) because
19206 // that could nullify any performance advantage that we hoped to gain from
19207 // this vector op hack. We do not expect any adverse effects (like denorm
19208 // penalties) with cast ops.
19209 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
19210 SDValue VecX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, X);
19211 SDValue VCastToInt = DAG.getNode(ToIntOpcode, DL, VecIntVT, VecX);
19212 SDValue VCastToFP = DAG.getNode(ToFPOpcode, DL, VecVT, VCastToInt);
19213 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VCastToFP, ZeroIdx);
19214}
19215
19216static SDValue lowerINT_TO_FP_vXi64(SDValue Op, const SDLoc &DL,
19217 SelectionDAG &DAG,
19218 const X86Subtarget &Subtarget) {
19219 bool IsStrict = Op->isStrictFPOpcode();
19220 MVT VT = Op->getSimpleValueType(0);
19221 SDValue Src = Op->getOperand(IsStrict ? 1 : 0);
19222
19223 if (Subtarget.hasDQI()) {
19224 assert(!Subtarget.hasVLX() && "Unexpected features");
19225
19226 assert((Src.getSimpleValueType() == MVT::v2i64 ||
19227 Src.getSimpleValueType() == MVT::v4i64) &&
19228 "Unsupported custom type");
19229
19230 // With AVX512DQ, but not VLX we need to widen to get a 512-bit result type.
19231 assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) &&
19232 "Unexpected VT!");
19233 MVT WideVT = VT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;
19234
19235 // Need to concat with zero vector for strict fp to avoid spurious
19236 // exceptions.
19237 SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64)
19238 : DAG.getUNDEF(MVT::v8i64);
19239 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src,
19240 DAG.getIntPtrConstant(0, DL));
19241 SDValue Res, Chain;
19242 if (IsStrict) {
19243 Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other},
19244 {Op->getOperand(0), Src});
19245 Chain = Res.getValue(1);
19246 } else {
19247 Res = DAG.getNode(Op.getOpcode(), DL, WideVT, Src);
19248 }
19249
19250 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19251 DAG.getIntPtrConstant(0, DL));
19252
19253 if (IsStrict)
19254 return DAG.getMergeValues({Res, Chain}, DL);
19255 return Res;
19256 }
19257
19258 bool IsSigned = Op->getOpcode() == ISD::SINT_TO_FP ||
19259 Op->getOpcode() == ISD::STRICT_SINT_TO_FP;
19260 if (VT != MVT::v4f32 || IsSigned)
19261 return SDValue();
19262
19263 SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
19264 SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
19265 SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
19266 DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One),
19267 DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One));
19268 SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT);
19269 SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src);
19270 SmallVector<SDValue, 4> SignCvts(4);
19271 SmallVector<SDValue, 4> Chains(4);
19272 for (int i = 0; i != 4; ++i) {
19273 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc,
19274 DAG.getIntPtrConstant(i, DL));
19275 if (IsStrict) {
19276 SignCvts[i] =
19277 DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {MVT::f32, MVT::Other},
19278 {Op.getOperand(0), Elt});
19279 Chains[i] = SignCvts[i].getValue(1);
19280 } else {
19281 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, DL, MVT::f32, Elt);
19282 }
19283 }
19284 SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts);
19285
19286 SDValue Slow, Chain;
19287 if (IsStrict) {
19288 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
19289 Slow = DAG.getNode(ISD::STRICT_FADD, DL, {MVT::v4f32, MVT::Other},
19290 {Chain, SignCvt, SignCvt});
19291 Chain = Slow.getValue(1);
19292 } else {
19293 Slow = DAG.getNode(ISD::FADD, DL, MVT::v4f32, SignCvt, SignCvt);
19294 }
19295
19296 IsNeg = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i32, IsNeg);
19297 SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt);
19298
19299 if (IsStrict)
19300 return DAG.getMergeValues({Cvt, Chain}, DL);
19301
19302 return Cvt;
19303}
19304
19305static SDValue promoteXINT_TO_FP(SDValue Op, const SDLoc &dl,
19306 SelectionDAG &DAG) {
19307 bool IsStrict = Op->isStrictFPOpcode();
19308 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
19309 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19310 MVT VT = Op.getSimpleValueType();
19311 MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
19312
19313 SDValue Rnd = DAG.getIntPtrConstant(0, dl);
19314 if (IsStrict)
19315 return DAG.getNode(
19316 ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other},
19317 {Chain,
19318 DAG.getNode(Op.getOpcode(), dl, {NVT, MVT::Other}, {Chain, Src}),
19319 Rnd});
19320 return DAG.getNode(ISD::FP_ROUND, dl, VT,
19321 DAG.getNode(Op.getOpcode(), dl, NVT, Src), Rnd);
19322}
19323
19324static bool isLegalConversion(MVT VT, bool IsSigned,
19325 const X86Subtarget &Subtarget) {
19326 if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned)
19327 return true;
19328 if (VT == MVT::v8i32 && Subtarget.hasAVX() && IsSigned)
19329 return true;
19330 if (Subtarget.hasVLX() && (VT == MVT::v4i32 || VT == MVT::v8i32))
19331 return true;
19332 if (Subtarget.useAVX512Regs()) {
19333 if (VT == MVT::v16i32)
19334 return true;
19335 if (VT == MVT::v8i64 && Subtarget.hasDQI())
19336 return true;
19337 }
19338 if (Subtarget.hasDQI() && Subtarget.hasVLX() &&
19339 (VT == MVT::v2i64 || VT == MVT::v4i64))
19340 return true;
19341 return false;
19342}
19343
19344SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
19345 SelectionDAG &DAG) const {
19346 bool IsStrict = Op->isStrictFPOpcode();
19347 unsigned OpNo = IsStrict ? 1 : 0;
19348 SDValue Src = Op.getOperand(OpNo);
19349 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
19350 MVT SrcVT = Src.getSimpleValueType();
19351 MVT VT = Op.getSimpleValueType();
19352 SDLoc dl(Op);
19353
19354 if (isSoftF16(VT, Subtarget))
19355 return promoteXINT_TO_FP(Op, dl, DAG);
19356 else if (isLegalConversion(SrcVT, true, Subtarget))
19357 return Op;
19358
19359 if (Subtarget.isTargetWin64() && SrcVT == MVT::i128)
19360 return LowerWin64_INT128_TO_FP(Op, DAG);
19361
19362 if (SDValue Extract = vectorizeExtractedCast(Op, dl, DAG, Subtarget))
19363 return Extract;
19364
19365 if (SDValue R = lowerFPToIntToFP(Op, dl, DAG, Subtarget))
19366 return R;
19367
19368 if (SrcVT.isVector()) {
19369 if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) {
19370 // Note: Since v2f64 is a legal type. We don't need to zero extend the
19371 // source for strict FP.
19372 if (IsStrict)
19373 return DAG.getNode(
19374 X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
19375 {Chain, DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
19376 DAG.getUNDEF(SrcVT))});
19377 return DAG.getNode(X86ISD::CVTSI2P, dl, VT,
19378 DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
19379 DAG.getUNDEF(SrcVT)));
19380 }
19381 if (SrcVT == MVT::v2i64 || SrcVT == MVT::v4i64)
19382 return lowerINT_TO_FP_vXi64(Op, dl, DAG, Subtarget);
19383
19384 return SDValue();
19385 }
19386
19387 assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 &&
19388 "Unknown SINT_TO_FP to lower!");
19389
19390 bool UseSSEReg = isScalarFPTypeInSSEReg(VT);
19391
19392 // These are really Legal; return the operand so the caller accepts it as
19393 // Legal.
19394 if (SrcVT == MVT::i32 && UseSSEReg)
19395 return Op;
19396 if (SrcVT == MVT::i64 && UseSSEReg && Subtarget.is64Bit())
19397 return Op;
19398
19399 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, dl, DAG, Subtarget))
19400 return V;
19401 if (SDValue V = LowerI64IntToFP16(Op, dl, DAG, Subtarget))
19402 return V;
19403
19404 // SSE doesn't have an i16 conversion so we need to promote.
19405 if (SrcVT == MVT::i16 && (UseSSEReg || VT == MVT::f128)) {
19406 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src);
19407 if (IsStrict)
19408 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
19409 {Chain, Ext});
19410
19411 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Ext);
19412 }
19413
19414 if (VT == MVT::f128 || !Subtarget.hasX87())
19415 return SDValue();
19416
19417 SDValue ValueToStore = Src;
19418 if (SrcVT == MVT::i64 && Subtarget.hasSSE2() && !Subtarget.is64Bit())
19419 // Bitcasting to f64 here allows us to do a single 64-bit store from
19420 // an SSE register, avoiding the store forwarding penalty that would come
19421 // with two 32-bit stores.
19422 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
19423
19424 unsigned Size = SrcVT.getStoreSize();
19425 Align Alignment(Size);
19426 MachineFunction &MF = DAG.getMachineFunction();
19427 auto PtrVT = getPointerTy(MF.getDataLayout());
19428 int SSFI = MF.getFrameInfo().CreateStackObject(Size, Alignment, false);
19429 MachinePointerInfo MPI =
19430 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI);
19431 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19432 Chain = DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, Alignment);
19433 std::pair<SDValue, SDValue> Tmp =
19434 BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Alignment, DAG);
19435
19436 if (IsStrict)
19437 return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
19438
19439 return Tmp.first;
19440}
19441
19442std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(
19443 EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer,
19444 MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const {
19445 // Build the FILD
19446 SDVTList Tys;
19447 bool useSSE = isScalarFPTypeInSSEReg(DstVT);
19448 if (useSSE)
19449 Tys = DAG.getVTList(MVT::f80, MVT::Other);
19450 else
19451 Tys = DAG.getVTList(DstVT, MVT::Other);
19452
19453 SDValue FILDOps[] = {Chain, Pointer};
19454 SDValue Result =
19455 DAG.getMemIntrinsicNode(X86ISD::FILD, DL, Tys, FILDOps, SrcVT, PtrInfo,
19456 Alignment, MachineMemOperand::MOLoad);
19457 Chain = Result.getValue(1);
19458
19459 if (useSSE) {
19460 MachineFunction &MF = DAG.getMachineFunction();
19461 unsigned SSFISize = DstVT.getStoreSize();
19462 int SSFI =
19463 MF.getFrameInfo().CreateStackObject(SSFISize, Align(SSFISize), false);
19464 auto PtrVT = getPointerTy(MF.getDataLayout());
19465 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19466 Tys = DAG.getVTList(MVT::Other);
19467 SDValue FSTOps[] = {Chain, Result, StackSlot};
19468 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
19469 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI),
19470 MachineMemOperand::MOStore, SSFISize, Align(SSFISize));
19471
19472 Chain =
19473 DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, FSTOps, DstVT, StoreMMO);
19474 Result = DAG.getLoad(
19475 DstVT, DL, Chain, StackSlot,
19476 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI));
19477 Chain = Result.getValue(1);
19478 }
19479
19480 return { Result, Chain };
19481}
19482
19483/// Horizontal vector math instructions may be slower than normal math with
19484/// shuffles. Limit horizontal op codegen based on size/speed trade-offs, uarch
19485/// implementation, and likely shuffle complexity of the alternate sequence.
19486static bool shouldUseHorizontalOp(bool IsSingleSource, SelectionDAG &DAG,
19487 const X86Subtarget &Subtarget) {
19488 bool IsOptimizingSize = DAG.shouldOptForSize();
19489 bool HasFastHOps = Subtarget.hasFastHorizontalOps();
19490 return !IsSingleSource || IsOptimizingSize || HasFastHOps;
19491}
19492
19493/// 64-bit unsigned integer to double expansion.
19494static SDValue LowerUINT_TO_FP_i64(SDValue Op, const SDLoc &dl,
19495 SelectionDAG &DAG,
19496 const X86Subtarget &Subtarget) {
19497 // We can't use this algorithm for strict fp. It produces -0.0 instead of +0.0
19498 // when converting 0 when rounding toward negative infinity. Caller will
19499 // fall back to Expand for when i64 or is legal or use FILD in 32-bit mode.
19500 assert(!Op->isStrictFPOpcode() && "Expected non-strict uint_to_fp!");
19501 // This algorithm is not obvious. Here it is what we're trying to output:
19502 /*
19503 movq %rax, %xmm0
19504 punpckldq (c0), %xmm0 // c0: (uint4){ 0x43300000U, 0x45300000U, 0U, 0U }
19505 subpd (c1), %xmm0 // c1: (double2){ 0x1.0p52, 0x1.0p52 * 0x1.0p32 }
19506 #ifdef __SSE3__
19507 haddpd %xmm0, %xmm0
19508 #else
19509 pshufd $0x4e, %xmm0, %xmm1
19510 addpd %xmm1, %xmm0
19511 #endif
19512 */
19513
19514 LLVMContext *Context = DAG.getContext();
19515
19516 // Build some magic constants.
19517 static const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 };
19518 Constant *C0 = ConstantDataVector::get(*Context, CV0);
19519 auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
19520 SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, Align(16));
19521
19522 SmallVector<Constant*,2> CV1;
19523 CV1.push_back(
19524 ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
19525 APInt(64, 0x4330000000000000ULL))));
19526 CV1.push_back(
19527 ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
19528 APInt(64, 0x4530000000000000ULL))));
19529 Constant *C1 = ConstantVector::get(CV1);
19530 SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, Align(16));
19531
19532 // Load the 64-bit value into an XMM register.
19533 SDValue XR1 =
19534 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(0));
19535 SDValue CLod0 = DAG.getLoad(
19536 MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
19537 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16));
19538 SDValue Unpck1 =
19539 getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0);
19540
19541 SDValue CLod1 = DAG.getLoad(
19542 MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,
19543 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16));
19544 SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);
19545 // TODO: Are there any fast-math-flags to propagate here?
19546 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
19547 SDValue Result;
19548
19549 if (Subtarget.hasSSE3() &&
19550 shouldUseHorizontalOp(true, DAG, Subtarget)) {
19551 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
19552 } else {
19553 SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
19554 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub);
19555 }
19556 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
19557 DAG.getIntPtrConstant(0, dl));
19558 return Result;
19559}
19560
19561/// 32-bit unsigned integer to float expansion.
19562static SDValue LowerUINT_TO_FP_i32(SDValue Op, const SDLoc &dl,
19563 SelectionDAG &DAG,
19564 const X86Subtarget &Subtarget) {
19565 unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
19566 // FP constant to bias correct the final result.
19567 SDValue Bias = DAG.getConstantFP(
19568 llvm::bit_cast<double>(0x4330000000000000ULL), dl, MVT::f64);
19569
19570 // Load the 32-bit value into an XMM register.
19571 SDValue Load =
19572 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Op.getOperand(OpNo));
19573
19574 // Zero out the upper parts of the register.
19575 Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG);
19576
19577 // Or the load with the bias.
19578 SDValue Or = DAG.getNode(
19579 ISD::OR, dl, MVT::v2i64,
19580 DAG.getBitcast(MVT::v2i64, Load),
19581 DAG.getBitcast(MVT::v2i64,
19582 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias)));
19583 Or =
19584 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
19585 DAG.getBitcast(MVT::v2f64, Or), DAG.getIntPtrConstant(0, dl));
19586
19587 if (Op.getNode()->isStrictFPOpcode()) {
19588 // Subtract the bias.
19589 // TODO: Are there any fast-math-flags to propagate here?
19590 SDValue Chain = Op.getOperand(0);
19591 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
19592 {Chain, Or, Bias});
19593
19594 if (Op.getValueType() == Sub.getValueType())
19595 return Sub;
19596
19597 // Handle final rounding.
19598 std::pair<SDValue, SDValue> ResultPair = DAG.getStrictFPExtendOrRound(
19599 Sub, Sub.getValue(1), dl, Op.getSimpleValueType());
19600
19601 return DAG.getMergeValues({ResultPair.first, ResultPair.second}, dl);
19602 }
19603
19604 // Subtract the bias.
19605 // TODO: Are there any fast-math-flags to propagate here?
19606 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
19607
19608 // Handle final rounding.
19609 return DAG.getFPExtendOrRound(Sub, dl, Op.getSimpleValueType());
19610}
19611
19612static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, const SDLoc &DL,
19613 SelectionDAG &DAG,
19614 const X86Subtarget &Subtarget) {
19615 if (Op.getSimpleValueType() != MVT::v2f64)
19616 return SDValue();
19617
19618 bool IsStrict = Op->isStrictFPOpcode();
19619
19620 SDValue N0 = Op.getOperand(IsStrict ? 1 : 0);
19621 assert(N0.getSimpleValueType() == MVT::v2i32 && "Unexpected input type");
19622
19623 if (Subtarget.hasAVX512()) {
19624 if (!Subtarget.hasVLX()) {
19625 // Let generic type legalization widen this.
19626 if (!IsStrict)
19627 return SDValue();
19628 // Otherwise pad the integer input with 0s and widen the operation.
19629 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19630 DAG.getConstant(0, DL, MVT::v2i32));
19631 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other},
19632 {Op.getOperand(0), N0});
19633 SDValue Chain = Res.getValue(1);
19634 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2f64, Res,
19635 DAG.getIntPtrConstant(0, DL));
19636 return DAG.getMergeValues({Res, Chain}, DL);
19637 }
19638
19639 // Legalize to v4i32 type.
19640 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19641 DAG.getUNDEF(MVT::v2i32));
19642 if (IsStrict)
19643 return DAG.getNode(X86ISD::STRICT_CVTUI2P, DL, {MVT::v2f64, MVT::Other},
19644 {Op.getOperand(0), N0});
19645 return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0);
19646 }
19647
19648 // Zero extend to 2i64, OR with the floating point representation of 2^52.
19649 // This gives us the floating point equivalent of 2^52 + the i32 integer
19650 // since double has 52-bits of mantissa. Then subtract 2^52 in floating
19651 // point leaving just our i32 integers in double format.
19652 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0);
19653 SDValue VBias = DAG.getConstantFP(
19654 llvm::bit_cast<double>(0x4330000000000000ULL), DL, MVT::v2f64);
19655 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn,
19656 DAG.getBitcast(MVT::v2i64, VBias));
19657 Or = DAG.getBitcast(MVT::v2f64, Or);
19658
19659 if (IsStrict)
19660 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v2f64, MVT::Other},
19661 {Op.getOperand(0), Or, VBias});
19662 return DAG.getNode(ISD::FSUB, DL, MVT::v2f64, Or, VBias);
19663}
19664
19665static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL,
19666 SelectionDAG &DAG,
19667 const X86Subtarget &Subtarget) {
19668 bool IsStrict = Op->isStrictFPOpcode();
19669 SDValue V = Op->getOperand(IsStrict ? 1 : 0);
19670 MVT VecIntVT = V.getSimpleValueType();
19671 assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&
19672 "Unsupported custom type");
19673
19674 if (Subtarget.hasAVX512()) {
19675 // With AVX512, but not VLX we need to widen to get a 512-bit result type.
19676 assert(!Subtarget.hasVLX() && "Unexpected features");
19677 MVT VT = Op->getSimpleValueType(0);
19678
19679 // v8i32->v8f64 is legal with AVX512 so just return it.
19680 if (VT == MVT::v8f64)
19681 return Op;
19682
19683 assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) &&
19684 "Unexpected VT!");
19685 MVT WideVT = VT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
19686 MVT WideIntVT = VT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
19687 // Need to concat with zero vector for strict fp to avoid spurious
19688 // exceptions.
19689 SDValue Tmp =
19690 IsStrict ? DAG.getConstant(0, DL, WideIntVT) : DAG.getUNDEF(WideIntVT);
19691 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideIntVT, Tmp, V,
19692 DAG.getIntPtrConstant(0, DL));
19693 SDValue Res, Chain;
19694 if (IsStrict) {
19695 Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other},
19696 {Op->getOperand(0), V});
19697 Chain = Res.getValue(1);
19698 } else {
19699 Res = DAG.getNode(ISD::UINT_TO_FP, DL, WideVT, V);
19700 }
19701
19702 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19703 DAG.getIntPtrConstant(0, DL));
19704
19705 if (IsStrict)
19706 return DAG.getMergeValues({Res, Chain}, DL);
19707 return Res;
19708 }
19709
19710 if (Subtarget.hasAVX() && VecIntVT == MVT::v4i32 &&
19711 Op->getSimpleValueType(0) == MVT::v4f64) {
19712 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V);
19713 Constant *Bias = ConstantFP::get(
19714 *DAG.getContext(),
19715 APFloat(APFloat::IEEEdouble(), APInt(64, 0x4330000000000000ULL)));
19716 auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
19717 SDValue CPIdx = DAG.getConstantPool(Bias, PtrVT, Align(8));
19718 SDVTList Tys = DAG.getVTList(MVT::v4f64, MVT::Other);
19719 SDValue Ops[] = {DAG.getEntryNode(), CPIdx};
19720 SDValue VBias = DAG.getMemIntrinsicNode(
19721 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64,
19722 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(8),
19723 MachineMemOperand::MOLoad);
19724
19725 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,
19726 DAG.getBitcast(MVT::v4i64, VBias));
19727 Or = DAG.getBitcast(MVT::v4f64, Or);
19728
19729 if (IsStrict)
19730 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v4f64, MVT::Other},
19731 {Op.getOperand(0), Or, VBias});
19732 return DAG.getNode(ISD::FSUB, DL, MVT::v4f64, Or, VBias);
19733 }
19734
19735 // The algorithm is the following:
19736 // #ifdef __SSE4_1__
19737 // uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
19738 // uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
19739 // (uint4) 0x53000000, 0xaa);
19740 // #else
19741 // uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
19742 // uint4 hi = (v >> 16) | (uint4) 0x53000000;
19743 // #endif
19744 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
19745 // return (float4) lo + fhi;
19746
19747 bool Is128 = VecIntVT == MVT::v4i32;
19748 MVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32;
19749 // If we convert to something else than the supported type, e.g., to v4f64,
19750 // abort early.
19751 if (VecFloatVT != Op->getSimpleValueType(0))
19752 return SDValue();
19753
19754 // In the #idef/#else code, we have in common:
19755 // - The vector of constants:
19756 // -- 0x4b000000
19757 // -- 0x53000000
19758 // - A shift:
19759 // -- v >> 16
19760
19761 // Create the splat vector for 0x4b000000.
19762 SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT);
19763 // Create the splat vector for 0x53000000.
19764 SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT);
19765
19766 // Create the right shift.
19767 SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT);
19768 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
19769
19770 SDValue Low, High;
19771 if (Subtarget.hasSSE41()) {
19772 MVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16;
19773 // uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
19774 SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow);
19775 SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
19776 // Low will be bitcasted right away, so do not bother bitcasting back to its
19777 // original type.
19778 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
19779 VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19780 // uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
19781 // (uint4) 0x53000000, 0xaa);
19782 SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
19783 SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift);
19784 // High will be bitcasted right away, so do not bother bitcasting back to
19785 // its original type.
19786 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
19787 VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19788 } else {
19789 SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
19790 // uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
19791 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
19792 Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow);
19793
19794 // uint4 hi = (v >> 16) | (uint4) 0x53000000;
19795 High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh);
19796 }
19797
19798 // Create the vector constant for (0x1.0p39f + 0x1.0p23f).
19799 SDValue VecCstFSub = DAG.getConstantFP(
19800 APFloat(APFloat::IEEEsingle(), APInt(32, 0x53000080)), DL, VecFloatVT);
19801
19802 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
19803 // NOTE: By using fsub of a positive constant instead of fadd of a negative
19804 // constant, we avoid reassociation in MachineCombiner when unsafe-fp-math is
19805 // enabled. See PR24512.
19806 SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High);
19807 // TODO: Are there any fast-math-flags to propagate here?
19808 // (float4) lo;
19809 SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low);
19810 // return (float4) lo + fhi;
19811 if (IsStrict) {
19812 SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other},
19813 {Op.getOperand(0), HighBitcast, VecCstFSub});
19814 return DAG.getNode(ISD::STRICT_FADD, DL, {VecFloatVT, MVT::Other},
19815 {FHigh.getValue(1), LowBitcast, FHigh});
19816 }
19817
19818 SDValue FHigh =
19819 DAG.getNode(ISD::FSUB, DL, VecFloatVT, HighBitcast, VecCstFSub);
19820 return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
19821}
19822
19823static SDValue lowerUINT_TO_FP_vec(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
19824 const X86Subtarget &Subtarget) {
19825 unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
19826 SDValue N0 = Op.getOperand(OpNo);
19827 MVT SrcVT = N0.getSimpleValueType();
19828
19829 switch (SrcVT.SimpleTy) {
19830 default:
19831 llvm_unreachable("Custom UINT_TO_FP is not supported!");
19832 case MVT::v2i32:
19833 return lowerUINT_TO_FP_v2i32(Op, dl, DAG, Subtarget);
19834 case MVT::v4i32:
19835 case MVT::v8i32:
19836 return lowerUINT_TO_FP_vXi32(Op, dl, DAG, Subtarget);
19837 case MVT::v2i64:
19838 case MVT::v4i64:
19839 return lowerINT_TO_FP_vXi64(Op, dl, DAG, Subtarget);
19840 }
19841}
19842
19843SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
19844 SelectionDAG &DAG) const {
19845 bool IsStrict = Op->isStrictFPOpcode();
19846 unsigned OpNo = IsStrict ? 1 : 0;
19847 SDValue Src = Op.getOperand(OpNo);
19848 SDLoc dl(Op);
19849 auto PtrVT = getPointerTy(DAG.getDataLayout());
19850 MVT SrcVT = Src.getSimpleValueType();
19851 MVT DstVT = Op->getSimpleValueType(0);
19852 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
19853
19854 // Bail out when we don't have native conversion instructions.
19855 if (DstVT == MVT::f128)
19856 return SDValue();
19857
19858 if (isSoftF16(DstVT, Subtarget))
19859 return promoteXINT_TO_FP(Op, dl, DAG);
19860 else if (isLegalConversion(SrcVT, false, Subtarget))
19861 return Op;
19862
19863 if (DstVT.isVector())
19864 return lowerUINT_TO_FP_vec(Op, dl, DAG, Subtarget);
19865
19866 if (Subtarget.isTargetWin64() && SrcVT == MVT::i128)
19867 return LowerWin64_INT128_TO_FP(Op, DAG);
19868
19869 if (SDValue Extract = vectorizeExtractedCast(Op, dl, DAG, Subtarget))
19870 return Extract;
19871
19872 if (Subtarget.hasAVX512() && isScalarFPTypeInSSEReg(DstVT) &&
19873 (SrcVT == MVT::i32 || (SrcVT == MVT::i64 && Subtarget.is64Bit()))) {
19874 // Conversions from unsigned i32 to f32/f64 are legal,
19875 // using VCVTUSI2SS/SD. Same for i64 in 64-bit mode.
19876 return Op;
19877 }
19878
19879 // Promote i32 to i64 and use a signed conversion on 64-bit targets.
19880 if (SrcVT == MVT::i32 && Subtarget.is64Bit()) {
19881 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Src);
19882 if (IsStrict)
19883 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DstVT, MVT::Other},
19884 {Chain, Src});
19885 return DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src);
19886 }
19887
19888 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, dl, DAG, Subtarget))
19889 return V;
19890 if (SDValue V = LowerI64IntToFP16(Op, dl, DAG, Subtarget))
19891 return V;
19892
19893 // The transform for i64->f64 isn't correct for 0 when rounding to negative
19894 // infinity. It produces -0.0, so disable under strictfp.
19895 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && Subtarget.hasSSE2() &&
19896 !IsStrict)
19897 return LowerUINT_TO_FP_i64(Op, dl, DAG, Subtarget);
19898 // The transform for i32->f64/f32 isn't correct for 0 when rounding to
19899 // negative infinity. So disable under strictfp. Using FILD instead.
19900 if (SrcVT == MVT::i32 && Subtarget.hasSSE2() && DstVT != MVT::f80 &&
19901 !IsStrict)
19902 return LowerUINT_TO_FP_i32(Op, dl, DAG, Subtarget);
19903 if (Subtarget.is64Bit() && SrcVT == MVT::i64 &&
19904 (DstVT == MVT::f32 || DstVT == MVT::f64))
19905 return SDValue();
19906
19907 // Make a 64-bit buffer, and use it to build an FILD.
19908 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64, 8);
19909 int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex();
19910 Align SlotAlign(8);
19911 MachinePointerInfo MPI =
19912 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI);
19913 if (SrcVT == MVT::i32) {
19914 SDValue OffsetSlot =
19915 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
19916 SDValue Store1 = DAG.getStore(Chain, dl, Src, StackSlot, MPI, SlotAlign);
19917 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
19918 OffsetSlot, MPI.getWithOffset(4), SlotAlign);
19919 std::pair<SDValue, SDValue> Tmp =
19920 BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, SlotAlign, DAG);
19921 if (IsStrict)
19922 return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
19923
19924 return Tmp.first;
19925 }
19926
19927 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP");
19928 SDValue ValueToStore = Src;
19929 if (isScalarFPTypeInSSEReg(Op.getValueType()) && !Subtarget.is64Bit()) {
19930 // Bitcasting to f64 here allows us to do a single 64-bit store from
19931 // an SSE register, avoiding the store forwarding penalty that would come
19932 // with two 32-bit stores.
19933 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
19934 }
19935 SDValue Store =
19936 DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, SlotAlign);
19937 // For i64 source, we need to add the appropriate power of 2 if the input
19938 // was negative. We must be careful to do the computation in x87 extended
19939 // precision, not in SSE.
19940 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
19941 SDValue Ops[] = {Store, StackSlot};
19942 SDValue Fild =
19943 DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, MVT::i64, MPI,
19944 SlotAlign, MachineMemOperand::MOLoad);
19945 Chain = Fild.getValue(1);
19946
19947 // Check whether the sign bit is set.
19948 SDValue SignSet = DAG.getSetCC(
19949 dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
19950 Op.getOperand(OpNo), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
19951
19952 // Build a 64 bit pair (FF, 0) in the constant pool, with FF in the hi bits.
19953 APInt FF(64, 0x5F80000000000000ULL);
19954 SDValue FudgePtr =
19955 DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF), PtrVT);
19956 Align CPAlignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlign();
19957
19958 // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
19959 SDValue Zero = DAG.getIntPtrConstant(0, dl);
19960 SDValue Four = DAG.getIntPtrConstant(4, dl);
19961 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, Four, Zero);
19962 FudgePtr = DAG.getNode(ISD::ADD, dl, PtrVT, FudgePtr, Offset);
19963
19964 // Load the value out, extending it from f32 to f80.
19965 SDValue Fudge = DAG.getExtLoad(
19966 ISD::EXTLOAD, dl, MVT::f80, Chain, FudgePtr,
19967 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
19968 CPAlignment);
19969 Chain = Fudge.getValue(1);
19970 // Extend everything to 80 bits to force it to be done on x87.
19971 // TODO: Are there any fast-math-flags to propagate here?
19972 if (IsStrict) {
19973 unsigned Opc = ISD::STRICT_FADD;
19974 // Windows needs the precision control changed to 80bits around this add.
19975 if (Subtarget.isOSWindows() && DstVT == MVT::f32)
19976 Opc = X86ISD::STRICT_FP80_ADD;
19977
19978 SDValue Add =
19979 DAG.getNode(Opc, dl, {MVT::f80, MVT::Other}, {Chain, Fild, Fudge});
19980 // STRICT_FP_ROUND can't handle equal types.
19981 if (DstVT == MVT::f80)
19982 return Add;
19983 return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {DstVT, MVT::Other},
19984 {Add.getValue(1), Add, DAG.getIntPtrConstant(0, dl)});
19985 }
19986 unsigned Opc = ISD::FADD;
19987 // Windows needs the precision control changed to 80bits around this add.
19988 if (Subtarget.isOSWindows() && DstVT == MVT::f32)
19989 Opc = X86ISD::FP80_ADD;
19990
19991 SDValue Add = DAG.getNode(Opc, dl, MVT::f80, Fild, Fudge);
19992 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add,
19993 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
19994}
19995
19996// If the given FP_TO_SINT (IsSigned) or FP_TO_UINT (!IsSigned) operation
19997// is legal, or has an fp128 or f16 source (which needs to be promoted to f32),
19998// just return an SDValue().
19999// Otherwise it is assumed to be a conversion from one of f32, f64 or f80
20000// to i16, i32 or i64, and we lower it to a legal sequence and return the
20001// result.
20002SDValue X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
20003 bool IsSigned,
20004 SDValue &Chain) const {
20005 bool IsStrict = Op->isStrictFPOpcode();
20006 SDLoc DL(Op);
20007
20008 EVT DstTy = Op.getValueType();
20009 SDValue Value = Op.getOperand(IsStrict ? 1 : 0);
20010 EVT TheVT = Value.getValueType();
20011 auto PtrVT = getPointerTy(DAG.getDataLayout());
20012
20013 if (TheVT != MVT::f32 && TheVT != MVT::f64 && TheVT != MVT::f80) {
20014 // f16 must be promoted before using the lowering in this routine.
20015 // fp128 does not use this lowering.
20016 return SDValue();
20017 }
20018
20019 // If using FIST to compute an unsigned i64, we'll need some fixup
20020 // to handle values above the maximum signed i64. A FIST is always
20021 // used for the 32-bit subtarget, but also for f80 on a 64-bit target.
20022 bool UnsignedFixup = !IsSigned && DstTy == MVT::i64;
20023
20024 // FIXME: This does not generate an invalid exception if the input does not
20025 // fit in i32. PR44019
20026 if (!IsSigned && DstTy != MVT::i64) {
20027 // Replace the fp-to-uint32 operation with an fp-to-sint64 FIST.
20028 // The low 32 bits of the fist result will have the correct uint32 result.
20029 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");
20030 DstTy = MVT::i64;
20031 }
20032
20033 assert(DstTy.getSimpleVT() <= MVT::i64 &&
20034 DstTy.getSimpleVT() >= MVT::i16 &&
20035 "Unknown FP_TO_INT to lower!");
20036
20037 // We lower FP->int64 into FISTP64 followed by a load from a temporary
20038 // stack slot.
20039 MachineFunction &MF = DAG.getMachineFunction();
20040 unsigned MemSize = DstTy.getStoreSize();
20041 int SSFI =
20042 MF.getFrameInfo().CreateStackObject(MemSize, Align(MemSize), false);
20043 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
20044
20045 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
20046
20047 SDValue Adjust; // 0x0 or 0x80000000, for result sign bit adjustment.
20048
20049 if (UnsignedFixup) {
20050 //
20051 // Conversion to unsigned i64 is implemented with a select,
20052 // depending on whether the source value fits in the range
20053 // of a signed i64. Let Thresh be the FP equivalent of
20054 // 0x8000000000000000ULL.
20055 //
20056 // Adjust = (Value >= Thresh) ? 0x80000000 : 0;
20057 // FltOfs = (Value >= Thresh) ? 0x80000000 : 0;
20058 // FistSrc = (Value - FltOfs);
20059 // Fist-to-mem64 FistSrc
20060 // Add 0 or 0x800...0ULL to the 64-bit result, which is equivalent
20061 // to XOR'ing the high 32 bits with Adjust.
20062 //
20063 // Being a power of 2, Thresh is exactly representable in all FP formats.
20064 // For X87 we'd like to use the smallest FP type for this constant, but
20065 // for DAG type consistency we have to match the FP operand type.
20066
20067 APFloat Thresh(APFloat::IEEEsingle(), APInt(32, 0x5f000000));
20068 LLVM_ATTRIBUTE_UNUSED APFloat::opStatus Status = APFloat::opOK;
20069 bool LosesInfo = false;
20070 if (TheVT == MVT::f64)
20071 // The rounding mode is irrelevant as the conversion should be exact.
20072 Status = Thresh.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
20073 &LosesInfo);
20074 else if (TheVT == MVT::f80)
20075 Status = Thresh.convert(APFloat::x87DoubleExtended(),
20076 APFloat::rmNearestTiesToEven, &LosesInfo);
20077
20078 assert(Status == APFloat::opOK && !LosesInfo &&
20079 "FP conversion should have been exact");
20080
20081 SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
20082
20083 EVT ResVT = getSetCCResultType(DAG.getDataLayout(),
20084 *DAG.getContext(), TheVT);
20085 SDValue Cmp;
20086 if (IsStrict) {
20087 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE, Chain,
20088 /*IsSignaling*/ true);
20089 Chain = Cmp.getValue(1);
20090 } else {
20091 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE);
20092 }
20093
20094 // Our preferred lowering of
20095 //
20096 // (Value >= Thresh) ? 0x8000000000000000ULL : 0
20097 //
20098 // is
20099 //
20100 // (Value >= Thresh) << 63
20101 //
20102 // but since we can get here after LegalOperations, DAGCombine might do the
20103 // wrong thing if we create a select. So, directly create the preferred
20104 // version.
20105 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Cmp);
20106 SDValue Const63 = DAG.getConstant(63, DL, MVT::i8);
20107 Adjust = DAG.getNode(ISD::SHL, DL, MVT::i64, Zext, Const63);
20108
20109 SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp, ThreshVal,
20110 DAG.getConstantFP(0.0, DL, TheVT));
20111
20112 if (IsStrict) {
20113 Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other},
20114 { Chain, Value, FltOfs });
20115 Chain = Value.getValue(1);
20116 } else
20117 Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs);
20118 }
20119
20120 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, SSFI);
20121
20122 // FIXME This causes a redundant load/store if the SSE-class value is already
20123 // in memory, such as if it is on the callstack.
20124 if (isScalarFPTypeInSSEReg(TheVT)) {
20125 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
20126 Chain = DAG.getStore(Chain, DL, Value, StackSlot, MPI);
20127 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
20128 SDValue Ops[] = { Chain, StackSlot };
20129
20130 unsigned FLDSize = TheVT.getStoreSize();
20131 assert(FLDSize <= MemSize && "Stack slot not big enough");
20132 MachineMemOperand *MMO = MF.getMachineMemOperand(
20133 MPI, MachineMemOperand::MOLoad, FLDSize, Align(FLDSize));
20134 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, TheVT, MMO);
20135 Chain = Value.getValue(1);
20136 }
20137
20138 // Build the FP_TO_INT*_IN_MEM
20139 MachineMemOperand *MMO = MF.getMachineMemOperand(
20140 MPI, MachineMemOperand::MOStore, MemSize, Align(MemSize));
20141 SDValue Ops[] = { Chain, Value, StackSlot };
20142 SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
20143 DAG.getVTList(MVT::Other),
20144 Ops, DstTy, MMO);
20145
20146 SDValue Res = DAG.getLoad(Op.getValueType(), DL, FIST, StackSlot, MPI);
20147 Chain = Res.getValue(1);
20148
20149 // If we need an unsigned fixup, XOR the result with adjust.
20150 if (UnsignedFixup)
20151 Res = DAG.getNode(ISD::XOR, DL, MVT::i64, Res, Adjust);
20152
20153 return Res;
20154}
20155
20156static SDValue LowerAVXExtend(SDValue Op, const SDLoc &dl, SelectionDAG &DAG,
20157 const X86Subtarget &Subtarget) {
20158 MVT VT = Op.getSimpleValueType();
20159 SDValue In = Op.getOperand(0);
20160 MVT InVT = In.getSimpleValueType();
20161 unsigned Opc = Op.getOpcode();
20162
20163 assert(VT.isVector() && InVT.isVector() && "Expected vector type");
20164 assert((Opc == ISD::ANY_EXTEND || Opc == ISD::ZERO_EXTEND) &&
20165 "Unexpected extension opcode");
20166 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
20167 "Expected same number of elements");
20168 assert((VT.getVectorElementType() == MVT::i16 ||
20169 VT.getVectorElementType() == MVT::i32 ||
20170 VT.getVectorElementType() == MVT::i64) &&
20171 "Unexpected element type");
20172 assert((InVT.getVectorElementType() == MVT::i8 ||
20173 InVT.getVectorElementType() == MVT::i16 ||
20174 InVT.getVectorElementType() == MVT::i32) &&
20175 "Unexpected element type");
20176
20177 unsigned ExtendInVecOpc = DAG.getOpcode_EXTEND_VECTOR_INREG(Opc);
20178
20179 if (VT == MVT::v32i16 && !Subtarget.hasBWI()) {
20180 assert(InVT == MVT::v32i8 && "Unexpected VT!");
20181 return splitVectorIntUnary(Op, DAG, dl);
20182 }
20183
20184 if (Subtarget.hasInt256())
20185 return Op;
20186
20187 // Optimize vectors in AVX mode:
20188 //
20189 // v8i16 -> v8i32
20190 // Use vpmovzwd for 4 lower elements v8i16 -> v4i32.
20191 // Use vpunpckhwd for 4 upper elements v8i16 -> v4i32.
20192 // Concat upper and lower parts.
20193 //
20194 // v4i32 -> v4i64
20195 // Use vpmovzdq for 4 lower elements v4i32 -> v2i64.
20196 // Use vpunpckhdq for 4 upper elements v4i32 -> v2i64.
20197 // Concat upper and lower parts.
20198 //
20199 MVT HalfVT = VT.getHalfNumVectorElementsVT();
20200 SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In);
20201
20202 // Short-circuit if we can determine that each 128-bit half is the same value.
20203 // Otherwise, this is difficult to match and optimize.
20204 if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(In))
20205 if (hasIdenticalHalvesShuffleMask(Shuf->getMask()))
20206 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpLo);
20207
20208 SDValue ZeroVec = DAG.getConstant(0, dl, InVT);
20209 SDValue Undef = DAG.getUNDEF(InVT);
20210 bool NeedZero = Opc == ISD::ZERO_EXTEND;
20211 SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
20212 OpHi = DAG.getBitcast(HalfVT, OpHi);
20213
20214 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
20215}
20216
20217// Helper to split and extend a v16i1 mask to v16i8 or v16i16.
20218static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,
20219 const SDLoc &dl, SelectionDAG &DAG) {
20220 assert((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT.");
20221 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20222 DAG.getIntPtrConstant(0, dl));
20223 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
20224 DAG.getIntPtrConstant(8, dl));
20225 Lo = DAG.getNode(ExtOpc, dl, MVT::v8i16, Lo);
20226 Hi = DAG.getNode(ExtOpc, dl, MVT::v8i16, Hi);
20227 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi);
20228 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20229}
20230
20231static SDValue LowerZERO_EXTEND_Mask(SDValue Op, const SDLoc &DL,
20232 const X86Subtarget &Subtarget,
20233 SelectionDAG &DAG) {
20234 MVT VT = Op->getSimpleValueType(0);
20235 SDValue In = Op->getOperand(0);
20236 MVT InVT = In.getSimpleValueType();
20237 assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!");
20238 unsigned NumElts = VT.getVectorNumElements();
20239
20240 // For all vectors, but vXi8 we can just emit a sign_extend and a shift. This
20241 // avoids a constant pool load.
20242 if (VT.getVectorElementType() != MVT::i8) {
20243 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);
20244 return DAG.getNode(ISD::SRL, DL, VT, Extend,
20245 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
20246 }
20247
20248 // Extend VT if BWI is not supported.
20249 MVT ExtVT = VT;
20250 if (!Subtarget.hasBWI()) {
20251 // If v16i32 is to be avoided, we'll need to split and concatenate.
20252 if (NumElts == 16 && !Subtarget.canExtendTo512DQ())
20253 return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG);
20254
20255 ExtVT = MVT::getVectorVT(MVT::i32, NumElts);
20256 }
20257
20258 // Widen to 512-bits if VLX is not supported.
20259 MVT WideVT = ExtVT;
20260 if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) {
20261 NumElts *= 512 / ExtVT.getSizeInBits();
20262 InVT = MVT::getVectorVT(MVT::i1, NumElts);
20263 In = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT, DAG.getUNDEF(InVT),
20264 In, DAG.getIntPtrConstant(0, DL));
20265 WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(),
20266 NumElts);
20267 }
20268
20269 SDValue One = DAG.getConstant(1, DL, WideVT);
20270 SDValue Zero = DAG.getConstant(0, DL, WideVT);
20271
20272 SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero);
20273
20274 // Truncate if we had to extend above.
20275 if (VT != ExtVT) {
20276 WideVT = MVT::getVectorVT(MVT::i8, NumElts);
20277 SelectedVal = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SelectedVal);
20278 }
20279
20280 // Extract back to 128/256-bit if we widened.
20281 if (WideVT != VT)
20282 SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal,
20283 DAG.getIntPtrConstant(0, DL));
20284
20285 return SelectedVal;
20286}
20287
20288static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
20289 SelectionDAG &DAG) {
20290 SDValue In = Op.getOperand(0);
20291 MVT SVT = In.getSimpleValueType();
20292 SDLoc DL(Op);
20293
20294 if (SVT.getVectorElementType() == MVT::i1)
20295 return LowerZERO_EXTEND_Mask(Op, DL, Subtarget, DAG);
20296
20297 assert(Subtarget.hasAVX() && "Expected AVX support");
20298 return LowerAVXExtend(Op, DL, DAG, Subtarget);
20299}
20300
20301/// Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
20302/// It makes use of the fact that vectors with enough leading sign/zero bits
20303/// prevent the PACKSS/PACKUS from saturating the results.
20304/// AVX2 (Int256) sub-targets require extra shuffling as the PACK*S operates
20305/// within each 128-bit lane.
20306static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
20307 const SDLoc &DL, SelectionDAG &DAG,
20308 const X86Subtarget &Subtarget) {
20309 assert((Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) &&
20310 "Unexpected PACK opcode");
20311 assert(DstVT.isVector() && "VT not a vector?");
20312
20313 // Requires SSE2 for PACKSS (SSE41 PACKUSDW is handled below).
20314 if (!Subtarget.hasSSE2())
20315 return SDValue();
20316
20317 EVT SrcVT = In.getValueType();
20318
20319 // No truncation required, we might get here due to recursive calls.
20320 if (SrcVT == DstVT)
20321 return In;
20322
20323 unsigned NumElems = SrcVT.getVectorNumElements();
20324 if (NumElems < 2 || !isPowerOf2_32(NumElems) )
20325 return SDValue();
20326
20327 unsigned DstSizeInBits = DstVT.getSizeInBits();
20328 unsigned SrcSizeInBits = SrcVT.getSizeInBits();
20329 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
20330 assert(SrcSizeInBits > DstSizeInBits && "Illegal truncation");
20331
20332 LLVMContext &Ctx = *DAG.getContext();
20333 EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);
20334 EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
20335
20336 // Pack to the largest type possible:
20337 // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB.
20338 EVT InVT = MVT::i16, OutVT = MVT::i8;
20339 if (SrcVT.getScalarSizeInBits() > 16 &&
20340 (Opcode == X86ISD::PACKSS || Subtarget.hasSSE41())) {
20341 InVT = MVT::i32;
20342 OutVT = MVT::i16;
20343 }
20344
20345 // Sub-128-bit truncation - widen to 128-bit src and pack in the lower half.
20346 // On pre-AVX512, pack the src in both halves to help value tracking.
20347 if (SrcSizeInBits <= 128) {
20348 InVT = EVT::getVectorVT(Ctx, InVT, 128 / InVT.getSizeInBits());
20349 OutVT = EVT::getVectorVT(Ctx, OutVT, 128 / OutVT.getSizeInBits());
20350 In = widenSubVector(In, false, Subtarget, DAG, DL, 128);
20351 SDValue LHS = DAG.getBitcast(InVT, In);
20352 SDValue RHS = Subtarget.hasAVX512() ? DAG.getUNDEF(InVT) : LHS;
20353 SDValue Res = DAG.getNode(Opcode, DL, OutVT, LHS, RHS);
20354 Res = extractSubVector(Res, 0, DAG, DL, SrcSizeInBits / 2);
20355 Res = DAG.getBitcast(PackedVT, Res);
20356 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20357 }
20358
20359 // Split lower/upper subvectors.
20360 SDValue Lo, Hi;
20361 std::tie(Lo, Hi) = splitVector(In, DAG, DL);
20362
20363 // If Hi is undef, then don't bother packing it and widen the result instead.
20364 if (Hi.isUndef()) {
20365 EVT DstHalfVT = DstVT.getHalfNumVectorElementsVT(Ctx);
20366 if (SDValue Res =
20367 truncateVectorWithPACK(Opcode, DstHalfVT, Lo, DL, DAG, Subtarget))
20368 return widenSubVector(Res, false, Subtarget, DAG, DL, DstSizeInBits);
20369 }
20370
20371 unsigned SubSizeInBits = SrcSizeInBits / 2;
20372 InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
20373 OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
20374
20375 // 256bit -> 128bit truncate - PACK lower/upper 128-bit subvectors.
20376 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
20377 Lo = DAG.getBitcast(InVT, Lo);
20378 Hi = DAG.getBitcast(InVT, Hi);
20379 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20380 return DAG.getBitcast(DstVT, Res);
20381 }
20382
20383 // AVX2: 512bit -> 256bit truncate - PACK lower/upper 256-bit subvectors.
20384 // AVX2: 512bit -> 128bit truncate - PACK(PACK, PACK).
20385 if (SrcVT.is512BitVector() && Subtarget.hasInt256()) {
20386 Lo = DAG.getBitcast(InVT, Lo);
20387 Hi = DAG.getBitcast(InVT, Hi);
20388 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
20389
20390 // 256-bit PACK(ARG0, ARG1) leaves us with ((LO0,LO1),(HI0,HI1)),
20391 // so we need to shuffle to get ((LO0,HI0),(LO1,HI1)).
20392 // Scale shuffle mask to avoid bitcasts and help ComputeNumSignBits.
20393 SmallVector<int, 64> Mask;
20394 int Scale = 64 / OutVT.getScalarSizeInBits();
20395 narrowShuffleMaskElts(Scale, { 0, 2, 1, 3 }, Mask);
20396 Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask);
20397
20398 if (DstVT.is256BitVector())
20399 return DAG.getBitcast(DstVT, Res);
20400
20401 // If 512bit -> 128bit truncate another stage.
20402 Res = DAG.getBitcast(PackedVT, Res);
20403 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20404 }
20405
20406 // Recursively pack lower/upper subvectors, concat result and pack again.
20407 assert(SrcSizeInBits >= 256 && "Expected 256-bit vector or greater");
20408
20409 if (PackedVT.is128BitVector()) {
20410 // Avoid CONCAT_VECTORS on sub-128bit nodes as these can fail after
20411 // type legalization.
20412 SDValue Res =
20413 truncateVectorWithPACK(Opcode, PackedVT, In, DL, DAG, Subtarget);
20414 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20415 }
20416
20417 EVT HalfPackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);
20418 Lo = truncateVectorWithPACK(Opcode, HalfPackedVT, Lo, DL, DAG, Subtarget);
20419 Hi = truncateVectorWithPACK(Opcode, HalfPackedVT, Hi, DL, DAG, Subtarget);
20420 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
20421 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
20422}
20423
20424/// Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.
20425/// e.g. trunc <8 x i32> X to <8 x i16> -->
20426/// MaskX = X & 0xffff (clear high bits to prevent saturation)
20427/// packus (extract_subv MaskX, 0), (extract_subv MaskX, 1)
20428static SDValue truncateVectorWithPACKUS(EVT DstVT, SDValue In, const SDLoc &DL,
20429 const X86Subtarget &Subtarget,
20430 SelectionDAG &DAG) {
20431 In = DAG.getZeroExtendInReg(In, DL, DstVT);
20432 return truncateVectorWithPACK(X86ISD::PACKUS, DstVT, In, DL, DAG, Subtarget);
20433}
20434
20435/// Truncate using inreg sign extension and X86ISD::PACKSS.
20436static SDValue truncateVectorWithPACKSS(EVT DstVT, SDValue In, const SDLoc &DL,
20437 const X86Subtarget &Subtarget,
20438 SelectionDAG &DAG) {
20439 EVT SrcVT = In.getValueType();
20440 In = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SrcVT, In,
20441 DAG.getValueType(DstVT));
20442 return truncateVectorWithPACK(X86ISD::PACKSS, DstVT, In, DL, DAG, Subtarget);
20443}
20444
20445/// Helper to determine if \p In truncated to \p DstVT has the necessary
20446/// signbits / leading zero bits to be truncated with PACKSS / PACKUS,
20447/// possibly by converting a SRL node to SRA for sign extension.
20448static SDValue matchTruncateWithPACK(unsigned &PackOpcode, EVT DstVT,
20449 SDValue In, const SDLoc &DL,
20450 SelectionDAG &DAG,
20451 const X86Subtarget &Subtarget) {
20452 // Requires SSE2.
20453 if (!Subtarget.hasSSE2())
20454 return SDValue();
20455
20456 EVT SrcVT = In.getValueType();
20457 EVT DstSVT = DstVT.getVectorElementType();
20458 EVT SrcSVT = SrcVT.getVectorElementType();
20459 unsigned NumDstEltBits = DstSVT.getSizeInBits();
20460 unsigned NumSrcEltBits = SrcSVT.getSizeInBits();
20461
20462 // Check we have a truncation suited for PACKSS/PACKUS.
20463 if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32 || SrcSVT == MVT::i64) &&
20464 (DstSVT == MVT::i8 || DstSVT == MVT::i16 || DstSVT == MVT::i32)))
20465 return SDValue();
20466
20467 assert(NumSrcEltBits > NumDstEltBits && "Bad truncation");
20468 unsigned NumStages = Log2_32(NumSrcEltBits / NumDstEltBits);
20469
20470 // Truncation from 128-bit to vXi32 can be better handled with PSHUFD.
20471 // Truncation to sub-64-bit vXi16 can be better handled with PSHUFD/PSHUFLW.
20472 // Truncation from v2i64 to v2i8 can be better handled with PSHUFB.
20473 if ((DstSVT == MVT::i32 && SrcVT.getSizeInBits() <= 128) ||
20474 (DstSVT == MVT::i16 && SrcVT.getSizeInBits() <= (64 * NumStages)) ||
20475 (DstVT == MVT::v2i8 && SrcVT == MVT::v2i64 && Subtarget.hasSSSE3()))
20476 return SDValue();
20477
20478 // Prefer to lower v4i64 -> v4i32 as a shuffle unless we can cheaply
20479 // split this for packing.
20480 if (SrcVT == MVT::v4i64 && DstVT == MVT::v4i32 &&
20481 !isFreeToSplitVector(In.getNode(), DAG) &&
20482 (!Subtarget.hasAVX() || DAG.ComputeNumSignBits(In) != 64))
20483 return SDValue();
20484
20485 // Don't truncate AVX512 targets as multiple PACK nodes stages.
20486 if (Subtarget.hasAVX512() && NumStages > 1)
20487 return SDValue();
20488
20489 unsigned NumPackedSignBits = std::min<unsigned>(NumDstEltBits, 16);
20490 unsigned NumPackedZeroBits = Subtarget.hasSSE41() ? NumPackedSignBits : 8;
20491
20492 // Truncate with PACKUS if we are truncating a vector with leading zero
20493 // bits that extend all the way to the packed/truncated value.
20494 // e.g. Masks, zext_in_reg, etc.
20495 // Pre-SSE41 we can only use PACKUSWB.
20496 KnownBits Known = DAG.computeKnownBits(In);
20497 if ((NumSrcEltBits - NumPackedZeroBits) <= Known.countMinLeadingZeros()) {
20498 PackOpcode = X86ISD::PACKUS;
20499 return In;
20500 }
20501
20502 // Truncate with PACKSS if we are truncating a vector with sign-bits
20503 // that extend all the way to the packed/truncated value.
20504 // e.g. Comparison result, sext_in_reg, etc.
20505 unsigned NumSignBits = DAG.ComputeNumSignBits(In);
20506
20507 // Don't use PACKSS for vXi64 -> vXi32 truncations unless we're dealing with
20508 // a sign splat (or AVX512 VPSRAQ support). ComputeNumSignBits struggles to
20509 // see through BITCASTs later on and combines/simplifications can't then use
20510 // it.
20511 if (DstSVT == MVT::i32 && NumSignBits != NumSrcEltBits &&
20512 !Subtarget.hasAVX512())
20513 return SDValue();
20514
20515 unsigned MinSignBits = NumSrcEltBits - NumPackedSignBits;
20516 if (MinSignBits < NumSignBits) {
20517 PackOpcode = X86ISD::PACKSS;
20518 return In;
20519 }
20520
20521 // If we have a srl that only generates signbits that we will discard in
20522 // the truncation then we can use PACKSS by converting the srl to a sra.
20523 // SimplifyDemandedBits often relaxes sra to srl so we need to reverse it.
20524 if (In.getOpcode() == ISD::SRL && In->hasOneUse())
20525 if (std::optional<uint64_t> ShAmt = DAG.getValidShiftAmount(In)) {
20526 if (*ShAmt == MinSignBits) {
20527 PackOpcode = X86ISD::PACKSS;
20528 return DAG.getNode(ISD::SRA, DL, SrcVT, In->ops());
20529 }
20530 }
20531
20532 return SDValue();
20533}
20534
20535/// This function lowers a vector truncation of 'extended sign-bits' or
20536/// 'extended zero-bits' values.
20537/// vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS/PACKUS operations.
20538static SDValue LowerTruncateVecPackWithSignBits(MVT DstVT, SDValue In,
20539 const SDLoc &DL,
20540 const X86Subtarget &Subtarget,
20541 SelectionDAG &DAG) {
20542 MVT SrcVT = In.getSimpleValueType();
20543 MVT DstSVT = DstVT.getVectorElementType();
20544 MVT SrcSVT = SrcVT.getVectorElementType();
20545 if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32 || SrcSVT == MVT::i64) &&
20546 (DstSVT == MVT::i8 || DstSVT == MVT::i16 || DstSVT == MVT::i32)))
20547 return SDValue();
20548
20549 // If the upper half of the source is undef, then attempt to split and
20550 // only truncate the lower half.
20551 if (DstVT.getSizeInBits() >= 128) {
20552 SmallVector<SDValue> LowerOps;
20553 if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20554 MVT DstHalfVT = DstVT.getHalfNumVectorElementsVT();
20555 if (SDValue Res = LowerTruncateVecPackWithSignBits(DstHalfVT, Lo, DL,
20556 Subtarget, DAG))
20557 return widenSubVector(Res, false, Subtarget, DAG, DL,
20558 DstVT.getSizeInBits());
20559 }
20560 }
20561
20562 unsigned PackOpcode;
20563 if (SDValue Src =
20564 matchTruncateWithPACK(PackOpcode, DstVT, In, DL, DAG, Subtarget))
20565 return truncateVectorWithPACK(PackOpcode, DstVT, Src, DL, DAG, Subtarget);
20566
20567 return SDValue();
20568}
20569
20570/// This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into
20571/// X86ISD::PACKUS/X86ISD::PACKSS operations.
20572static SDValue LowerTruncateVecPack(MVT DstVT, SDValue In, const SDLoc &DL,
20573 const X86Subtarget &Subtarget,
20574 SelectionDAG &DAG) {
20575 MVT SrcVT = In.getSimpleValueType();
20576 MVT DstSVT = DstVT.getVectorElementType();
20577 MVT SrcSVT = SrcVT.getVectorElementType();
20578 unsigned NumElems = DstVT.getVectorNumElements();
20579 if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32 || SrcSVT == MVT::i64) &&
20580 (DstSVT == MVT::i8 || DstSVT == MVT::i16) && isPowerOf2_32(NumElems) &&
20581 NumElems >= 8))
20582 return SDValue();
20583
20584 // SSSE3's pshufb results in less instructions in the cases below.
20585 if (Subtarget.hasSSSE3() && NumElems == 8) {
20586 if (SrcSVT == MVT::i16)
20587 return SDValue();
20588 if (SrcSVT == MVT::i32 && (DstSVT == MVT::i8 || !Subtarget.hasSSE41()))
20589 return SDValue();
20590 }
20591
20592 // If the upper half of the source is undef, then attempt to split and
20593 // only truncate the lower half.
20594 if (DstVT.getSizeInBits() >= 128) {
20595 SmallVector<SDValue> LowerOps;
20596 if (SDValue Lo = isUpperSubvectorUndef(In, DL, DAG)) {
20597 MVT DstHalfVT = DstVT.getHalfNumVectorElementsVT();
20598 if (SDValue Res = LowerTruncateVecPack(DstHalfVT, Lo, DL, Subtarget, DAG))
20599 return widenSubVector(Res, false, Subtarget, DAG, DL,
20600 DstVT.getSizeInBits());
20601 }
20602 }
20603
20604 // SSE2 provides PACKUS for only 2 x v8i16 -> v16i8 and SSE4.1 provides PACKUS
20605 // for 2 x v4i32 -> v8i16. For SSSE3 and below, we need to use PACKSS to
20606 // truncate 2 x v4i32 to v8i16.
20607 if (Subtarget.hasSSE41() || DstSVT == MVT::i8)
20608 return truncateVectorWithPACKUS(DstVT, In, DL, Subtarget, DAG);
20609
20610 if (SrcSVT == MVT::i16 || SrcSVT == MVT::i32)
20611 return truncateVectorWithPACKSS(DstVT, In, DL, Subtarget, DAG);
20612
20613 // Special case vXi64 -> vXi16, shuffle to vXi32 and then use PACKSS.
20614 if (DstSVT == MVT::i16 && SrcSVT == MVT::i64) {
20615 MVT TruncVT = MVT::getVectorVT(MVT::i32, NumElems);
20616 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, In);
20617 return truncateVectorWithPACKSS(DstVT, Trunc, DL, Subtarget, DAG);
20618 }
20619
20620 return SDValue();
20621}
20622
20623static SDValue LowerTruncateVecI1(SDValue Op, const SDLoc &DL,
20624 SelectionDAG &DAG,
20625 const X86Subtarget &Subtarget) {
20626 MVT VT = Op.getSimpleValueType();
20627 SDValue In = Op.getOperand(0);
20628 MVT InVT = In.getSimpleValueType();
20629 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type.");
20630
20631 // Shift LSB to MSB and use VPMOVB/W2M or TESTD/Q.
20632 unsigned ShiftInx = InVT.getScalarSizeInBits() - 1;
20633 if (InVT.getScalarSizeInBits() <= 16) {
20634 if (Subtarget.hasBWI()) {
20635 // legal, will go to VPMOVB2M, VPMOVW2M
20636 if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
20637 // We need to shift to get the lsb into sign position.
20638 // Shift packed bytes not supported natively, bitcast to word
20639 MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits()/16);
20640 In = DAG.getNode(ISD::SHL, DL, ExtVT,
20641 DAG.getBitcast(ExtVT, In),
20642 DAG.getConstant(ShiftInx, DL, ExtVT));
20643 In = DAG.getBitcast(InVT, In);
20644 }
20645 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT),
20646 In, ISD::SETGT);
20647 }
20648 // Use TESTD/Q, extended vector to packed dword/qword.
20649 assert((InVT.is256BitVector() || InVT.is128BitVector()) &&
20650 "Unexpected vector type.");
20651 unsigned NumElts = InVT.getVectorNumElements();
20652 assert((NumElts == 8 || NumElts == 16) && "Unexpected number of elements");
20653 // We need to change to a wider element type that we have support for.
20654 // For 8 element vectors this is easy, we either extend to v8i32 or v8i64.
20655 // For 16 element vectors we extend to v16i32 unless we are explicitly
20656 // trying to avoid 512-bit vectors. If we are avoiding 512-bit vectors
20657 // we need to split into two 8 element vectors which we can extend to v8i32,
20658 // truncate and concat the results. There's an additional complication if
20659 // the original type is v16i8. In that case we can't split the v16i8
20660 // directly, so we need to shuffle high elements to low and use
20661 // sign_extend_vector_inreg.
20662 if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) {
20663 SDValue Lo, Hi;
20664 if (InVT == MVT::v16i8) {
20665 Lo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, In);
20666 Hi = DAG.getVectorShuffle(
20667 InVT, DL, In, In,
20668 {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1});
20669 Hi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, Hi);
20670 } else {
20671 assert(InVT == MVT::v16i16 && "Unexpected VT!");
20672 Lo = extract128BitVector(In, 0, DAG, DL);
20673 Hi = extract128BitVector(In, 8, DAG, DL);
20674 }
20675 // We're split now, just emit two truncates and a concat. The two
20676 // truncates will trigger legalization to come back to this function.
20677 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo);
20678 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi);
20679 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
20680 }
20681 // We either have 8 elements or we're allowed to use 512-bit vectors.
20682 // If we have VLX, we want to use the narrowest vector that can get the
20683 // job done so we use vXi32.
20684 MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts);
20685 MVT ExtVT = MVT::getVectorVT(EltVT, NumElts);
20686 In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
20687 InVT = ExtVT;
20688 ShiftInx = InVT.getScalarSizeInBits() - 1;
20689 }
20690
20691 if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
20692 // We need to shift to get the lsb into sign position.
20693 In = DAG.getNode(ISD::SHL, DL, InVT, In,
20694 DAG.getConstant(ShiftInx, DL, InVT));
20695 }
20696 // If we have DQI, emit a pattern that will be iseled as vpmovq2m/vpmovd2m.
20697 if (Subtarget.hasDQI())
20698 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT);
20699 return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE);
20700}
20701
20702SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
20703 SDLoc DL(Op);
20704 MVT VT = Op.getSimpleValueType();
20705 SDValue In = Op.getOperand(0);
20706 MVT InVT = In.getSimpleValueType();
20707 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
20708 "Invalid TRUNCATE operation");
20709
20710 // If we're called by the type legalizer, handle a few cases.
20711 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20712 if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(InVT)) {
20713 if ((InVT == MVT::v8i64 || InVT == MVT::v16i32 || InVT == MVT::v16i64) &&
20714 VT.is128BitVector() && Subtarget.hasAVX512()) {
20715 assert((InVT == MVT::v16i64 || Subtarget.hasVLX()) &&
20716 "Unexpected subtarget!");
20717 // The default behavior is to truncate one step, concatenate, and then
20718 // truncate the remainder. We'd rather produce two 64-bit results and
20719 // concatenate those.
20720 SDValue Lo, Hi;
20721 std::tie(Lo, Hi) = DAG.SplitVector(In, DL);
20722
20723 EVT LoVT, HiVT;
20724 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
20725
20726 Lo = DAG.getNode(ISD::TRUNCATE, DL, LoVT, Lo);
20727 Hi = DAG.getNode(ISD::TRUNCATE, DL, HiVT, Hi);
20728 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
20729 }
20730
20731 // Pre-AVX512 (or prefer-256bit) see if we can make use of PACKSS/PACKUS.
20732 if (!Subtarget.hasAVX512() ||
20733 (InVT.is512BitVector() && VT.is256BitVector()))
20734 if (SDValue SignPack =
20735 LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG))
20736 return SignPack;
20737
20738 // Pre-AVX512 see if we can make use of PACKSS/PACKUS.
20739 if (!Subtarget.hasAVX512())
20740 return LowerTruncateVecPack(VT, In, DL, Subtarget, DAG);
20741
20742 // Otherwise let default legalization handle it.
20743 return SDValue();
20744 }
20745
20746 if (VT.getVectorElementType() == MVT::i1)
20747 return LowerTruncateVecI1(Op, DL, DAG, Subtarget);
20748
20749 // Attempt to truncate with PACKUS/PACKSS even on AVX512 if we'd have to
20750 // concat from subvectors to use VPTRUNC etc.
20751 if (!Subtarget.hasAVX512() || isFreeToSplitVector(In.getNode(), DAG))
20752 if (SDValue SignPack =
20753 LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG))
20754 return SignPack;
20755
20756 // vpmovqb/w/d, vpmovdb/w, vpmovwb
20757 if (Subtarget.hasAVX512()) {
20758 if (InVT == MVT::v32i16 && !Subtarget.hasBWI()) {
20759 assert(VT == MVT::v32i8 && "Unexpected VT!");
20760 return splitVectorIntUnary(Op, DAG, DL);
20761 }
20762
20763 // word to byte only under BWI. Otherwise we have to promoted to v16i32
20764 // and then truncate that. But we should only do that if we haven't been
20765 // asked to avoid 512-bit vectors. The actual promotion to v16i32 will be
20766 // handled by isel patterns.
20767 if (InVT != MVT::v16i16 || Subtarget.hasBWI() ||
20768 Subtarget.canExtendTo512DQ())
20769 return Op;
20770 }
20771
20772 // Handle truncation of V256 to V128 using shuffles.
20773 assert(VT.is128BitVector() && InVT.is256BitVector() && "Unexpected types!");
20774
20775 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {
20776 // On AVX2, v4i64 -> v4i32 becomes VPERMD.
20777 if (Subtarget.hasInt256()) {
20778 static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1};
20779 In = DAG.getBitcast(MVT::v8i32, In);
20780 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask);
20781 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
20782 DAG.getIntPtrConstant(0, DL));
20783 }
20784
20785 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20786 DAG.getIntPtrConstant(0, DL));
20787 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20788 DAG.getIntPtrConstant(2, DL));
20789 static const int ShufMask[] = {0, 2, 4, 6};
20790 return DAG.getVectorShuffle(VT, DL, DAG.getBitcast(MVT::v4i32, OpLo),
20791 DAG.getBitcast(MVT::v4i32, OpHi), ShufMask);
20792 }
20793
20794 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {
20795 // On AVX2, v8i32 -> v8i16 becomes PSHUFB.
20796 if (Subtarget.hasInt256()) {
20797 // The PSHUFB mask:
20798 static const int ShufMask1[] = { 0, 1, 4, 5, 8, 9, 12, 13,
20799 -1, -1, -1, -1, -1, -1, -1, -1,
20800 16, 17, 20, 21, 24, 25, 28, 29,
20801 -1, -1, -1, -1, -1, -1, -1, -1 };
20802 In = DAG.getBitcast(MVT::v32i8, In);
20803 In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);
20804 In = DAG.getBitcast(MVT::v4i64, In);
20805
20806 static const int ShufMask2[] = {0, 2, -1, -1};
20807 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2);
20808 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
20809 DAG.getIntPtrConstant(0, DL));
20810 return DAG.getBitcast(MVT::v8i16, In);
20811 }
20812
20813 return Subtarget.hasSSE41()
20814 ? truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG)
20815 : truncateVectorWithPACKSS(VT, In, DL, Subtarget, DAG);
20816 }
20817
20818 if (VT == MVT::v16i8 && InVT == MVT::v16i16)
20819 return truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG);
20820
20821 llvm_unreachable("All 256->128 cases should have been handled above!");
20822}
20823
20824// We can leverage the specific way the "cvttps2dq/cvttpd2dq" instruction
20825// behaves on out of range inputs to generate optimized conversions.
20826static SDValue expandFP_TO_UINT_SSE(MVT VT, SDValue Src, const SDLoc &dl,
20827 SelectionDAG &DAG,
20828 const X86Subtarget &Subtarget) {
20829 MVT SrcVT = Src.getSimpleValueType();
20830 unsigned DstBits = VT.getScalarSizeInBits();
20831 assert(DstBits == 32 && "expandFP_TO_UINT_SSE - only vXi32 supported");
20832
20833 // Calculate the converted result for values in the range 0 to
20834 // 2^31-1 ("Small") and from 2^31 to 2^32-1 ("Big").
20835 SDValue Small = DAG.getNode(X86ISD::CVTTP2SI, dl, VT, Src);
20836 SDValue Big =
20837 DAG.getNode(X86ISD::CVTTP2SI, dl, VT,
20838 DAG.getNode(ISD::FSUB, dl, SrcVT, Src,
20839 DAG.getConstantFP(2147483648.0f, dl, SrcVT)));
20840
20841 // The "CVTTP2SI" instruction conveniently sets the sign bit if
20842 // and only if the value was out of range. So we can use that
20843 // as our indicator that we rather use "Big" instead of "Small".
20844 //
20845 // Use "Small" if "IsOverflown" has all bits cleared
20846 // and "0x80000000 | Big" if all bits in "IsOverflown" are set.
20847
20848 // AVX1 can't use the signsplat masking for 256-bit vectors - we have to
20849 // use the slightly slower blendv select instead.
20850 if (VT == MVT::v8i32 && !Subtarget.hasAVX2()) {
20851 SDValue Overflow = DAG.getNode(ISD::OR, dl, VT, Small, Big);
20852 return DAG.getNode(X86ISD::BLENDV, dl, VT, Small, Overflow, Small);
20853 }
20854
20855 SDValue IsOverflown =
20856 DAG.getNode(X86ISD::VSRAI, dl, VT, Small,
20857 DAG.getTargetConstant(DstBits - 1, dl, MVT::i8));
20858 return DAG.getNode(ISD::OR, dl, VT, Small,
20859 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown));
20860}
20861
20862SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
20863 bool IsStrict = Op->isStrictFPOpcode();
20864 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
20865 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
20866 MVT VT = Op->getSimpleValueType(0);
20867 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
20868 SDValue Chain = IsStrict ? Op->getOperand(0) : SDValue();
20869 MVT SrcVT = Src.getSimpleValueType();
20870 SDLoc dl(Op);
20871
20872 SDValue Res;
20873 if (isSoftF16(SrcVT, Subtarget)) {
20874 MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
20875 if (IsStrict)
20876 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},
20877 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
20878 {NVT, MVT::Other}, {Chain, Src})});
20879 return DAG.getNode(Op.getOpcode(), dl, VT,
20880 DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
20881 } else if (isTypeLegal(SrcVT) && isLegalConversion(VT, IsSigned, Subtarget)) {
20882 return Op;
20883 }
20884
20885 if (VT.isVector()) {
20886 if (VT == MVT::v2i1 && SrcVT == MVT::v2f64) {
20887 MVT ResVT = MVT::v4i32;
20888 MVT TruncVT = MVT::v4i1;
20889 unsigned Opc;
20890 if (IsStrict)
20891 Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
20892 else
20893 Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
20894
20895 if (!IsSigned && !Subtarget.hasVLX()) {
20896 assert(Subtarget.useAVX512Regs() && "Unexpected features!");
20897 // Widen to 512-bits.
20898 ResVT = MVT::v8i32;
20899 TruncVT = MVT::v8i1;
20900 Opc = Op.getOpcode();
20901 // Need to concat with zero vector for strict fp to avoid spurious
20902 // exceptions.
20903 // TODO: Should we just do this for non-strict as well?
20904 SDValue Tmp = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v8f64)
20905 : DAG.getUNDEF(MVT::v8f64);
20906 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8f64, Tmp, Src,
20907 DAG.getIntPtrConstant(0, dl));
20908 }
20909 if (IsStrict) {
20910 Res = DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {Chain, Src});
20911 Chain = Res.getValue(1);
20912 } else {
20913 Res = DAG.getNode(Opc, dl, ResVT, Src);
20914 }
20915
20916 Res = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Res);
20917 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i1, Res,
20918 DAG.getIntPtrConstant(0, dl));
20919 if (IsStrict)
20920 return DAG.getMergeValues({Res, Chain}, dl);
20921 return Res;
20922 }
20923
20924 if (Subtarget.hasFP16() && SrcVT.getVectorElementType() == MVT::f16) {
20925 if (VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16)
20926 return Op;
20927
20928 MVT ResVT = VT;
20929 MVT EleVT = VT.getVectorElementType();
20930 if (EleVT != MVT::i64)
20931 ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;
20932
20933 if (SrcVT != MVT::v8f16) {
20934 SDValue Tmp =
20935 IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT);
20936 SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);
20937 Ops[0] = Src;
20938 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);
20939 }
20940
20941 if (IsStrict) {
20942 Res = DAG.getNode(IsSigned ? X86ISD::STRICT_CVTTP2SI
20943 : X86ISD::STRICT_CVTTP2UI,
20944 dl, {ResVT, MVT::Other}, {Chain, Src});
20945 Chain = Res.getValue(1);
20946 } else {
20947 Res = DAG.getNode(IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI, dl,
20948 ResVT, Src);
20949 }
20950
20951 // TODO: Need to add exception check code for strict FP.
20952 if (EleVT.getSizeInBits() < 16) {
20953 ResVT = MVT::getVectorVT(EleVT, 8);
20954 Res = DAG.getNode(ISD::TRUNCATE, dl, ResVT, Res);
20955 }
20956
20957 if (ResVT != VT)
20958 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
20959 DAG.getIntPtrConstant(0, dl));
20960
20961 if (IsStrict)
20962 return DAG.getMergeValues({Res, Chain}, dl);
20963 return Res;
20964 }
20965
20966 // v8f32/v16f32/v8f64->v8i16/v16i16 need to widen first.
20967 if (VT.getVectorElementType() == MVT::i16) {
20968 assert((SrcVT.getVectorElementType() == MVT::f32 ||
20969 SrcVT.getVectorElementType() == MVT::f64) &&
20970 "Expected f32/f64 vector!");
20971 MVT NVT = VT.changeVectorElementType(MVT::i32);
20972 if (IsStrict) {
20973 Res = DAG.getNode(IsSigned ? ISD::STRICT_FP_TO_SINT
20974 : ISD::STRICT_FP_TO_UINT,
20975 dl, {NVT, MVT::Other}, {Chain, Src});
20976 Chain = Res.getValue(1);
20977 } else {
20978 Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl,
20979 NVT, Src);
20980 }
20981
20982 // TODO: Need to add exception check code for strict FP.
20983 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20984
20985 if (IsStrict)
20986 return DAG.getMergeValues({Res, Chain}, dl);
20987 return Res;
20988 }
20989
20990 // v8f64->v8i32 is legal, but we need v8i32 to be custom for v8f32.
20991 if (VT == MVT::v8i32 && SrcVT == MVT::v8f64) {
20992 assert(!IsSigned && "Expected unsigned conversion!");
20993 assert(Subtarget.useAVX512Regs() && "Requires avx512f");
20994 return Op;
20995 }
20996
20997 // Widen vXi32 fp_to_uint with avx512f to 512-bit source.
20998 if ((VT == MVT::v4i32 || VT == MVT::v8i32) &&
20999 (SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v8f32) &&
21000 Subtarget.useAVX512Regs()) {
21001 assert(!IsSigned && "Expected unsigned conversion!");
21002 assert(!Subtarget.hasVLX() && "Unexpected features!");
21003 MVT WideVT = SrcVT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
21004 MVT ResVT = SrcVT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
21005 // Need to concat with zero vector for strict fp to avoid spurious
21006 // exceptions.
21007 // TODO: Should we just do this for non-strict as well?
21008 SDValue Tmp =
21009 IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
21010 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
21011 DAG.getIntPtrConstant(0, dl));
21012
21013 if (IsStrict) {
21014 Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, dl, {ResVT, MVT::Other},
21015 {Chain, Src});
21016 Chain = Res.getValue(1);
21017 } else {
21018 Res = DAG.getNode(ISD::FP_TO_UINT, dl, ResVT, Src);
21019 }
21020
21021 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
21022 DAG.getIntPtrConstant(0, dl));
21023
21024 if (IsStrict)
21025 return DAG.getMergeValues({Res, Chain}, dl);
21026 return Res;
21027 }
21028
21029 // Widen vXi64 fp_to_uint/fp_to_sint with avx512dq to 512-bit source.
21030 if ((VT == MVT::v2i64 || VT == MVT::v4i64) &&
21031 (SrcVT == MVT::v2f64 || SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32) &&
21032 Subtarget.useAVX512Regs() && Subtarget.hasDQI()) {
21033 assert(!Subtarget.hasVLX() && "Unexpected features!");
21034 MVT WideVT = SrcVT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;
21035 // Need to concat with zero vector for strict fp to avoid spurious
21036 // exceptions.
21037 // TODO: Should we just do this for non-strict as well?
21038 SDValue Tmp =
21039 IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
21040 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
21041 DAG.getIntPtrConstant(0, dl));
21042
21043 if (IsStrict) {
21044 Res = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},
21045 {Chain, Src});
21046 Chain = Res.getValue(1);
21047 } else {
21048 Res = DAG.getNode(Op.getOpcode(), dl, MVT::v8i64, Src);
21049 }
21050
21051 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
21052 DAG.getIntPtrConstant(0, dl));
21053
21054 if (IsStrict)
21055 return DAG.getMergeValues({Res, Chain}, dl);
21056 return Res;
21057 }
21058
21059 if (VT == MVT::v2i64 && SrcVT == MVT::v2f32) {
21060 if (!Subtarget.hasVLX()) {
21061 // Non-strict nodes without VLX can we widened to v4f32->v4i64 by type
21062 // legalizer and then widened again by vector op legalization.
21063 if (!IsStrict)
21064 return SDValue();
21065
21066 SDValue Zero = DAG.getConstantFP(0.0, dl, MVT::v2f32);
21067 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f32,
21068 {Src, Zero, Zero, Zero});
21069 Tmp = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},
21070 {Chain, Tmp});
21071 SDValue Chain = Tmp.getValue(1);
21072 Tmp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Tmp,
21073 DAG.getIntPtrConstant(0, dl));
21074 return DAG.getMergeValues({Tmp, Chain}, dl);
21075 }
21076
21077 assert(Subtarget.hasDQI() && Subtarget.hasVLX() && "Requires AVX512DQVL");
21078 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
21079 DAG.getUNDEF(MVT::v2f32));
21080 if (IsStrict) {
21081 unsigned Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI
21082 : X86ISD::STRICT_CVTTP2UI;
21083 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp});
21084 }
21085 unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
21086 return DAG.getNode(Opc, dl, VT, Tmp);
21087 }
21088
21089 // Generate optimized instructions for pre AVX512 unsigned conversions from
21090 // vXf32 to vXi32.
21091 if ((VT == MVT::v4i32 && SrcVT == MVT::v4f32) ||
21092 (VT == MVT::v4i32 && SrcVT == MVT::v4f64) ||
21093 (VT == MVT::v8i32 && SrcVT == MVT::v8f32)) {
21094 assert(!IsSigned && "Expected unsigned conversion!");
21095 return expandFP_TO_UINT_SSE(VT, Src, dl, DAG, Subtarget);
21096 }
21097
21098 return SDValue();
21099 }
21100
21101 assert(!VT.isVector());
21102
21103 bool UseSSEReg = isScalarFPTypeInSSEReg(SrcVT);
21104
21105 if (!IsSigned && UseSSEReg) {
21106 // Conversions from f32/f64 with AVX512 should be legal.
21107 if (Subtarget.hasAVX512())
21108 return Op;
21109
21110 // We can leverage the specific way the "cvttss2si/cvttsd2si" instruction
21111 // behaves on out of range inputs to generate optimized conversions.
21112 if (!IsStrict && ((VT == MVT::i32 && !Subtarget.is64Bit()) ||
21113 (VT == MVT::i64 && Subtarget.is64Bit()))) {
21114 unsigned DstBits = VT.getScalarSizeInBits();
21115 APInt UIntLimit = APInt::getSignMask(DstBits);
21116 SDValue FloatOffset = DAG.getNode(ISD::UINT_TO_FP, dl, SrcVT,
21117 DAG.getConstant(UIntLimit, dl, VT));
21118 MVT SrcVecVT = MVT::getVectorVT(SrcVT, 128 / SrcVT.getScalarSizeInBits());
21119
21120 // Calculate the converted result for values in the range:
21121 // (i32) 0 to 2^31-1 ("Small") and from 2^31 to 2^32-1 ("Big").
21122 // (i64) 0 to 2^63-1 ("Small") and from 2^63 to 2^64-1 ("Big").
21123 SDValue Small =
21124 DAG.getNode(X86ISD::CVTTS2SI, dl, VT,
21125 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT, Src));
21126 SDValue Big = DAG.getNode(
21127 X86ISD::CVTTS2SI, dl, VT,
21128 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT,
21129 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FloatOffset)));
21130
21131 // The "CVTTS2SI" instruction conveniently sets the sign bit if
21132 // and only if the value was out of range. So we can use that
21133 // as our indicator that we rather use "Big" instead of "Small".
21134 //
21135 // Use "Small" if "IsOverflown" has all bits cleared
21136 // and "0x80000000 | Big" if all bits in "IsOverflown" are set.
21137 SDValue IsOverflown = DAG.getNode(
21138 ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8));
21139 return DAG.getNode(ISD::OR, dl, VT, Small,
21140 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown));
21141 }
21142
21143 // Use default expansion for i64.
21144 if (VT == MVT::i64)
21145 return SDValue();
21146
21147 assert(VT == MVT::i32 && "Unexpected VT!");
21148
21149 // Promote i32 to i64 and use a signed operation on 64-bit targets.
21150 // FIXME: This does not generate an invalid exception if the input does not
21151 // fit in i32. PR44019
21152 if (Subtarget.is64Bit()) {
21153 if (IsStrict) {
21154 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i64, MVT::Other},
21155 {Chain, Src});
21156 Chain = Res.getValue(1);
21157 } else
21158 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i64, Src);
21159
21160 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
21161 if (IsStrict)
21162 return DAG.getMergeValues({Res, Chain}, dl);
21163 return Res;
21164 }
21165
21166 // Use default expansion for SSE1/2 targets without SSE3. With SSE3 we can
21167 // use fisttp which will be handled later.
21168 if (!Subtarget.hasSSE3())
21169 return SDValue();
21170 }
21171
21172 // Promote i16 to i32 if we can use a SSE operation or the type is f128.
21173 // FIXME: This does not generate an invalid exception if the input does not
21174 // fit in i16. PR44019
21175 if (VT == MVT::i16 && (UseSSEReg || SrcVT == MVT::f128)) {
21176 assert(IsSigned && "Expected i16 FP_TO_UINT to have been promoted!");
21177 if (IsStrict) {
21178 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i32, MVT::Other},
21179 {Chain, Src});
21180 Chain = Res.getValue(1);
21181 } else
21182 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src);
21183
21184 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
21185 if (IsStrict)
21186 return DAG.getMergeValues({Res, Chain}, dl);
21187 return Res;
21188 }
21189
21190 // If this is a FP_TO_SINT using SSEReg we're done.
21191 if (UseSSEReg && IsSigned)
21192 return Op;
21193
21194 // fp128 needs to use a libcall.
21195 if (SrcVT == MVT::f128) {
21196 RTLIB::Libcall LC;
21197 if (IsSigned)
21198 LC = RTLIB::getFPTOSINT(SrcVT, VT);
21199 else
21200 LC = RTLIB::getFPTOUINT(SrcVT, VT);
21201
21202 MakeLibCallOptions CallOptions;
21203 std::pair<SDValue, SDValue> Tmp =
21204 makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
21205
21206 if (IsStrict)
21207 return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
21208
21209 return Tmp.first;
21210 }
21211
21212 // Fall back to X87.
21213 if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned, Chain)) {
21214 if (IsStrict)
21215 return DAG.getMergeValues({V, Chain}, dl);
21216 return V;
21217 }
21218
21219 llvm_unreachable("Expected FP_TO_INTHelper to handle all remaining cases.");
21220}
21221
21222SDValue X86TargetLowering::LowerLRINT_LLRINT(SDValue Op,
21223 SelectionDAG &DAG) const {
21224 SDValue Src = Op.getOperand(0);
21225 EVT DstVT = Op.getSimpleValueType();
21226 MVT SrcVT = Src.getSimpleValueType();
21227
21228 if (SrcVT.isVector())
21229 return DstVT.getScalarType() == MVT::i32 ? Op : SDValue();
21230
21231 if (SrcVT == MVT::f16)
21232 return SDValue();
21233
21234 // If the source is in an SSE register, the node is Legal.
21235 if (isScalarFPTypeInSSEReg(SrcVT))
21236 return Op;
21237
21238 return LRINT_LLRINTHelper(Op.getNode(), DAG);
21239}
21240
21241SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N,
21242 SelectionDAG &DAG) const {
21243 EVT DstVT = N->getValueType(0);
21244 SDValue Src = N->getOperand(0);
21245 EVT SrcVT = Src.getValueType();
21246
21247 if (SrcVT != MVT::f32 && SrcVT != MVT::f64 && SrcVT != MVT::f80) {
21248 // f16 must be promoted before using the lowering in this routine.
21249 // fp128 does not use this lowering.
21250 return SDValue();
21251 }
21252
21253 SDLoc DL(N);
21254 SDValue Chain = DAG.getEntryNode();
21255
21256 bool UseSSE = isScalarFPTypeInSSEReg(SrcVT);
21257
21258 // If we're converting from SSE, the stack slot needs to hold both types.
21259 // Otherwise it only needs to hold the DstVT.
21260 EVT OtherVT = UseSSE ? SrcVT : DstVT;
21261 SDValue StackPtr = DAG.CreateStackTemporary(DstVT, OtherVT);
21262 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
21263 MachinePointerInfo MPI =
21264 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
21265
21266 if (UseSSE) {
21267 assert(DstVT == MVT::i64 && "Invalid LRINT/LLRINT to lower!");
21268 Chain = DAG.getStore(Chain, DL, Src, StackPtr, MPI);
21269 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
21270 SDValue Ops[] = { Chain, StackPtr };
21271
21272 Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI,
21273 /*Align*/ std::nullopt,
21274 MachineMemOperand::MOLoad);
21275 Chain = Src.getValue(1);
21276 }
21277
21278 SDValue StoreOps[] = { Chain, Src, StackPtr };
21279 Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other),
21280 StoreOps, DstVT, MPI, /*Align*/ std::nullopt,
21281 MachineMemOperand::MOStore);
21282
21283 return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI);
21284}
21285
21286SDValue
21287X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const {
21288 // This is based on the TargetLowering::expandFP_TO_INT_SAT implementation,
21289 // but making use of X86 specifics to produce better instruction sequences.
21290 SDNode *Node = Op.getNode();
21291 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
21292 unsigned FpToIntOpcode = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
21293 SDLoc dl(SDValue(Node, 0));
21294 SDValue Src = Node->getOperand(0);
21295
21296 // There are three types involved here: SrcVT is the source floating point
21297 // type, DstVT is the type of the result, and TmpVT is the result of the
21298 // intermediate FP_TO_*INT operation we'll use (which may be a promotion of
21299 // DstVT).
21300 EVT SrcVT = Src.getValueType();
21301 EVT DstVT = Node->getValueType(0);
21302 EVT TmpVT = DstVT;
21303
21304 // This code is only for floats and doubles. Fall back to generic code for
21305 // anything else.
21306 if (!isScalarFPTypeInSSEReg(SrcVT) || isSoftF16(SrcVT, Subtarget))
21307 return SDValue();
21308
21309 EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
21310 unsigned SatWidth = SatVT.getScalarSizeInBits();
21311 unsigned DstWidth = DstVT.getScalarSizeInBits();
21312 unsigned TmpWidth = TmpVT.getScalarSizeInBits();
21313 assert(SatWidth <= DstWidth && SatWidth <= TmpWidth &&
21314 "Expected saturation width smaller than result width");
21315
21316 // Promote result of FP_TO_*INT to at least 32 bits.
21317 if (TmpWidth < 32) {
21318 TmpVT = MVT::i32;
21319 TmpWidth = 32;
21320 }
21321
21322 // Promote conversions to unsigned 32-bit to 64-bit, because it will allow
21323 // us to use a native signed conversion instead.
21324 if (SatWidth == 32 && !IsSigned && Subtarget.is64Bit()) {
21325 TmpVT = MVT::i64;
21326 TmpWidth = 64;
21327 }
21328
21329 // If the saturation width is smaller than the size of the temporary result,
21330 // we can always use signed conversion, which is native.
21331 if (SatWidth < TmpWidth)
21332 FpToIntOpcode = ISD::FP_TO_SINT;
21333
21334 // Determine minimum and maximum integer values and their corresponding
21335 // floating-point values.
21336 APInt MinInt, MaxInt;
21337 if (IsSigned) {
21338 MinInt = APInt::getSignedMinValue(SatWidth).sext(DstWidth);
21339 MaxInt = APInt::getSignedMaxValue(SatWidth).sext(DstWidth);
21340 } else {
21341 MinInt = APInt::getMinValue(SatWidth).zext(DstWidth);
21342 MaxInt = APInt::getMaxValue(SatWidth).zext(DstWidth);
21343 }
21344
21345 const fltSemantics &Sem = SrcVT.getFltSemantics();
21346 APFloat MinFloat(Sem);
21347 APFloat MaxFloat(Sem);
21348
21349 APFloat::opStatus MinStatus = MinFloat.convertFromAPInt(
21350 MinInt, IsSigned, APFloat::rmTowardZero);
21351 APFloat::opStatus MaxStatus = MaxFloat.convertFromAPInt(
21352 MaxInt, IsSigned, APFloat::rmTowardZero);
21353 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact)
21354 && !(MaxStatus & APFloat::opStatus::opInexact);
21355
21356 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
21357 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
21358
21359 // If the integer bounds are exactly representable as floats, emit a
21360 // min+max+fptoi sequence. Otherwise use comparisons and selects.
21361 if (AreExactFloatBounds) {
21362 if (DstVT != TmpVT) {
21363 // Clamp by MinFloat from below. If Src is NaN, propagate NaN.
21364 SDValue MinClamped = DAG.getNode(
21365 X86ISD::FMAX, dl, SrcVT, MinFloatNode, Src);
21366 // Clamp by MaxFloat from above. If Src is NaN, propagate NaN.
21367 SDValue BothClamped = DAG.getNode(
21368 X86ISD::FMIN, dl, SrcVT, MaxFloatNode, MinClamped);
21369 // Convert clamped value to integer.
21370 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, BothClamped);
21371
21372 // NaN will become INDVAL, with the top bit set and the rest zero.
21373 // Truncation will discard the top bit, resulting in zero.
21374 return DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt);
21375 }
21376
21377 // Clamp by MinFloat from below. If Src is NaN, the result is MinFloat.
21378 SDValue MinClamped = DAG.getNode(
21379 X86ISD::FMAX, dl, SrcVT, Src, MinFloatNode);
21380 // Clamp by MaxFloat from above. NaN cannot occur.
21381 SDValue BothClamped = DAG.getNode(
21382 X86ISD::FMINC, dl, SrcVT, MinClamped, MaxFloatNode);
21383 // Convert clamped value to integer.
21384 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped);
21385
21386 if (!IsSigned) {
21387 // In the unsigned case we're done, because we mapped NaN to MinFloat,
21388 // which is zero.
21389 return FpToInt;
21390 }
21391
21392 // Otherwise, select zero if Src is NaN.
21393 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21394 return DAG.getSelectCC(
21395 dl, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
21396 }
21397
21398 SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
21399 SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
21400
21401 // Result of direct conversion, which may be selected away.
21402 SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, Src);
21403
21404 if (DstVT != TmpVT) {
21405 // NaN will become INDVAL, with the top bit set and the rest zero.
21406 // Truncation will discard the top bit, resulting in zero.
21407 FpToInt = DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt);
21408 }
21409
21410 SDValue Select = FpToInt;
21411 // For signed conversions where we saturate to the same size as the
21412 // result type of the fptoi instructions, INDVAL coincides with integer
21413 // minimum, so we don't need to explicitly check it.
21414 if (!IsSigned || SatWidth != TmpVT.getScalarSizeInBits()) {
21415 // If Src ULT MinFloat, select MinInt. In particular, this also selects
21416 // MinInt if Src is NaN.
21417 Select = DAG.getSelectCC(
21418 dl, Src, MinFloatNode, MinIntNode, Select, ISD::CondCode::SETULT);
21419 }
21420
21421 // If Src OGT MaxFloat, select MaxInt.
21422 Select = DAG.getSelectCC(
21423 dl, Src, MaxFloatNode, MaxIntNode, Select, ISD::CondCode::SETOGT);
21424
21425 // In the unsigned case we are done, because we mapped NaN to MinInt, which
21426 // is already zero. The promoted case was already handled above.
21427 if (!IsSigned || DstVT != TmpVT) {
21428 return Select;
21429 }
21430
21431 // Otherwise, select 0 if Src is NaN.
21432 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
21433 return DAG.getSelectCC(
21434 dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
21435}
21436
21437SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
21438 bool IsStrict = Op->isStrictFPOpcode();
21439
21440 SDLoc DL(Op);
21441 MVT VT = Op.getSimpleValueType();
21442 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21443 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21444 MVT SVT = In.getSimpleValueType();
21445
21446 // Let f16->f80 get lowered to a libcall, except for darwin, where we should
21447 // lower it to an fp_extend via f32 (as only f16<>f32 libcalls are available)
21448 if (VT == MVT::f128 || (SVT == MVT::f16 && VT == MVT::f80 &&
21449 !Subtarget.getTargetTriple().isOSDarwin()))
21450 return SDValue();
21451
21452 if ((SVT == MVT::v8f16 && Subtarget.hasF16C()) ||
21453 (SVT == MVT::v16f16 && Subtarget.useAVX512Regs()))
21454 return Op;
21455
21456 if (SVT == MVT::f16) {
21457 if (Subtarget.hasFP16())
21458 return Op;
21459
21460 if (VT != MVT::f32) {
21461 if (IsStrict)
21462 return DAG.getNode(
21463 ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other},
21464 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, DL,
21465 {MVT::f32, MVT::Other}, {Chain, In})});
21466
21467 return DAG.getNode(ISD::FP_EXTEND, DL, VT,
21468 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, In));
21469 }
21470
21471 if (!Subtarget.hasF16C()) {
21472 if (!Subtarget.getTargetTriple().isOSDarwin())
21473 return SDValue();
21474
21475 assert(VT == MVT::f32 && SVT == MVT::f16 && "unexpected extend libcall");
21476
21477 // Need a libcall, but ABI for f16 is soft-float on MacOS.
21478 TargetLowering::CallLoweringInfo CLI(DAG);
21479 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
21480
21481 In = DAG.getBitcast(MVT::i16, In);
21482 TargetLowering::ArgListTy Args;
21483 TargetLowering::ArgListEntry Entry;
21484 Entry.Node = In;
21485 Entry.Ty = EVT(MVT::i16).getTypeForEVT(*DAG.getContext());
21486 Entry.IsSExt = false;
21487 Entry.IsZExt = true;
21488 Args.push_back(Entry);
21489
21490 SDValue Callee = DAG.getExternalSymbol(
21491 getLibcallName(RTLIB::FPEXT_F16_F32),
21492 getPointerTy(DAG.getDataLayout()));
21493 CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
21494 CallingConv::C, EVT(VT).getTypeForEVT(*DAG.getContext()), Callee,
21495 std::move(Args));
21496
21497 SDValue Res;
21498 std::tie(Res,Chain) = LowerCallTo(CLI);
21499 if (IsStrict)
21500 Res = DAG.getMergeValues({Res, Chain}, DL);
21501
21502 return Res;
21503 }
21504
21505 In = DAG.getBitcast(MVT::i16, In);
21506 In = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16,
21507 getZeroVector(MVT::v8i16, Subtarget, DAG, DL), In,
21508 DAG.getIntPtrConstant(0, DL));
21509 SDValue Res;
21510 if (IsStrict) {
21511 Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, DL, {MVT::v4f32, MVT::Other},
21512 {Chain, In});
21513 Chain = Res.getValue(1);
21514 } else {
21515 Res = DAG.getNode(X86ISD::CVTPH2PS, DL, MVT::v4f32, In,
21516 DAG.getTargetConstant(4, DL, MVT::i32));
21517 }
21518 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Res,
21519 DAG.getIntPtrConstant(0, DL));
21520 if (IsStrict)
21521 return DAG.getMergeValues({Res, Chain}, DL);
21522 return Res;
21523 }
21524
21525 if (!SVT.isVector() || SVT.getVectorElementType() == MVT::bf16)
21526 return Op;
21527
21528 if (SVT.getVectorElementType() == MVT::f16) {
21529 if (Subtarget.hasFP16() && isTypeLegal(SVT))
21530 return Op;
21531 assert(Subtarget.hasF16C() && "Unexpected features!");
21532 if (SVT == MVT::v2f16)
21533 In = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f16, In,
21534 DAG.getUNDEF(MVT::v2f16));
21535 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f16, In,
21536 DAG.getUNDEF(MVT::v4f16));
21537 if (IsStrict)
21538 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
21539 {Op->getOperand(0), Res});
21540 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
21541 } else if (VT == MVT::v4f64 || VT == MVT::v8f64) {
21542 return Op;
21543 }
21544
21545 assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!");
21546
21547 SDValue Res =
21548 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, In, DAG.getUNDEF(SVT));
21549 if (IsStrict)
21550 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
21551 {Op->getOperand(0), Res});
21552 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
21553}
21554
21555SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
21556 bool IsStrict = Op->isStrictFPOpcode();
21557
21558 SDLoc DL(Op);
21559 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21560 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
21561 MVT VT = Op.getSimpleValueType();
21562 MVT SVT = In.getSimpleValueType();
21563
21564 if (SVT == MVT::f128 || (VT == MVT::f16 && SVT == MVT::f80))
21565 return SDValue();
21566
21567 if (VT == MVT::f16 && (SVT == MVT::f64 || SVT == MVT::f32) &&
21568 !Subtarget.hasFP16() && (SVT == MVT::f64 || !Subtarget.hasF16C())) {
21569 if (!Subtarget.getTargetTriple().isOSDarwin())
21570 return SDValue();
21571
21572 // We need a libcall but the ABI for f16 libcalls on MacOS is soft.
21573 TargetLowering::CallLoweringInfo CLI(DAG);
21574 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
21575
21576 TargetLowering::ArgListTy Args;
21577 TargetLowering::ArgListEntry Entry;
21578 Entry.Node = In;
21579 Entry.Ty = EVT(SVT).getTypeForEVT(*DAG.getContext());
21580 Entry.IsSExt = false;
21581 Entry.IsZExt = true;
21582 Args.push_back(Entry);
21583
21584 SDValue Callee = DAG.getExternalSymbol(
21585 getLibcallName(SVT == MVT::f64 ? RTLIB::FPROUND_F64_F16
21586 : RTLIB::FPROUND_F32_F16),
21587 getPointerTy(DAG.getDataLayout()));
21588 CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
21589 CallingConv::C, EVT(MVT::i16).getTypeForEVT(*DAG.getContext()), Callee,
21590 std::move(Args));
21591
21592 SDValue Res;
21593 std::tie(Res, Chain) = LowerCallTo(CLI);
21594
21595 Res = DAG.getBitcast(MVT::f16, Res);
21596
21597 if (IsStrict)
21598 Res = DAG.getMergeValues({Res, Chain}, DL);
21599
21600 return Res;
21601 }
21602
21603 if (VT.getScalarType() == MVT::bf16) {
21604 if (SVT.getScalarType() == MVT::f32 &&
21605 ((Subtarget.hasBF16() && Subtarget.hasVLX()) ||
21606 Subtarget.hasAVXNECONVERT()))
21607 return Op;
21608 return SDValue();
21609 }
21610
21611 if (VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16()) {
21612 if (!Subtarget.hasF16C() || SVT.getScalarType() != MVT::f32)
21613 return SDValue();
21614
21615 if (VT.isVector())
21616 return Op;
21617
21618 SDValue Res;
21619 SDValue Rnd = DAG.getTargetConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, DL,
21620 MVT::i32);
21621 if (IsStrict) {
21622 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4f32,
21623 DAG.getConstantFP(0, DL, MVT::v4f32), In,
21624 DAG.getIntPtrConstant(0, DL));
21625 Res = DAG.getNode(X86ISD::STRICT_CVTPS2PH, DL, {MVT::v8i16, MVT::Other},
21626 {Chain, Res, Rnd});
21627 Chain = Res.getValue(1);
21628 } else {
21629 // FIXME: Should we use zeros for upper elements for non-strict?
21630 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, In);
21631 Res = DAG.getNode(X86ISD::CVTPS2PH, DL, MVT::v8i16, Res, Rnd);
21632 }
21633
21634 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res,
21635 DAG.getIntPtrConstant(0, DL));
21636 Res = DAG.getBitcast(MVT::f16, Res);
21637
21638 if (IsStrict)
21639 return DAG.getMergeValues({Res, Chain}, DL);
21640
21641 return Res;
21642 }
21643
21644 return Op;
21645}
21646
21647static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) {
21648 bool IsStrict = Op->isStrictFPOpcode();
21649 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
21650 assert(Src.getValueType() == MVT::i16 && Op.getValueType() == MVT::f32 &&
21651 "Unexpected VT!");
21652
21653 SDLoc dl(Op);
21654 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16,
21655 DAG.getConstant(0, dl, MVT::v8i16), Src,
21656 DAG.getIntPtrConstant(0, dl));
21657
21658 SDValue Chain;
21659 if (IsStrict) {
21660 Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {MVT::v4f32, MVT::Other},
21661 {Op.getOperand(0), Res});
21662 Chain = Res.getValue(1);
21663 } else {
21664 Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
21665 }
21666
21667 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
21668 DAG.getIntPtrConstant(0, dl));
21669
21670 if (IsStrict)
21671 return DAG.getMergeValues({Res, Chain}, dl);
21672
21673 return Res;
21674}
21675
21676static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) {
21677 bool IsStrict = Op->isStrictFPOpcode();
21678 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
21679 assert(Src.getValueType() == MVT::f32 && Op.getValueType() == MVT::i16 &&
21680 "Unexpected VT!");
21681
21682 SDLoc dl(Op);
21683 SDValue Res, Chain;
21684 if (IsStrict) {
21685 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v4f32,
21686 DAG.getConstantFP(0, dl, MVT::v4f32), Src,
21687 DAG.getIntPtrConstant(0, dl));
21688 Res = DAG.getNode(
21689 X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},
21690 {Op.getOperand(0), Res, DAG.getTargetConstant(4, dl, MVT::i32)});
21691 Chain = Res.getValue(1);
21692 } else {
21693 // FIXME: Should we use zeros for upper elements for non-strict?
21694 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, Src);
21695 Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
21696 DAG.getTargetConstant(4, dl, MVT::i32));
21697 }
21698
21699 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Res,
21700 DAG.getIntPtrConstant(0, dl));
21701
21702 if (IsStrict)
21703 return DAG.getMergeValues({Res, Chain}, dl);
21704
21705 return Res;
21706}
21707
21708SDValue X86TargetLowering::LowerFP_TO_BF16(SDValue Op,
21709 SelectionDAG &DAG) const {
21710 SDLoc DL(Op);
21711
21712 MVT SVT = Op.getOperand(0).getSimpleValueType();
21713 if (SVT == MVT::f32 && ((Subtarget.hasBF16() && Subtarget.hasVLX()) ||
21714 Subtarget.hasAVXNECONVERT())) {
21715 SDValue Res;
21716 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, Op.getOperand(0));
21717 Res = DAG.getNode(X86ISD::CVTNEPS2BF16, DL, MVT::v8bf16, Res);
21718 Res = DAG.getBitcast(MVT::v8i16, Res);
21719 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res,
21720 DAG.getIntPtrConstant(0, DL));
21721 }
21722
21723 MakeLibCallOptions CallOptions;
21724 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, MVT::bf16);
21725 SDValue Res =
21726 makeLibCall(DAG, LC, MVT::f16, Op.getOperand(0), CallOptions, DL).first;
21727 return DAG.getBitcast(MVT::i16, Res);
21728}
21729
21730/// Depending on uarch and/or optimizing for size, we might prefer to use a
21731/// vector operation in place of the typical scalar operation.
21732static SDValue lowerAddSubToHorizontalOp(SDValue Op, const SDLoc &DL,
21733 SelectionDAG &DAG,
21734 const X86Subtarget &Subtarget) {
21735 // If both operands have other uses, this is probably not profitable.
21736 SDValue LHS = Op.getOperand(0);
21737 SDValue RHS = Op.getOperand(1);
21738 if (!LHS.hasOneUse() && !RHS.hasOneUse())
21739 return Op;
21740
21741 // FP horizontal add/sub were added with SSE3. Integer with SSSE3.
21742 bool IsFP = Op.getSimpleValueType().isFloatingPoint();
21743 if (IsFP && !Subtarget.hasSSE3())
21744 return Op;
21745 if (!IsFP && !Subtarget.hasSSSE3())
21746 return Op;
21747
21748 // Extract from a common vector.
21749 if (LHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
21750 RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
21751 LHS.getOperand(0) != RHS.getOperand(0) ||
21752 !isa<ConstantSDNode>(LHS.getOperand(1)) ||
21753 !isa<ConstantSDNode>(RHS.getOperand(1)) ||
21754 !shouldUseHorizontalOp(true, DAG, Subtarget))
21755 return Op;
21756
21757 // Allow commuted 'hadd' ops.
21758 // TODO: Allow commuted (f)sub by negating the result of (F)HSUB?
21759 unsigned HOpcode;
21760 switch (Op.getOpcode()) {
21761 // clang-format off
21762 case ISD::ADD: HOpcode = X86ISD::HADD; break;
21763 case ISD::SUB: HOpcode = X86ISD::HSUB; break;
21764 case ISD::FADD: HOpcode = X86ISD::FHADD; break;
21765 case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
21766 default:
21767 llvm_unreachable("Trying to lower unsupported opcode to horizontal op");
21768 // clang-format on
21769 }
21770 unsigned LExtIndex = LHS.getConstantOperandVal(1);
21771 unsigned RExtIndex = RHS.getConstantOperandVal(1);
21772 if ((LExtIndex & 1) == 1 && (RExtIndex & 1) == 0 &&
21773 (HOpcode == X86ISD::HADD || HOpcode == X86ISD::FHADD))
21774 std::swap(LExtIndex, RExtIndex);
21775
21776 if ((LExtIndex & 1) != 0 || RExtIndex != (LExtIndex + 1))
21777 return Op;
21778
21779 SDValue X = LHS.getOperand(0);
21780 EVT VecVT = X.getValueType();
21781 unsigned BitWidth = VecVT.getSizeInBits();
21782 unsigned NumLanes = BitWidth / 128;
21783 unsigned NumEltsPerLane = VecVT.getVectorNumElements() / NumLanes;
21784 assert((BitWidth == 128 || BitWidth == 256 || BitWidth == 512) &&
21785 "Not expecting illegal vector widths here");
21786
21787 // Creating a 256-bit horizontal op would be wasteful, and there is no 512-bit
21788 // equivalent, so extract the 256/512-bit source op to 128-bit if we can.
21789 if (BitWidth == 256 || BitWidth == 512) {
21790 unsigned LaneIdx = LExtIndex / NumEltsPerLane;
21791 X = extract128BitVector(X, LaneIdx * NumEltsPerLane, DAG, DL);
21792 LExtIndex %= NumEltsPerLane;
21793 }
21794
21795 // add (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hadd X, X), 0
21796 // add (extractelt (X, 1), extractelt (X, 0)) --> extractelt (hadd X, X), 0
21797 // add (extractelt (X, 2), extractelt (X, 3)) --> extractelt (hadd X, X), 1
21798 // sub (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hsub X, X), 0
21799 SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
21800 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp,
21801 DAG.getIntPtrConstant(LExtIndex / 2, DL));
21802}
21803
21804/// Depending on uarch and/or optimizing for size, we might prefer to use a
21805/// vector operation in place of the typical scalar operation.
21806SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
21807 assert((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) &&
21808 "Only expecting float/double");
21809 return lowerAddSubToHorizontalOp(Op, SDLoc(Op), DAG, Subtarget);
21810}
21811
21812/// ISD::FROUND is defined to round to nearest with ties rounding away from 0.
21813/// This mode isn't supported in hardware on X86. But as long as we aren't
21814/// compiling with trapping math, we can emulate this with
21815/// trunc(X + copysign(nextafter(0.5, 0.0), X)).
21816static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) {
21817 SDValue N0 = Op.getOperand(0);
21818 SDLoc dl(Op);
21819 MVT VT = Op.getSimpleValueType();
21820
21821 // N0 += copysign(nextafter(0.5, 0.0), N0)
21822 const fltSemantics &Sem = VT.getFltSemantics();
21823 bool Ignored;
21824 APFloat Point5Pred = APFloat(0.5f);
21825 Point5Pred.convert(Sem, APFloat::rmNearestTiesToEven, &Ignored);
21826 Point5Pred.next(/*nextDown*/true);
21827
21828 SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT,
21829 DAG.getConstantFP(Point5Pred, dl, VT), N0);
21830 N0 = DAG.getNode(ISD::FADD, dl, VT, N0, Adder);
21831
21832 // Truncate the result to remove fraction.
21833 return DAG.getNode(ISD::FTRUNC, dl, VT, N0);
21834}
21835
21836/// The only differences between FABS and FNEG are the mask and the logic op.
21837/// FNEG also has a folding opportunity for FNEG(FABS(x)).
21838static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
21839 assert((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) &&
21840 "Wrong opcode for lowering FABS or FNEG.");
21841
21842 bool IsFABS = (Op.getOpcode() == ISD::FABS);
21843
21844 // If this is a FABS and it has an FNEG user, bail out to fold the combination
21845 // into an FNABS. We'll lower the FABS after that if it is still in use.
21846 if (IsFABS)
21847 for (SDNode *User : Op->uses())
21848 if (User->getOpcode() == ISD::FNEG)
21849 return Op;
21850
21851 SDLoc dl(Op);
21852 MVT VT = Op.getSimpleValueType();
21853
21854 bool IsF128 = (VT == MVT::f128);
21855 assert(VT.isFloatingPoint() && VT != MVT::f80 &&
21856 DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
21857 "Unexpected type in LowerFABSorFNEG");
21858
21859 // FIXME: Use function attribute "OptimizeForSize" and/or CodeGenOptLevel to
21860 // decide if we should generate a 16-byte constant mask when we only need 4 or
21861 // 8 bytes for the scalar case.
21862
21863 // There are no scalar bitwise logical SSE/AVX instructions, so we
21864 // generate a 16-byte vector constant and logic op even for the scalar case.
21865 // Using a 16-byte mask allows folding the load of the mask with
21866 // the logic op, so it can save (~4 bytes) on code size.
21867 bool IsFakeVector = !VT.isVector() && !IsF128;
21868 MVT LogicVT = VT;
21869 if (IsFakeVector)
21870 LogicVT = (VT == MVT::f64) ? MVT::v2f64
21871 : (VT == MVT::f32) ? MVT::v4f32
21872 : MVT::v8f16;
21873
21874 unsigned EltBits = VT.getScalarSizeInBits();
21875 // For FABS, mask is 0x7f...; for FNEG, mask is 0x80...
21876 APInt MaskElt = IsFABS ? APInt::getSignedMaxValue(EltBits) :
21877 APInt::getSignMask(EltBits);
21878 const fltSemantics &Sem = VT.getFltSemantics();
21879 SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
21880
21881 SDValue Op0 = Op.getOperand(0);
21882 bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
21883 unsigned LogicOp = IsFABS ? X86ISD::FAND :
21884 IsFNABS ? X86ISD::FOR :
21885 X86ISD::FXOR;
21886 SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
21887
21888 if (VT.isVector() || IsF128)
21889 return DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
21890
21891 // For the scalar case extend to a 128-bit vector, perform the logic op,
21892 // and extract the scalar result back out.
21893 Operand = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Operand);
21894 SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
21895 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode,
21896 DAG.getIntPtrConstant(0, dl));
21897}
21898
21899static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
21900 SDValue Mag = Op.getOperand(0);
21901 SDValue Sign = Op.getOperand(1);
21902 SDLoc dl(Op);
21903
21904 // If the sign operand is smaller, extend it first.
21905 MVT VT = Op.getSimpleValueType();
21906 if (Sign.getSimpleValueType().bitsLT(VT))
21907 Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign);
21908
21909 // And if it is bigger, shrink it first.
21910 if (Sign.getSimpleValueType().bitsGT(VT))
21911 Sign = DAG.getNode(ISD::FP_ROUND, dl, VT, Sign,
21912 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
21913
21914 // At this point the operands and the result should have the same
21915 // type, and that won't be f80 since that is not custom lowered.
21916 bool IsF128 = (VT == MVT::f128);
21917 assert(VT.isFloatingPoint() && VT != MVT::f80 &&
21918 DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
21919 "Unexpected type in LowerFCOPYSIGN");
21920
21921 const fltSemantics &Sem = VT.getFltSemantics();
21922
21923 // Perform all scalar logic operations as 16-byte vectors because there are no
21924 // scalar FP logic instructions in SSE.
21925 // TODO: This isn't necessary. If we used scalar types, we might avoid some
21926 // unnecessary splats, but we might miss load folding opportunities. Should
21927 // this decision be based on OptimizeForSize?
21928 bool IsFakeVector = !VT.isVector() && !IsF128;
21929 MVT LogicVT = VT;
21930 if (IsFakeVector)
21931 LogicVT = (VT == MVT::f64) ? MVT::v2f64
21932 : (VT == MVT::f32) ? MVT::v4f32
21933 : MVT::v8f16;
21934
21935 // The mask constants are automatically splatted for vector types.
21936 unsigned EltSizeInBits = VT.getScalarSizeInBits();
21937 SDValue SignMask = DAG.getConstantFP(
21938 APFloat(Sem, APInt::getSignMask(EltSizeInBits)), dl, LogicVT);
21939 SDValue MagMask = DAG.getConstantFP(
21940 APFloat(Sem, APInt::getSignedMaxValue(EltSizeInBits)), dl, LogicVT);
21941
21942 // First, clear all bits but the sign bit from the second operand (sign).
21943 if (IsFakeVector)
21944 Sign = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Sign);
21945 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask);
21946
21947 // Next, clear the sign bit from the first operand (magnitude).
21948 // TODO: If we had general constant folding for FP logic ops, this check
21949 // wouldn't be necessary.
21950 SDValue MagBits;
21951 if (ConstantFPSDNode *Op0CN = isConstOrConstSplatFP(Mag)) {
21952 APFloat APF = Op0CN->getValueAPF();
21953 APF.clearSign();
21954 MagBits = DAG.getConstantFP(APF, dl, LogicVT);
21955 } else {
21956 // If the magnitude operand wasn't a constant, we need to AND out the sign.
21957 if (IsFakeVector)
21958 Mag = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Mag);
21959 MagBits = DAG.getNode(X86ISD::FAND, dl, LogicVT, Mag, MagMask);
21960 }
21961
21962 // OR the magnitude value with the sign bit.
21963 SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit);
21964 return !IsFakeVector ? Or : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or,
21965 DAG.getIntPtrConstant(0, dl));
21966}
21967
21968static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
21969 SDValue N0 = Op.getOperand(0);
21970 SDLoc dl(Op);
21971 MVT VT = Op.getSimpleValueType();
21972
21973 MVT OpVT = N0.getSimpleValueType();
21974 assert((OpVT == MVT::f32 || OpVT == MVT::f64) &&
21975 "Unexpected type for FGETSIGN");
21976
21977 // Lower ISD::FGETSIGN to (AND (X86ISD::MOVMSK ...) 1).
21978 MVT VecVT = (OpVT == MVT::f32 ? MVT::v4f32 : MVT::v2f64);
21979 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0);
21980 Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res);
21981 Res = DAG.getZExtOrTrunc(Res, dl, VT);
21982 Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT));
21983 return Res;
21984}
21985
21986/// Helper for attempting to create a X86ISD::BT node.
21987static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG) {
21988 // If Src is i8, promote it to i32 with any_extend. There is no i8 BT
21989 // instruction. Since the shift amount is in-range-or-undefined, we know
21990 // that doing a bittest on the i32 value is ok. We extend to i32 because
21991 // the encoding for the i16 version is larger than the i32 version.
21992 // Also promote i16 to i32 for performance / code size reason.
21993 if (Src.getValueType().getScalarSizeInBits() < 32)
21994 Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src);
21995
21996 // No legal type found, give up.
21997 if (!DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType()))
21998 return SDValue();
21999
22000 // See if we can use the 32-bit instruction instead of the 64-bit one for a
22001 // shorter encoding. Since the former takes the modulo 32 of BitNo and the
22002 // latter takes the modulo 64, this is only valid if the 5th bit of BitNo is
22003 // known to be zero.
22004 if (Src.getValueType() == MVT::i64 &&
22005 DAG.MaskedValueIsZero(BitNo, APInt(BitNo.getValueSizeInBits(), 32)))
22006 Src = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Src);
22007
22008 // If the operand types disagree, extend the shift amount to match. Since
22009 // BT ignores high bits (like shifts) we can use anyextend.
22010 if (Src.getValueType() != BitNo.getValueType()) {
22011 // Peek through a mask/modulo operation.
22012 // TODO: DAGCombine fails to do this as it just checks isTruncateFree, but
22013 // we probably need a better IsDesirableToPromoteOp to handle this as well.
22014 if (BitNo.getOpcode() == ISD::AND && BitNo->hasOneUse())
22015 BitNo = DAG.getNode(ISD::AND, DL, Src.getValueType(),
22016 DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(),
22017 BitNo.getOperand(0)),
22018 DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(),
22019 BitNo.getOperand(1)));
22020 else
22021 BitNo = DAG.getNode(ISD::ANY_EXTEND, DL, Src.getValueType(), BitNo);
22022 }
22023
22024 return DAG.getNode(X86ISD::BT, DL, MVT::i32, Src, BitNo);
22025}
22026
22027/// Helper for creating a X86ISD::SETCC node.
22028static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
22029 SelectionDAG &DAG) {
22030 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
22031 DAG.getTargetConstant(Cond, dl, MVT::i8), EFLAGS);
22032}
22033
22034/// Recursive helper for combineVectorSizedSetCCEquality() to see if we have a
22035/// recognizable memcmp expansion.
22036static bool isOrXorXorTree(SDValue X, bool Root = true) {
22037 if (X.getOpcode() == ISD::OR)
22038 return isOrXorXorTree(X.getOperand(0), false) &&
22039 isOrXorXorTree(X.getOperand(1), false);
22040 if (Root)
22041 return false;
22042 return X.getOpcode() == ISD::XOR;
22043}
22044
22045/// Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp
22046/// expansion.
22047template <typename F>
22048static SDValue emitOrXorXorTree(SDValue X, const SDLoc &DL, SelectionDAG &DAG,
22049 EVT VecVT, EVT CmpVT, bool HasPT, F SToV) {
22050 SDValue Op0 = X.getOperand(0);
22051 SDValue Op1 = X.getOperand(1);
22052 if (X.getOpcode() == ISD::OR) {
22053 SDValue A = emitOrXorXorTree(Op0, DL, DAG, VecVT, CmpVT, HasPT, SToV);
22054 SDValue B = emitOrXorXorTree(Op1, DL, DAG, VecVT, CmpVT, HasPT, SToV);
22055 if (VecVT != CmpVT)
22056 return DAG.getNode(ISD::OR, DL, CmpVT, A, B);
22057 if (HasPT)
22058 return DAG.getNode(ISD::OR, DL, VecVT, A, B);
22059 return DAG.getNode(ISD::AND, DL, CmpVT, A, B);
22060 }
22061 if (X.getOpcode() == ISD::XOR) {
22062 SDValue A = SToV(Op0);
22063 SDValue B = SToV(Op1);
22064 if (VecVT != CmpVT)
22065 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETNE);
22066 if (HasPT)
22067 return DAG.getNode(ISD::XOR, DL, VecVT, A, B);
22068 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETEQ);
22069 }
22070 llvm_unreachable("Impossible");
22071}
22072
22073/// Try to map a 128-bit or larger integer comparison to vector instructions
22074/// before type legalization splits it up into chunks.
22075static SDValue combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y,
22076 ISD::CondCode CC,
22077 const SDLoc &DL,
22078 SelectionDAG &DAG,
22079 const X86Subtarget &Subtarget) {
22080 assert((CC == ISD::SETNE || CC == ISD::SETEQ) && "Bad comparison predicate");
22081
22082 // We're looking for an oversized integer equality comparison.
22083 EVT OpVT = X.getValueType();
22084 unsigned OpSize = OpVT.getSizeInBits();
22085 if (!OpVT.isScalarInteger() || OpSize < 128)
22086 return SDValue();
22087
22088 // Ignore a comparison with zero because that gets special treatment in
22089 // EmitTest(). But make an exception for the special case of a pair of
22090 // logically-combined vector-sized operands compared to zero. This pattern may
22091 // be generated by the memcmp expansion pass with oversized integer compares
22092 // (see PR33325).
22093 bool IsOrXorXorTreeCCZero = isNullConstant(Y) && isOrXorXorTree(X);
22094 if (isNullConstant(Y) && !IsOrXorXorTreeCCZero)
22095 return SDValue();
22096
22097 // Don't perform this combine if constructing the vector will be expensive.
22098 auto IsVectorBitCastCheap = [](SDValue X) {
22099 X = peekThroughBitcasts(X);
22100 return isa<ConstantSDNode>(X) || X.getValueType().isVector() ||
22101 X.getOpcode() == ISD::LOAD;
22102 };
22103 if ((!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y)) &&
22104 !IsOrXorXorTreeCCZero)
22105 return SDValue();
22106
22107 // Use XOR (plus OR) and PTEST after SSE4.1 for 128/256-bit operands.
22108 // Use PCMPNEQ (plus OR) and KORTEST for 512-bit operands.
22109 // Otherwise use PCMPEQ (plus AND) and mask testing.
22110 bool NoImplicitFloatOps =
22111 DAG.getMachineFunction().getFunction().hasFnAttribute(
22112 Attribute::NoImplicitFloat);
22113 if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
22114 ((OpSize == 128 && Subtarget.hasSSE2()) ||
22115 (OpSize == 256 && Subtarget.hasAVX()) ||
22116 (OpSize == 512 && Subtarget.useAVX512Regs()))) {
22117 bool HasPT = Subtarget.hasSSE41();
22118
22119 // PTEST and MOVMSK are slow on Knights Landing and Knights Mill and widened
22120 // vector registers are essentially free. (Technically, widening registers
22121 // prevents load folding, but the tradeoff is worth it.)
22122 bool PreferKOT = Subtarget.preferMaskRegisters();
22123 bool NeedZExt = PreferKOT && !Subtarget.hasVLX() && OpSize != 512;
22124
22125 EVT VecVT = MVT::v16i8;
22126 EVT CmpVT = PreferKOT ? MVT::v16i1 : VecVT;
22127 if (OpSize == 256) {
22128 VecVT = MVT::v32i8;
22129 CmpVT = PreferKOT ? MVT::v32i1 : VecVT;
22130 }
22131 EVT CastVT = VecVT;
22132 bool NeedsAVX512FCast = false;
22133 if (OpSize == 512 || NeedZExt) {
22134 if (Subtarget.hasBWI()) {
22135 VecVT = MVT::v64i8;
22136 CmpVT = MVT::v64i1;
22137 if (OpSize == 512)
22138 CastVT = VecVT;
22139 } else {
22140 VecVT = MVT::v16i32;
22141 CmpVT = MVT::v16i1;
22142 CastVT = OpSize == 512 ? VecVT
22143 : OpSize == 256 ? MVT::v8i32
22144 : MVT::v4i32;
22145 NeedsAVX512FCast = true;
22146 }
22147 }
22148
22149 auto ScalarToVector = [&](SDValue X) -> SDValue {
22150 bool TmpZext = false;
22151 EVT TmpCastVT = CastVT;
22152 if (X.getOpcode() == ISD::ZERO_EXTEND) {
22153 SDValue OrigX = X.getOperand(0);
22154 unsigned OrigSize = OrigX.getScalarValueSizeInBits();
22155 if (OrigSize < OpSize) {
22156 if (OrigSize == 128) {
22157 TmpCastVT = NeedsAVX512FCast ? MVT::v4i32 : MVT::v16i8;
22158 X = OrigX;
22159 TmpZext = true;
22160 } else if (OrigSize == 256) {
22161 TmpCastVT = NeedsAVX512FCast ? MVT::v8i32 : MVT::v32i8;
22162 X = OrigX;
22163 TmpZext = true;
22164 }
22165 }
22166 }
22167 X = DAG.getBitcast(TmpCastVT, X);
22168 if (!NeedZExt && !TmpZext)
22169 return X;
22170 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT,
22171 DAG.getConstant(0, DL, VecVT), X,
22172 DAG.getVectorIdxConstant(0, DL));
22173 };
22174
22175 SDValue Cmp;
22176 if (IsOrXorXorTreeCCZero) {
22177 // This is a bitwise-combined equality comparison of 2 pairs of vectors:
22178 // setcc i128 (or (xor A, B), (xor C, D)), 0, eq|ne
22179 // Use 2 vector equality compares and 'and' the results before doing a
22180 // MOVMSK.
22181 Cmp = emitOrXorXorTree(X, DL, DAG, VecVT, CmpVT, HasPT, ScalarToVector);
22182 } else {
22183 SDValue VecX = ScalarToVector(X);
22184 SDValue VecY = ScalarToVector(Y);
22185 if (VecVT != CmpVT) {
22186 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
22187 } else if (HasPT) {
22188 Cmp = DAG.getNode(ISD::XOR, DL, VecVT, VecX, VecY);
22189 } else {
22190 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETEQ);
22191 }
22192 }
22193 // AVX512 should emit a setcc that will lower to kortest.
22194 if (VecVT != CmpVT) {
22195 EVT KRegVT = CmpVT == MVT::v64i1 ? MVT::i64
22196 : CmpVT == MVT::v32i1 ? MVT::i32
22197 : MVT::i16;
22198 return DAG.getSetCC(DL, VT, DAG.getBitcast(KRegVT, Cmp),
22199 DAG.getConstant(0, DL, KRegVT), CC);
22200 }
22201 if (HasPT) {
22202 SDValue BCCmp =
22203 DAG.getBitcast(OpSize == 256 ? MVT::v4i64 : MVT::v2i64, Cmp);
22204 SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);
22205 X86::CondCode X86CC = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
22206 SDValue X86SetCC = getSETCC(X86CC, PT, DL, DAG);
22207 return DAG.getNode(ISD::TRUNCATE, DL, VT, X86SetCC.getValue(0));
22208 }
22209 // If all bytes match (bitmask is 0x(FFFF)FFFF), that's equality.
22210 // setcc i128 X, Y, eq --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, eq
22211 // setcc i128 X, Y, ne --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, ne
22212 assert(Cmp.getValueType() == MVT::v16i8 &&
22213 "Non 128-bit vector on pre-SSE41 target");
22214 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
22215 SDValue FFFFs = DAG.getConstant(0xFFFF, DL, MVT::i32);
22216 return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC);
22217 }
22218
22219 return SDValue();
22220}
22221
22222/// Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...))
22223/// style scalarized (associative) reduction patterns. Partial reductions
22224/// are supported when the pointer SrcMask is non-null.
22225/// TODO - move this to SelectionDAG?
22226static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp,
22227 SmallVectorImpl<SDValue> &SrcOps,
22228 SmallVectorImpl<APInt> *SrcMask = nullptr) {
22229 SmallVector<SDValue, 8> Opnds;
22230 DenseMap<SDValue, APInt> SrcOpMap;
22231 EVT VT = MVT::Other;
22232
22233 // Recognize a special case where a vector is casted into wide integer to
22234 // test all 0s.
22235 assert(Op.getOpcode() == unsigned(BinOp) &&
22236 "Unexpected bit reduction opcode");
22237 Opnds.push_back(Op.getOperand(0));
22238 Opnds.push_back(Op.getOperand(1));
22239
22240 for (unsigned Slot = 0, e = Opnds.size(); Slot < e; ++Slot) {
22241 SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
22242 // BFS traverse all BinOp operands.
22243 if (I->getOpcode() == unsigned(BinOp)) {
22244 Opnds.push_back(I->getOperand(0));
22245 Opnds.push_back(I->getOperand(1));
22246 // Re-evaluate the number of nodes to be traversed.
22247 e += 2; // 2 more nodes (LHS and RHS) are pushed.
22248 continue;
22249 }
22250
22251 // Quit if a non-EXTRACT_VECTOR_ELT
22252 if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
22253 return false;
22254
22255 // Quit if without a constant index.
22256 auto *Idx = dyn_cast<ConstantSDNode>(I->getOperand(1));
22257 if (!Idx)
22258 return false;
22259
22260 SDValue Src = I->getOperand(0);
22261 DenseMap<SDValue, APInt>::iterator M = SrcOpMap.find(Src);
22262 if (M == SrcOpMap.end()) {
22263 VT = Src.getValueType();
22264 // Quit if not the same type.
22265 if (!SrcOpMap.empty() && VT != SrcOpMap.begin()->first.getValueType())
22266 return false;
22267 unsigned NumElts = VT.getVectorNumElements();
22268 APInt EltCount = APInt::getZero(NumElts);
22269 M = SrcOpMap.insert(std::make_pair(Src, EltCount)).first;
22270 SrcOps.push_back(Src);
22271 }
22272
22273 // Quit if element already used.
22274 unsigned CIdx = Idx->getZExtValue();
22275 if (M->second[CIdx])
22276 return false;
22277 M->second.setBit(CIdx);
22278 }
22279
22280 if (SrcMask) {
22281 // Collect the source partial masks.
22282 for (SDValue &SrcOp : SrcOps)
22283 SrcMask->push_back(SrcOpMap[SrcOp]);
22284 } else {
22285 // Quit if not all elements are used.
22286 for (const auto &I : SrcOpMap)
22287 if (!I.second.isAllOnes())
22288 return false;
22289 }
22290
22291 return true;
22292}
22293
22294// Helper function for comparing all bits of two vectors.
22295static SDValue LowerVectorAllEqual(const SDLoc &DL, SDValue LHS, SDValue RHS,
22296 ISD::CondCode CC, const APInt &OriginalMask,
22297 const X86Subtarget &Subtarget,
22298 SelectionDAG &DAG, X86::CondCode &X86CC) {
22299 EVT VT = LHS.getValueType();
22300 unsigned ScalarSize = VT.getScalarSizeInBits();
22301 if (OriginalMask.getBitWidth() != ScalarSize) {
22302 assert(ScalarSize == 1 && "Element Mask vs Vector bitwidth mismatch");
22303 return SDValue();
22304 }
22305
22306 // Quit if not convertable to legal scalar or 128/256-bit vector.
22307 if (!llvm::has_single_bit<uint32_t>(VT.getSizeInBits()))
22308 return SDValue();
22309
22310 // FCMP may use ISD::SETNE when nnan - early out if we manage to get here.
22311 if (VT.isFloatingPoint())
22312 return SDValue();
22313
22314 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
22315 X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE);
22316
22317 APInt Mask = OriginalMask;
22318
22319 auto MaskBits = [&](SDValue Src) {
22320 if (Mask.isAllOnes())
22321 return Src;
22322 EVT SrcVT = Src.getValueType();
22323 SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT);
22324 return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue);
22325 };
22326
22327 // For sub-128-bit vector, cast to (legal) integer and compare with zero.
22328 if (VT.getSizeInBits() < 128) {
22329 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
22330 if (!DAG.getTargetLoweringInfo().isTypeLegal(IntVT)) {
22331 if (IntVT != MVT::i64)
22332 return SDValue();
22333 auto SplitLHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(LHS)), DL,
22334 MVT::i32, MVT::i32);
22335 auto SplitRHS = DAG.SplitScalar(DAG.getBitcast(IntVT, MaskBits(RHS)), DL,
22336 MVT::i32, MVT::i32);
22337 SDValue Lo =
22338 DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.first, SplitRHS.first);
22339 SDValue Hi =
22340 DAG.getNode(ISD::XOR, DL, MVT::i32, SplitLHS.second, SplitRHS.second);
22341 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
22342 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi),
22343 DAG.getConstant(0, DL, MVT::i32));
22344 }
22345 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
22346 DAG.getBitcast(IntVT, MaskBits(LHS)),
22347 DAG.getBitcast(IntVT, MaskBits(RHS)));
22348 }
22349
22350 // Without PTEST, a masked v2i64 or-reduction is not faster than
22351 // scalarization.
22352 bool UseKORTEST = Subtarget.useAVX512Regs();
22353 bool UsePTEST = Subtarget.hasSSE41();
22354 if (!UsePTEST && !Mask.isAllOnes() && ScalarSize > 32)
22355 return SDValue();
22356
22357 // Split down to 128/256/512-bit vector.
22358 unsigned TestSize = UseKORTEST ? 512 : (Subtarget.hasAVX() ? 256 : 128);
22359
22360 // If the input vector has vector elements wider than the target test size,
22361 // then cast to <X x i64> so it will safely split.
22362 if (ScalarSize > TestSize) {
22363 if (!Mask.isAllOnes())
22364 return SDValue();
22365 VT = EVT::getVectorVT(*DAG.getContext(), MVT::i64, VT.getSizeInBits() / 64);
22366 LHS = DAG.getBitcast(VT, LHS);
22367 RHS = DAG.getBitcast(VT, RHS);
22368 Mask = APInt::getAllOnes(64);
22369 }
22370
22371 if (VT.getSizeInBits() > TestSize) {
22372 KnownBits KnownRHS = DAG.computeKnownBits(RHS);
22373 if (KnownRHS.isConstant() && KnownRHS.getConstant() == Mask) {
22374 // If ICMP(AND(LHS,MASK),MASK) - reduce using AND splits.
22375 while (VT.getSizeInBits() > TestSize) {
22376 auto Split = DAG.SplitVector(LHS, DL);
22377 VT = Split.first.getValueType();
22378 LHS = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second);
22379 }
22380 RHS = DAG.getAllOnesConstant(DL, VT);
22381 } else if (!UsePTEST && !KnownRHS.isZero()) {
22382 // MOVMSK Special Case:
22383 // ALLOF(CMPEQ(X,Y)) -> AND(CMPEQ(X[0],Y[0]),CMPEQ(X[1],Y[1]),....)
22384 MVT SVT = ScalarSize >= 32 ? MVT::i32 : MVT::i8;
22385 VT = MVT::getVectorVT(SVT, VT.getSizeInBits() / SVT.getSizeInBits());
22386 LHS = DAG.getBitcast(VT, MaskBits(LHS));
22387 RHS = DAG.getBitcast(VT, MaskBits(RHS));
22388 EVT BoolVT = VT.changeVectorElementType(MVT::i1);
22389 SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETEQ);
22390 V = DAG.getSExtOrTrunc(V, DL, VT);
22391 while (VT.getSizeInBits() > TestSize) {
22392 auto Split = DAG.SplitVector(V, DL);
22393 VT = Split.first.getValueType();
22394 V = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second);
22395 }
22396 V = DAG.getNOT(DL, V, VT);
22397 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
22398 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
22399 DAG.getConstant(0, DL, MVT::i32));
22400 } else {
22401 // Convert to a ICMP_EQ(XOR(LHS,RHS),0) pattern.
22402 SDValue V = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
22403 while (VT.getSizeInBits() > TestSize) {
22404 auto Split = DAG.SplitVector(V, DL);
22405 VT = Split.first.getValueType();
22406 V = DAG.getNode(ISD::OR, DL, VT, Split.first, Split.second);
22407 }
22408 LHS = V;
22409 RHS = DAG.getConstant(0, DL, VT);
22410 }
22411 }
22412
22413 if (UseKORTEST && VT.is512BitVector()) {
22414 MVT TestVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
22415 MVT BoolVT = TestVT.changeVectorElementType(MVT::i1);
22416 LHS = DAG.getBitcast(TestVT, MaskBits(LHS));
22417 RHS = DAG.getBitcast(TestVT, MaskBits(RHS));
22418 SDValue V = DAG.getSetCC(DL, BoolVT, LHS, RHS, ISD::SETNE);
22419 return DAG.getNode(X86ISD::KORTEST, DL, MVT::i32, V, V);
22420 }
22421
22422 if (UsePTEST) {
22423 MVT TestVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
22424 LHS = DAG.getBitcast(TestVT, MaskBits(LHS));
22425 RHS = DAG.getBitcast(TestVT, MaskBits(RHS));
22426 SDValue V = DAG.getNode(ISD::XOR, DL, TestVT, LHS, RHS);
22427 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V);
22428 }
22429
22430 assert(VT.getSizeInBits() == 128 && "Failure to split to 128-bits");
22431 MVT MaskVT = ScalarSize >= 32 ? MVT::v4i32 : MVT::v16i8;
22432 LHS = DAG.getBitcast(MaskVT, MaskBits(LHS));
22433 RHS = DAG.getBitcast(MaskVT, MaskBits(RHS));
22434 SDValue V = DAG.getNode(X86ISD::PCMPEQ, DL, MaskVT, LHS, RHS);
22435 V = DAG.getNOT(DL, V, MaskVT);
22436 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
22437 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,
22438 DAG.getConstant(0, DL, MVT::i32));
22439}
22440
22441// Check whether an AND/OR'd reduction tree is PTEST-able, or if we can fallback
22442// to CMP(MOVMSK(PCMPEQB(X,Y))).
22443static SDValue MatchVectorAllEqualTest(SDValue LHS, SDValue RHS,
22444 ISD::CondCode CC, const SDLoc &DL,
22445 const X86Subtarget &Subtarget,
22446 SelectionDAG &DAG,
22447 X86::CondCode &X86CC) {
22448 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
22449
22450 bool CmpNull = isNullConstant(RHS);
22451 bool CmpAllOnes = isAllOnesConstant(RHS);
22452 if (!CmpNull && !CmpAllOnes)
22453 return SDValue();
22454
22455 SDValue Op = LHS;
22456 if (!Subtarget.hasSSE2() || !Op->hasOneUse())
22457 return SDValue();
22458
22459 // Check whether we're masking/truncating an OR-reduction result, in which
22460 // case track the masked bits.
22461 // TODO: Add CmpAllOnes support.
22462 APInt Mask = APInt::getAllOnes(Op.getScalarValueSizeInBits());
22463 if (CmpNull) {
22464 switch (Op.getOpcode()) {
22465 case ISD::TRUNCATE: {
22466 SDValue Src = Op.getOperand(0);
22467 Mask = APInt::getLowBitsSet(Src.getScalarValueSizeInBits(),
22468 Op.getScalarValueSizeInBits());
22469 Op = Src;
22470 break;
22471 }
22472 case ISD::AND: {
22473 if (auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
22474 Mask = Cst->getAPIntValue();
22475 Op = Op.getOperand(0);
22476 }
22477 break;
22478 }
22479 }
22480 }
22481
22482 ISD::NodeType LogicOp = CmpNull ? ISD::OR : ISD::AND;
22483
22484 // Match icmp(or(extract(X,0),extract(X,1)),0) anyof reduction patterns.
22485 // Match icmp(and(extract(X,0),extract(X,1)),-1) allof reduction patterns.
22486 SmallVector<SDValue, 8> VecIns;
22487 if (Op.getOpcode() == LogicOp && matchScalarReduction(Op, LogicOp, VecIns)) {
22488 EVT VT = VecIns[0].getValueType();
22489 assert(llvm::all_of(VecIns,
22490 [VT](SDValue V) { return VT == V.getValueType(); }) &&
22491 "Reduction source vector mismatch");
22492
22493 // Quit if not splittable to scalar/128/256/512-bit vector.
22494 if (!llvm::has_single_bit<uint32_t>(VT.getSizeInBits()))
22495 return SDValue();
22496
22497 // If more than one full vector is evaluated, AND/OR them first before
22498 // PTEST.
22499 for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1;
22500 Slot += 2, e += 1) {
22501 // Each iteration will AND/OR 2 nodes and append the result until there is
22502 // only 1 node left, i.e. the final value of all vectors.
22503 SDValue LHS = VecIns[Slot];
22504 SDValue RHS = VecIns[Slot + 1];
22505 VecIns.push_back(DAG.getNode(LogicOp, DL, VT, LHS, RHS));
22506 }
22507
22508 return LowerVectorAllEqual(DL, VecIns.back(),
22509 CmpNull ? DAG.getConstant(0, DL, VT)
22510 : DAG.getAllOnesConstant(DL, VT),
22511 CC, Mask, Subtarget, DAG, X86CC);
22512 }
22513
22514 // Match icmp(reduce_or(X),0) anyof reduction patterns.
22515 // Match icmp(reduce_and(X),-1) allof reduction patterns.
22516 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
22517 ISD::NodeType BinOp;
22518 if (SDValue Match =
22519 DAG.matchBinOpReduction(Op.getNode(), BinOp, {LogicOp})) {
22520 EVT MatchVT = Match.getValueType();
22521 return LowerVectorAllEqual(DL, Match,
22522 CmpNull ? DAG.getConstant(0, DL, MatchVT)
22523 : DAG.getAllOnesConstant(DL, MatchVT),
22524 CC, Mask, Subtarget, DAG, X86CC);
22525 }
22526 }
22527
22528 if (Mask.isAllOnes()) {
22529 assert(!Op.getValueType().isVector() &&
22530 "Illegal vector type for reduction pattern");
22531 SDValue Src = peekThroughBitcasts(Op);
22532 if (Src.getValueType().isFixedLengthVector() &&
22533 Src.getValueType().getScalarType() == MVT::i1) {
22534 // Match icmp(bitcast(icmp_ne(X,Y)),0) reduction patterns.
22535 // Match icmp(bitcast(icmp_eq(X,Y)),-1) reduction patterns.
22536 if (Src.getOpcode() == ISD::SETCC) {
22537 SDValue LHS = Src.getOperand(0);
22538 SDValue RHS = Src.getOperand(1);
22539 EVT LHSVT = LHS.getValueType();
22540 ISD::CondCode SrcCC = cast<CondCodeSDNode>(Src.getOperand(2))->get();
22541 if (SrcCC == (CmpNull ? ISD::SETNE : ISD::SETEQ) &&
22542 llvm::has_single_bit<uint32_t>(LHSVT.getSizeInBits())) {
22543 APInt SrcMask = APInt::getAllOnes(LHSVT.getScalarSizeInBits());
22544 return LowerVectorAllEqual(DL, LHS, RHS, CC, SrcMask, Subtarget, DAG,
22545 X86CC);
22546 }
22547 }
22548 // Match icmp(bitcast(vXi1 trunc(Y)),0) reduction patterns.
22549 // Match icmp(bitcast(vXi1 trunc(Y)),-1) reduction patterns.
22550 // Peek through truncation, mask the LSB and compare against zero/LSB.
22551 if (Src.getOpcode() == ISD::TRUNCATE) {
22552 SDValue Inner = Src.getOperand(0);
22553 EVT InnerVT = Inner.getValueType();
22554 if (llvm::has_single_bit<uint32_t>(InnerVT.getSizeInBits())) {
22555 unsigned BW = InnerVT.getScalarSizeInBits();
22556 APInt SrcMask = APInt(BW, 1);
22557 APInt Cmp = CmpNull ? APInt::getZero(BW) : SrcMask;
22558 return LowerVectorAllEqual(DL, Inner,
22559 DAG.getConstant(Cmp, DL, InnerVT), CC,
22560 SrcMask, Subtarget, DAG, X86CC);
22561 }
22562 }
22563 }
22564 }
22565
22566 return SDValue();
22567}
22568
22569/// return true if \c Op has a use that doesn't just read flags.
22570static bool hasNonFlagsUse(SDValue Op) {
22571 for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE;
22572 ++UI) {
22573 SDNode *User = *UI;
22574 unsigned UOpNo = UI.getOperandNo();
22575 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) {
22576 // Look pass truncate.
22577 UOpNo = User->use_begin().getOperandNo();
22578 User = *User->use_begin();
22579 }
22580
22581 if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC &&
22582 !(User->getOpcode() == ISD::SELECT && UOpNo == 0))
22583 return true;
22584 }
22585 return false;
22586}
22587
22588// Transform to an x86-specific ALU node with flags if there is a chance of
22589// using an RMW op or only the flags are used. Otherwise, leave
22590// the node alone and emit a 'cmp' or 'test' instruction.
22591static bool isProfitableToUseFlagOp(SDValue Op) {
22592 for (SDNode *U : Op->uses())
22593 if (U->getOpcode() != ISD::CopyToReg &&
22594 U->getOpcode() != ISD::SETCC &&
22595 U->getOpcode() != ISD::STORE)
22596 return false;
22597
22598 return true;
22599}
22600
22601/// Emit nodes that will be selected as "test Op0,Op0", or something
22602/// equivalent.
22603static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
22604 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
22605 // CF and OF aren't always set the way we want. Determine which
22606 // of these we need.
22607 bool NeedCF = false;
22608 bool NeedOF = false;
22609 switch (X86CC) {
22610 default: break;
22611 case X86::COND_A: case X86::COND_AE:
22612 case X86::COND_B: case X86::COND_BE:
22613 NeedCF = true;
22614 break;
22615 case X86::COND_G: case X86::COND_GE:
22616 case X86::COND_L: case X86::COND_LE:
22617 case X86::COND_O: case X86::COND_NO: {
22618 // Check if we really need to set the
22619 // Overflow flag. If NoSignedWrap is present
22620 // that is not actually needed.
22621 switch (Op->getOpcode()) {
22622 case ISD::ADD:
22623 case ISD::SUB:
22624 case ISD::MUL:
22625 case ISD::SHL:
22626 if (Op.getNode()->getFlags().hasNoSignedWrap())
22627 break;
22628 [[fallthrough]];
22629 default:
22630 NeedOF = true;
22631 break;
22632 }
22633 break;
22634 }
22635 }
22636 // See if we can use the EFLAGS value from the operand instead of
22637 // doing a separate TEST. TEST always sets OF and CF to 0, so unless
22638 // we prove that the arithmetic won't overflow, we can't use OF or CF.
22639 if (Op.getResNo() != 0 || NeedOF || NeedCF) {
22640 // Emit a CMP with 0, which is the TEST pattern.
22641 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
22642 DAG.getConstant(0, dl, Op.getValueType()));
22643 }
22644 unsigned Opcode = 0;
22645 unsigned NumOperands = 0;
22646
22647 SDValue ArithOp = Op;
22648
22649 // NOTICE: In the code below we use ArithOp to hold the arithmetic operation
22650 // which may be the result of a CAST. We use the variable 'Op', which is the
22651 // non-casted variable when we check for possible users.
22652 switch (ArithOp.getOpcode()) {
22653 case ISD::AND:
22654 // If the primary 'and' result isn't used, don't bother using X86ISD::AND,
22655 // because a TEST instruction will be better.
22656 if (!hasNonFlagsUse(Op))
22657 break;
22658
22659 [[fallthrough]];
22660 case ISD::ADD:
22661 case ISD::SUB:
22662 case ISD::OR:
22663 case ISD::XOR:
22664 if (!isProfitableToUseFlagOp(Op))
22665 break;
22666
22667 // Otherwise use a regular EFLAGS-setting instruction.
22668 switch (ArithOp.getOpcode()) {
22669 // clang-format off
22670 default: llvm_unreachable("unexpected operator!");
22671 case ISD::ADD: Opcode = X86ISD::ADD; break;
22672 case ISD::SUB: Opcode = X86ISD::SUB; break;
22673 case ISD::XOR: Opcode = X86ISD::XOR; break;
22674 case ISD::AND: Opcode = X86ISD::AND; break;
22675 case ISD::OR: Opcode = X86ISD::OR; break;
22676 // clang-format on
22677 }
22678
22679 NumOperands = 2;
22680 break;
22681 case X86ISD::ADD:
22682 case X86ISD::SUB:
22683 case X86ISD::OR:
22684 case X86ISD::XOR:
22685 case X86ISD::AND:
22686 return SDValue(Op.getNode(), 1);
22687 case ISD::SSUBO:
22688 case ISD::USUBO: {
22689 // /USUBO/SSUBO will become a X86ISD::SUB and we can use its Z flag.
22690 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
22691 return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0),
22692 Op->getOperand(1)).getValue(1);
22693 }
22694 default:
22695 break;
22696 }
22697
22698 if (Opcode == 0) {
22699 // Emit a CMP with 0, which is the TEST pattern.
22700 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
22701 DAG.getConstant(0, dl, Op.getValueType()));
22702 }
22703 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
22704 SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
22705
22706 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
22707 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), New);
22708 return SDValue(New.getNode(), 1);
22709}
22710
22711/// Emit nodes that will be selected as "cmp Op0,Op1", or something
22712/// equivalent.
22713static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
22714 const SDLoc &dl, SelectionDAG &DAG,
22715 const X86Subtarget &Subtarget) {
22716 if (isNullConstant(Op1))
22717 return EmitTest(Op0, X86CC, dl, DAG, Subtarget);
22718
22719 EVT CmpVT = Op0.getValueType();
22720
22721 assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 ||
22722 CmpVT == MVT::i32 || CmpVT == MVT::i64) && "Unexpected VT!");
22723
22724 // Only promote the compare up to I32 if it is a 16 bit operation
22725 // with an immediate. 16 bit immediates are to be avoided unless the target
22726 // isn't slowed down by length changing prefixes, we're optimizing for
22727 // codesize or the comparison is with a folded load.
22728 if (CmpVT == MVT::i16 && !Subtarget.hasFastImm16() &&
22729 !X86::mayFoldLoad(Op0, Subtarget) && !X86::mayFoldLoad(Op1, Subtarget) &&
22730 !DAG.getMachineFunction().getFunction().hasMinSize()) {
22731 auto *COp0 = dyn_cast<ConstantSDNode>(Op0);
22732 auto *COp1 = dyn_cast<ConstantSDNode>(Op1);
22733 // Don't do this if the immediate can fit in 8-bits.
22734 if ((COp0 && !COp0->getAPIntValue().isSignedIntN(8)) ||
22735 (COp1 && !COp1->getAPIntValue().isSignedIntN(8))) {
22736 unsigned ExtendOp =
22737 isX86CCSigned(X86CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
22738 if (X86CC == X86::COND_E || X86CC == X86::COND_NE) {
22739 // For equality comparisons try to use SIGN_EXTEND if the input was
22740 // truncate from something with enough sign bits.
22741 if (Op0.getOpcode() == ISD::TRUNCATE) {
22742 if (DAG.ComputeMaxSignificantBits(Op0.getOperand(0)) <= 16)
22743 ExtendOp = ISD::SIGN_EXTEND;
22744 } else if (Op1.getOpcode() == ISD::TRUNCATE) {
22745 if (DAG.ComputeMaxSignificantBits(Op1.getOperand(0)) <= 16)
22746 ExtendOp = ISD::SIGN_EXTEND;
22747 }
22748 }
22749
22750 CmpVT = MVT::i32;
22751 Op0 = DAG.getNode(ExtendOp, dl, CmpVT, Op0);
22752 Op1 = DAG.getNode(ExtendOp, dl, CmpVT, Op1);
22753 }
22754 }
22755
22756 // Try to shrink i64 compares if the input has enough zero bits.
22757 // TODO: Add sign-bits equivalent for isX86CCSigned(X86CC)?
22758 if (CmpVT == MVT::i64 && !isX86CCSigned(X86CC) &&
22759 Op0.hasOneUse() && // Hacky way to not break CSE opportunities with sub.
22760 DAG.MaskedValueIsZero(Op1, APInt::getHighBitsSet(64, 32)) &&
22761 DAG.MaskedValueIsZero(Op0, APInt::getHighBitsSet(64, 32))) {
22762 CmpVT = MVT::i32;
22763 Op0 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op0);
22764 Op1 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op1);
22765 }
22766
22767 // 0-x == y --> x+y == 0
22768 // 0-x != y --> x+y != 0
22769 if (Op0.getOpcode() == ISD::SUB && isNullConstant(Op0.getOperand(0)) &&
22770 Op0.hasOneUse() && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
22771 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
22772 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(1), Op1);
22773 return Add.getValue(1);
22774 }
22775
22776 // x == 0-y --> x+y == 0
22777 // x != 0-y --> x+y != 0
22778 if (Op1.getOpcode() == ISD::SUB && isNullConstant(Op1.getOperand(0)) &&
22779 Op1.hasOneUse() && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
22780 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
22781 SDValue Add = DAG.getNode(X86ISD::ADD, dl, VTs, Op0, Op1.getOperand(1));
22782 return Add.getValue(1);
22783 }
22784
22785 // Use SUB instead of CMP to enable CSE between SUB and CMP.
22786 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
22787 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
22788 return Sub.getValue(1);
22789}
22790
22791bool X86TargetLowering::isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond,
22792 EVT VT) const {
22793 return !VT.isVector() || Cond != ISD::CondCode::SETEQ;
22794}
22795
22796bool X86TargetLowering::optimizeFMulOrFDivAsShiftAddBitcast(
22797 SDNode *N, SDValue, SDValue IntPow2) const {
22798 if (N->getOpcode() == ISD::FDIV)
22799 return true;
22800
22801 EVT FPVT = N->getValueType(0);
22802 EVT IntVT = IntPow2.getValueType();
22803
22804 // This indicates a non-free bitcast.
22805 // TODO: This is probably overly conservative as we will need to scale the
22806 // integer vector anyways for the int->fp cast.
22807 if (FPVT.isVector() &&
22808 FPVT.getScalarSizeInBits() != IntVT.getScalarSizeInBits())
22809 return false;
22810
22811 return true;
22812}
22813
22814/// Check if replacement of SQRT with RSQRT should be disabled.
22815bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
22816 EVT VT = Op.getValueType();
22817
22818 // We don't need to replace SQRT with RSQRT for half type.
22819 if (VT.getScalarType() == MVT::f16)
22820 return true;
22821
22822 // We never want to use both SQRT and RSQRT instructions for the same input.
22823 if (DAG.doesNodeExist(X86ISD::FRSQRT, DAG.getVTList(VT), Op))
22824 return false;
22825
22826 if (VT.isVector())
22827 return Subtarget.hasFastVectorFSQRT();
22828 return Subtarget.hasFastScalarFSQRT();
22829}
22830
22831/// The minimum architected relative accuracy is 2^-12. We need one
22832/// Newton-Raphson step to have a good float result (24 bits of precision).
22833SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
22834 SelectionDAG &DAG, int Enabled,
22835 int &RefinementSteps,
22836 bool &UseOneConstNR,
22837 bool Reciprocal) const {
22838 SDLoc DL(Op);
22839 EVT VT = Op.getValueType();
22840
22841 // SSE1 has rsqrtss and rsqrtps. AVX adds a 256-bit variant for rsqrtps.
22842 // It is likely not profitable to do this for f64 because a double-precision
22843 // rsqrt estimate with refinement on x86 prior to FMA requires at least 16
22844 // instructions: convert to single, rsqrtss, convert back to double, refine
22845 // (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
22846 // along with FMA, this could be a throughput win.
22847 // TODO: SQRT requires SSE2 to prevent the introduction of an illegal v4i32
22848 // after legalize types.
22849 if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
22850 (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) ||
22851 (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) ||
22852 (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
22853 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
22854 if (RefinementSteps == ReciprocalEstimate::Unspecified)
22855 RefinementSteps = 1;
22856
22857 UseOneConstNR = false;
22858 // There is no FSQRT for 512-bits, but there is RSQRT14.
22859 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT;
22860 SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op);
22861 if (RefinementSteps == 0 && !Reciprocal)
22862 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Op, Estimate);
22863 return Estimate;
22864 }
22865
22866 if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) &&
22867 Subtarget.hasFP16()) {
22868 assert(Reciprocal && "Don't replace SQRT with RSQRT for half type");
22869 if (RefinementSteps == ReciprocalEstimate::Unspecified)
22870 RefinementSteps = 0;
22871
22872 if (VT == MVT::f16) {
22873 SDValue Zero = DAG.getIntPtrConstant(0, DL);
22874 SDValue Undef = DAG.getUNDEF(MVT::v8f16);
22875 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
22876 Op = DAG.getNode(X86ISD::RSQRT14S, DL, MVT::v8f16, Undef, Op);
22877 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
22878 }
22879
22880 return DAG.getNode(X86ISD::RSQRT14, DL, VT, Op);
22881 }
22882 return SDValue();
22883}
22884
22885/// The minimum architected relative accuracy is 2^-12. We need one
22886/// Newton-Raphson step to have a good float result (24 bits of precision).
22887SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
22888 int Enabled,
22889 int &RefinementSteps) const {
22890 SDLoc DL(Op);
22891 EVT VT = Op.getValueType();
22892
22893 // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
22894 // It is likely not profitable to do this for f64 because a double-precision
22895 // reciprocal estimate with refinement on x86 prior to FMA requires
22896 // 15 instructions: convert to single, rcpss, convert back to double, refine
22897 // (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
22898 // along with FMA, this could be a throughput win.
22899
22900 if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
22901 (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
22902 (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
22903 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
22904 // Enable estimate codegen with 1 refinement step for vector division.
22905 // Scalar division estimates are disabled because they break too much
22906 // real-world code. These defaults are intended to match GCC behavior.
22907 if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified)
22908 return SDValue();
22909
22910 if (RefinementSteps == ReciprocalEstimate::Unspecified)
22911 RefinementSteps = 1;
22912
22913 // There is no FSQRT for 512-bits, but there is RCP14.
22914 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP;
22915 return DAG.getNode(Opcode, DL, VT, Op);
22916 }
22917
22918 if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) &&
22919 Subtarget.hasFP16()) {
22920 if (RefinementSteps == ReciprocalEstimate::Unspecified)
22921 RefinementSteps = 0;
22922
22923 if (VT == MVT::f16) {
22924 SDValue Zero = DAG.getIntPtrConstant(0, DL);
22925 SDValue Undef = DAG.getUNDEF(MVT::v8f16);
22926 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);
22927 Op = DAG.getNode(X86ISD::RCP14S, DL, MVT::v8f16, Undef, Op);
22928 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);
22929 }
22930
22931 return DAG.getNode(X86ISD::RCP14, DL, VT, Op);
22932 }
22933 return SDValue();
22934}
22935
22936/// If we have at least two divisions that use the same divisor, convert to
22937/// multiplication by a reciprocal. This may need to be adjusted for a given
22938/// CPU if a division's cost is not at least twice the cost of a multiplication.
22939/// This is because we still need one division to calculate the reciprocal and
22940/// then we need two multiplies by that reciprocal as replacements for the
22941/// original divisions.
22942unsigned X86TargetLowering::combineRepeatedFPDivisors() const {
22943 return 2;
22944}
22945
22946SDValue
22947X86TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
22948 SelectionDAG &DAG,
22949 SmallVectorImpl<SDNode *> &Created) const {
22950 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
22951 if (isIntDivCheap(N->getValueType(0), Attr))
22952 return SDValue(N,0); // Lower SDIV as SDIV
22953
22954 assert((Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()) &&
22955 "Unexpected divisor!");
22956
22957 // Only perform this transform if CMOV is supported otherwise the select
22958 // below will become a branch.
22959 if (!Subtarget.canUseCMOV())
22960 return SDValue();
22961
22962 // fold (sdiv X, pow2)
22963 EVT VT = N->getValueType(0);
22964 // FIXME: Support i8.
22965 if (VT != MVT::i16 && VT != MVT::i32 &&
22966 !(Subtarget.is64Bit() && VT == MVT::i64))
22967 return SDValue();
22968
22969 // If the divisor is 2 or -2, the default expansion is better.
22970 if (Divisor == 2 ||
22971 Divisor == APInt(Divisor.getBitWidth(), -2, /*isSigned*/ true))
22972 return SDValue();
22973
22974 return TargetLowering::buildSDIVPow2WithCMov(N, Divisor, DAG, Created);
22975}
22976
22977/// Result of 'and' is compared against zero. Change to a BT node if possible.
22978/// Returns the BT node and the condition code needed to use it.
22979static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC, const SDLoc &dl,
22980 SelectionDAG &DAG, X86::CondCode &X86CC) {
22981 assert(And.getOpcode() == ISD::AND && "Expected AND node!");
22982 SDValue Op0 = And.getOperand(0);
22983 SDValue Op1 = And.getOperand(1);
22984 if (Op0.getOpcode() == ISD::TRUNCATE)
22985 Op0 = Op0.getOperand(0);
22986 if (Op1.getOpcode() == ISD::TRUNCATE)
22987 Op1 = Op1.getOperand(0);
22988
22989 SDValue Src, BitNo;
22990 if (Op1.getOpcode() == ISD::SHL)
22991 std::swap(Op0, Op1);
22992 if (Op0.getOpcode() == ISD::SHL) {
22993 if (isOneConstant(Op0.getOperand(0))) {
22994 // If we looked past a truncate, check that it's only truncating away
22995 // known zeros.
22996 unsigned BitWidth = Op0.getValueSizeInBits();
22997 unsigned AndBitWidth = And.getValueSizeInBits();
22998 if (BitWidth > AndBitWidth) {
22999 KnownBits Known = DAG.computeKnownBits(Op0);
23000 if (Known.countMinLeadingZeros() < BitWidth - AndBitWidth)
23001 return SDValue();
23002 }
23003 Src = Op1;
23004 BitNo = Op0.getOperand(1);
23005 }
23006 } else if (Op1.getOpcode() == ISD::Constant) {
23007 ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1);
23008 uint64_t AndRHSVal = AndRHS->getZExtValue();
23009 SDValue AndLHS = Op0;
23010
23011 if (AndRHSVal == 1 && AndLHS.getOpcode() == ISD::SRL) {
23012 Src = AndLHS.getOperand(0);
23013 BitNo = AndLHS.getOperand(1);
23014 } else {
23015 // Use BT if the immediate can't be encoded in a TEST instruction or we
23016 // are optimizing for size and the immedaite won't fit in a byte.
23017 bool OptForSize = DAG.shouldOptForSize();
23018 if ((!isUInt<32>(AndRHSVal) || (OptForSize && !isUInt<8>(AndRHSVal))) &&
23019 isPowerOf2_64(AndRHSVal)) {
23020 Src = AndLHS;
23021 BitNo = DAG.getConstant(Log2_64_Ceil(AndRHSVal), dl,
23022 Src.getValueType());
23023 }
23024 }
23025 }
23026
23027 // No patterns found, give up.
23028 if (!Src.getNode())
23029 return SDValue();
23030
23031 // Remove any bit flip.
23032 if (isBitwiseNot(Src)) {
23033 Src = Src.getOperand(0);
23034 CC = CC == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
23035 }
23036
23037 // Attempt to create the X86ISD::BT node.
23038 if (SDValue BT = getBT(Src, BitNo, dl, DAG)) {
23039 X86CC = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
23040 return BT;
23041 }
23042
23043 return SDValue();
23044}
23045
23046// Check if pre-AVX condcode can be performed by a single FCMP op.
23047static bool cheapX86FSETCC_SSE(ISD::CondCode SetCCOpcode) {
23048 return (SetCCOpcode != ISD::SETONE) && (SetCCOpcode != ISD::SETUEQ);
23049}
23050
23051/// Turns an ISD::CondCode into a value suitable for SSE floating-point mask
23052/// CMPs.
23053static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
23054 SDValue &Op1, bool &IsAlwaysSignaling) {
23055 unsigned SSECC;
23056 bool Swap = false;
23057
23058 // SSE Condition code mapping:
23059 // 0 - EQ
23060 // 1 - LT
23061 // 2 - LE
23062 // 3 - UNORD
23063 // 4 - NEQ
23064 // 5 - NLT
23065 // 6 - NLE
23066 // 7 - ORD
23067 switch (SetCCOpcode) {
23068 // clang-format off
23069 default: llvm_unreachable("Unexpected SETCC condition");
23070 case ISD::SETOEQ:
23071 case ISD::SETEQ: SSECC = 0; break;
23072 case ISD::SETOGT:
23073 case ISD::SETGT: Swap = true; [[fallthrough]];
23074 case ISD::SETLT:
23075 case ISD::SETOLT: SSECC = 1; break;
23076 case ISD::SETOGE:
23077 case ISD::SETGE: Swap = true; [[fallthrough]];
23078 case ISD::SETLE:
23079 case ISD::SETOLE: SSECC = 2; break;
23080 case ISD::SETUO: SSECC = 3; break;
23081 case ISD::SETUNE:
23082 case ISD::SETNE: SSECC = 4; break;
23083 case ISD::SETULE: Swap = true; [[fallthrough]];
23084 case ISD::SETUGE: SSECC = 5; break;
23085 case ISD::SETULT: Swap = true; [[fallthrough]];
23086 case ISD::SETUGT: SSECC = 6; break;
23087 case ISD::SETO: SSECC = 7; break;
23088 case ISD::SETUEQ: SSECC = 8; break;
23089 case ISD::SETONE: SSECC = 12; break;
23090 // clang-format on
23091 }
23092 if (Swap)
23093 std::swap(Op0, Op1);
23094
23095 switch (SetCCOpcode) {
23096 default:
23097 IsAlwaysSignaling = true;
23098 break;
23099 case ISD::SETEQ:
23100 case ISD::SETOEQ:
23101 case ISD::SETUEQ:
23102 case ISD::SETNE:
23103 case ISD::SETONE:
23104 case ISD::SETUNE:
23105 case ISD::SETO:
23106 case ISD::SETUO:
23107 IsAlwaysSignaling = false;
23108 break;
23109 }
23110
23111 return SSECC;
23112}
23113
23114/// Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then
23115/// concatenate the result back.
23116static SDValue splitIntVSETCC(EVT VT, SDValue LHS, SDValue RHS,
23117 ISD::CondCode Cond, SelectionDAG &DAG,
23118 const SDLoc &dl) {
23119 assert(VT.isInteger() && VT == LHS.getValueType() &&
23120 VT == RHS.getValueType() && "Unsupported VTs!");
23121
23122 SDValue CC = DAG.getCondCode(Cond);
23123
23124 // Extract the LHS Lo/Hi vectors
23125 SDValue LHS1, LHS2;
23126 std::tie(LHS1, LHS2) = splitVector(LHS, DAG, dl);
23127
23128 // Extract the RHS Lo/Hi vectors
23129 SDValue RHS1, RHS2;
23130 std::tie(RHS1, RHS2) = splitVector(RHS, DAG, dl);
23131
23132 // Issue the operation on the smaller types and concatenate the result back
23133 EVT LoVT, HiVT;
23134 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
23135 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
23136 DAG.getNode(ISD::SETCC, dl, LoVT, LHS1, RHS1, CC),
23137 DAG.getNode(ISD::SETCC, dl, HiVT, LHS2, RHS2, CC));
23138}
23139
23140static SDValue LowerIntVSETCC_AVX512(SDValue Op, const SDLoc &dl,
23141 SelectionDAG &DAG) {
23142 SDValue Op0 = Op.getOperand(0);
23143 SDValue Op1 = Op.getOperand(1);
23144 SDValue CC = Op.getOperand(2);
23145 MVT VT = Op.getSimpleValueType();
23146 assert(VT.getVectorElementType() == MVT::i1 &&
23147 "Cannot set masked compare for this operation");
23148
23149 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
23150
23151 // Prefer SETGT over SETLT.
23152 if (SetCCOpcode == ISD::SETLT) {
23153 SetCCOpcode = ISD::getSetCCSwappedOperands(SetCCOpcode);
23154 std::swap(Op0, Op1);
23155 }
23156
23157 return DAG.getSetCC(dl, VT, Op0, Op1, SetCCOpcode);
23158}
23159
23160/// Given a buildvector constant, return a new vector constant with each element
23161/// incremented or decremented. If incrementing or decrementing would result in
23162/// unsigned overflow or underflow or this is not a simple vector constant,
23163/// return an empty value.
23164static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc,
23165 bool NSW) {
23166 auto *BV = dyn_cast<BuildVectorSDNode>(V.getNode());
23167 if (!BV || !V.getValueType().isSimple())
23168 return SDValue();
23169
23170 MVT VT = V.getSimpleValueType();
23171 MVT EltVT = VT.getVectorElementType();
23172 unsigned NumElts = VT.getVectorNumElements();
23173 SmallVector<SDValue, 8> NewVecC;
23174 SDLoc DL(V);
23175 for (unsigned i = 0; i < NumElts; ++i) {
23176 auto *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
23177 if (!Elt || Elt->isOpaque() || Elt->getSimpleValueType(0) != EltVT)
23178 return SDValue();
23179
23180 // Avoid overflow/underflow.
23181 const APInt &EltC = Elt->getAPIntValue();
23182 if ((IsInc && EltC.isMaxValue()) || (!IsInc && EltC.isZero()))
23183 return SDValue();
23184 if (NSW && ((IsInc && EltC.isMaxSignedValue()) ||
23185 (!IsInc && EltC.isMinSignedValue())))
23186 return SDValue();
23187
23188 NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT));
23189 }
23190
23191 return DAG.getBuildVector(VT, DL, NewVecC);
23192}
23193
23194/// As another special case, use PSUBUS[BW] when it's profitable. E.g. for
23195/// Op0 u<= Op1:
23196/// t = psubus Op0, Op1
23197/// pcmpeq t, <0..0>
23198static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,
23199 ISD::CondCode Cond, const SDLoc &dl,
23200 const X86Subtarget &Subtarget,
23201 SelectionDAG &DAG) {
23202 if (!Subtarget.hasSSE2())
23203 return SDValue();
23204
23205 MVT VET = VT.getVectorElementType();
23206 if (VET != MVT::i8 && VET != MVT::i16)
23207 return SDValue();
23208
23209 switch (Cond) {
23210 default:
23211 return SDValue();
23212 case ISD::SETULT: {
23213 // If the comparison is against a constant we can turn this into a
23214 // setule. With psubus, setule does not require a swap. This is
23215 // beneficial because the constant in the register is no longer
23216 // destructed as the destination so it can be hoisted out of a loop.
23217 // Only do this pre-AVX since vpcmp* is no longer destructive.
23218 if (Subtarget.hasAVX())
23219 return SDValue();
23220 SDValue ULEOp1 =
23221 incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false);
23222 if (!ULEOp1)
23223 return SDValue();
23224 Op1 = ULEOp1;
23225 break;
23226 }
23227 case ISD::SETUGT: {
23228 // If the comparison is against a constant, we can turn this into a setuge.
23229 // This is beneficial because materializing a constant 0 for the PCMPEQ is
23230 // probably cheaper than XOR+PCMPGT using 2 different vector constants:
23231 // cmpgt (xor X, SignMaskC) CmpC --> cmpeq (usubsat (CmpC+1), X), 0
23232 SDValue UGEOp1 =
23233 incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false);
23234 if (!UGEOp1)
23235 return SDValue();
23236 Op1 = Op0;
23237 Op0 = UGEOp1;
23238 break;
23239 }
23240 // Psubus is better than flip-sign because it requires no inversion.
23241 case ISD::SETUGE:
23242 std::swap(Op0, Op1);
23243 break;
23244 case ISD::SETULE:
23245 break;
23246 }
23247
23248 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1);
23249 return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
23250 DAG.getConstant(0, dl, VT));
23251}
23252
23253static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
23254 SelectionDAG &DAG) {
23255 bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC ||
23256 Op.getOpcode() == ISD::STRICT_FSETCCS;
23257 SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
23258 SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
23259 SDValue CC = Op.getOperand(IsStrict ? 3 : 2);
23260 MVT VT = Op->getSimpleValueType(0);
23261 ISD::CondCode Cond = cast<CondCodeSDNode>(CC)->get();
23262 bool isFP = Op1.getSimpleValueType().isFloatingPoint();
23263 SDLoc dl(Op);
23264
23265 if (isFP) {
23266 MVT EltVT = Op0.getSimpleValueType().getVectorElementType();
23267 assert(EltVT == MVT::f16 || EltVT == MVT::f32 || EltVT == MVT::f64);
23268 if (isSoftF16(EltVT, Subtarget))
23269 return SDValue();
23270
23271 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
23272 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
23273
23274 // If we have a strict compare with a vXi1 result and the input is 128/256
23275 // bits we can't use a masked compare unless we have VLX. If we use a wider
23276 // compare like we do for non-strict, we might trigger spurious exceptions
23277 // from the upper elements. Instead emit a AVX compare and convert to mask.
23278 unsigned Opc;
23279 if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1 &&
23280 (!IsStrict || Subtarget.hasVLX() ||
23281 Op0.getSimpleValueType().is512BitVector())) {
23282#ifndef NDEBUG
23283 unsigned Num = VT.getVectorNumElements();
23284 assert(Num <= 16 || (Num == 32 && EltVT == MVT::f16));
23285#endif
23286 Opc = IsStrict ? X86ISD::STRICT_CMPM : X86ISD::CMPM;
23287 } else {
23288 Opc = IsStrict ? X86ISD::STRICT_CMPP : X86ISD::CMPP;
23289 // The SSE/AVX packed FP comparison nodes are defined with a
23290 // floating-point vector result that matches the operand type. This allows
23291 // them to work with an SSE1 target (integer vector types are not legal).
23292 VT = Op0.getSimpleValueType();
23293 }
23294
23295 SDValue Cmp;
23296 bool IsAlwaysSignaling;
23297 unsigned SSECC = translateX86FSETCC(Cond, Op0, Op1, IsAlwaysSignaling);
23298 if (!Subtarget.hasAVX()) {
23299 // TODO: We could use following steps to handle a quiet compare with
23300 // signaling encodings.
23301 // 1. Get ordered masks from a quiet ISD::SETO
23302 // 2. Use the masks to mask potential unordered elements in operand A, B
23303 // 3. Get the compare results of masked A, B
23304 // 4. Calculating final result using the mask and result from 3
23305 // But currently, we just fall back to scalar operations.
23306 if (IsStrict && IsAlwaysSignaling && !IsSignaling)
23307 return SDValue();
23308
23309 // Insert an extra signaling instruction to raise exception.
23310 if (IsStrict && !IsAlwaysSignaling && IsSignaling) {
23311 SDValue SignalCmp = DAG.getNode(
23312 Opc, dl, {VT, MVT::Other},
23313 {Chain, Op0, Op1, DAG.getTargetConstant(1, dl, MVT::i8)}); // LT_OS
23314 // FIXME: It seems we need to update the flags of all new strict nodes.
23315 // Otherwise, mayRaiseFPException in MI will return false due to
23316 // NoFPExcept = false by default. However, I didn't find it in other
23317 // patches.
23318 SignalCmp->setFlags(Op->getFlags());
23319 Chain = SignalCmp.getValue(1);
23320 }
23321
23322 // In the two cases not handled by SSE compare predicates (SETUEQ/SETONE),
23323 // emit two comparisons and a logic op to tie them together.
23324 if (!cheapX86FSETCC_SSE(Cond)) {
23325 // LLVM predicate is SETUEQ or SETONE.
23326 unsigned CC0, CC1;
23327 unsigned CombineOpc;
23328 if (Cond == ISD::SETUEQ) {
23329 CC0 = 3; // UNORD
23330 CC1 = 0; // EQ
23331 CombineOpc = X86ISD::FOR;
23332 } else {
23333 assert(Cond == ISD::SETONE);
23334 CC0 = 7; // ORD
23335 CC1 = 4; // NEQ
23336 CombineOpc = X86ISD::FAND;
23337 }
23338
23339 SDValue Cmp0, Cmp1;
23340 if (IsStrict) {
23341 Cmp0 = DAG.getNode(
23342 Opc, dl, {VT, MVT::Other},
23343 {Chain, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)});
23344 Cmp1 = DAG.getNode(
23345 Opc, dl, {VT, MVT::Other},
23346 {Chain, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)});
23347 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1),
23348 Cmp1.getValue(1));
23349 } else {
23350 Cmp0 = DAG.getNode(
23351 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8));
23352 Cmp1 = DAG.getNode(
23353 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8));
23354 }
23355 Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
23356 } else {
23357 if (IsStrict) {
23358 Cmp = DAG.getNode(
23359 Opc, dl, {VT, MVT::Other},
23360 {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
23361 Chain = Cmp.getValue(1);
23362 } else
23363 Cmp = DAG.getNode(
23364 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
23365 }
23366 } else {
23367 // Handle all other FP comparisons here.
23368 if (IsStrict) {
23369 // Make a flip on already signaling CCs before setting bit 4 of AVX CC.
23370 SSECC |= (IsAlwaysSignaling ^ IsSignaling) << 4;
23371 Cmp = DAG.getNode(
23372 Opc, dl, {VT, MVT::Other},
23373 {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
23374 Chain = Cmp.getValue(1);
23375 } else
23376 Cmp = DAG.getNode(
23377 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
23378 }
23379
23380 if (VT.getFixedSizeInBits() >
23381 Op.getSimpleValueType().getFixedSizeInBits()) {
23382 // We emitted a compare with an XMM/YMM result. Finish converting to a
23383 // mask register using a vptestm.
23384 EVT CastVT = EVT(VT).changeVectorElementTypeToInteger();
23385 Cmp = DAG.getBitcast(CastVT, Cmp);
23386 Cmp = DAG.getSetCC(dl, Op.getSimpleValueType(), Cmp,
23387 DAG.getConstant(0, dl, CastVT), ISD::SETNE);
23388 } else {
23389 // If this is SSE/AVX CMPP, bitcast the result back to integer to match
23390 // the result type of SETCC. The bitcast is expected to be optimized
23391 // away during combining/isel.
23392 Cmp = DAG.getBitcast(Op.getSimpleValueType(), Cmp);
23393 }
23394
23395 if (IsStrict)
23396 return DAG.getMergeValues({Cmp, Chain}, dl);
23397
23398 return Cmp;
23399 }
23400
23401 assert(!IsStrict && "Strict SETCC only handles FP operands.");
23402
23403 [[maybe_unused]] MVT VTOp0 = Op0.getSimpleValueType();
23404 assert(VTOp0 == Op1.getSimpleValueType() &&
23405 "Expected operands with same type!");
23406 assert(VT.getVectorNumElements() == VTOp0.getVectorNumElements() &&
23407 "Invalid number of packed elements for source and destination!");
23408
23409 // The non-AVX512 code below works under the assumption that source and
23410 // destination types are the same.
23411 assert((Subtarget.hasAVX512() || (VT == VTOp0)) &&
23412 "Value types for source and destination must be the same!");
23413
23414 // The result is boolean, but operands are int/float
23415 if (VT.getVectorElementType() == MVT::i1) {
23416 // In AVX-512 architecture setcc returns mask with i1 elements,
23417 // But there is no compare instruction for i8 and i16 elements in KNL.
23418 assert((VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) &&
23419 "Unexpected operand type");
23420 return LowerIntVSETCC_AVX512(Op, dl, DAG);
23421 }
23422
23423 // Lower using XOP integer comparisons.
23424 if (VT.is128BitVector() && Subtarget.hasXOP()) {
23425 // Translate compare code to XOP PCOM compare mode.
23426 unsigned CmpMode = 0;
23427 switch (Cond) {
23428 // clang-format off
23429 default: llvm_unreachable("Unexpected SETCC condition");
23430 case ISD::SETULT:
23431 case ISD::SETLT: CmpMode = 0x00; break;
23432 case ISD::SETULE:
23433 case ISD::SETLE: CmpMode = 0x01; break;
23434 case ISD::SETUGT:
23435 case ISD::SETGT: CmpMode = 0x02; break;
23436 case ISD::SETUGE:
23437 case ISD::SETGE: CmpMode = 0x03; break;
23438 case ISD::SETEQ: CmpMode = 0x04; break;
23439 case ISD::SETNE: CmpMode = 0x05; break;
23440 // clang-format on
23441 }
23442
23443 // Are we comparing unsigned or signed integers?
23444 unsigned Opc =
23445 ISD::isUnsignedIntSetCC(Cond) ? X86ISD::VPCOMU : X86ISD::VPCOM;
23446
23447 return DAG.getNode(Opc, dl, VT, Op0, Op1,
23448 DAG.getTargetConstant(CmpMode, dl, MVT::i8));
23449 }
23450
23451 // (X & Y) != 0 --> (X & Y) == Y iff Y is power-of-2.
23452 // Revert part of the simplifySetCCWithAnd combine, to avoid an invert.
23453 if (Cond == ISD::SETNE && ISD::isBuildVectorAllZeros(Op1.getNode())) {
23454 SDValue BC0 = peekThroughBitcasts(Op0);
23455 if (BC0.getOpcode() == ISD::AND) {
23456 APInt UndefElts;
23457 SmallVector<APInt, 64> EltBits;
23458 if (getTargetConstantBitsFromNode(
23459 BC0.getOperand(1), VT.getScalarSizeInBits(), UndefElts, EltBits,
23460 /*AllowWholeUndefs*/ false, /*AllowPartialUndefs*/ false)) {
23461 if (llvm::all_of(EltBits, [](APInt &V) { return V.isPowerOf2(); })) {
23462 Cond = ISD::SETEQ;
23463 Op1 = DAG.getBitcast(VT, BC0.getOperand(1));
23464 }
23465 }
23466 }
23467 }
23468
23469 // ICMP_EQ(AND(X,C),C) -> SRA(SHL(X,LOG2(C)),BW-1) iff C is power-of-2.
23470 if (Cond == ISD::SETEQ && Op0.getOpcode() == ISD::AND &&
23471 Op0.getOperand(1) == Op1 && Op0.hasOneUse()) {
23472 ConstantSDNode *C1 = isConstOrConstSplat(Op1);
23473 if (C1 && C1->getAPIntValue().isPowerOf2()) {
23474 unsigned BitWidth = VT.getScalarSizeInBits();
23475 unsigned ShiftAmt = BitWidth - C1->getAPIntValue().logBase2() - 1;
23476
23477 SDValue Result = Op0.getOperand(0);
23478 Result = DAG.getNode(ISD::SHL, dl, VT, Result,
23479 DAG.getConstant(ShiftAmt, dl, VT));
23480 Result = DAG.getNode(ISD::SRA, dl, VT, Result,
23481 DAG.getConstant(BitWidth - 1, dl, VT));
23482 return Result;
23483 }
23484 }
23485
23486 // Break 256-bit integer vector compare into smaller ones.
23487 if (VT.is256BitVector() && !Subtarget.hasInt256())
23488 return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23489
23490 // Break 512-bit integer vector compare into smaller ones.
23491 // TODO: Try harder to use VPCMPx + VPMOV2x?
23492 if (VT.is512BitVector())
23493 return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl);
23494
23495 // If we have a limit constant, try to form PCMPGT (signed cmp) to avoid
23496 // not-of-PCMPEQ:
23497 // X != INT_MIN --> X >s INT_MIN
23498 // X != INT_MAX --> X <s INT_MAX --> INT_MAX >s X
23499 // +X != 0 --> +X >s 0
23500 APInt ConstValue;
23501 if (Cond == ISD::SETNE &&
23502 ISD::isConstantSplatVector(Op1.getNode(), ConstValue)) {
23503 if (ConstValue.isMinSignedValue())
23504 Cond = ISD::SETGT;
23505 else if (ConstValue.isMaxSignedValue())
23506 Cond = ISD::SETLT;
23507 else if (ConstValue.isZero() && DAG.SignBitIsZero(Op0))
23508 Cond = ISD::SETGT;
23509 }
23510
23511 // If both operands are known non-negative, then an unsigned compare is the
23512 // same as a signed compare and there's no need to flip signbits.
23513 // TODO: We could check for more general simplifications here since we're
23514 // computing known bits.
23515 bool FlipSigns = ISD::isUnsignedIntSetCC(Cond) &&
23516 !(DAG.SignBitIsZero(Op0) && DAG.SignBitIsZero(Op1));
23517
23518 // Special case: Use min/max operations for unsigned compares.
23519 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23520 if (ISD::isUnsignedIntSetCC(Cond) &&
23521 (FlipSigns || ISD::isTrueWhenEqual(Cond)) &&
23522 TLI.isOperationLegal(ISD::UMIN, VT)) {
23523 // If we have a constant operand, increment/decrement it and change the
23524 // condition to avoid an invert.
23525 if (Cond == ISD::SETUGT) {
23526 // X > C --> X >= (C+1) --> X == umax(X, C+1)
23527 if (SDValue UGTOp1 =
23528 incDecVectorConstant(Op1, DAG, /*IsInc*/ true, /*NSW*/ false)) {
23529 Op1 = UGTOp1;
23530 Cond = ISD::SETUGE;
23531 }
23532 }
23533 if (Cond == ISD::SETULT) {
23534 // X < C --> X <= (C-1) --> X == umin(X, C-1)
23535 if (SDValue ULTOp1 =
23536 incDecVectorConstant(Op1, DAG, /*IsInc*/ false, /*NSW*/ false)) {
23537 Op1 = ULTOp1;
23538 Cond = ISD::SETULE;
23539 }
23540 }
23541 bool Invert = false;
23542 unsigned Opc;
23543 switch (Cond) {
23544 // clang-format off
23545 default: llvm_unreachable("Unexpected condition code");
23546 case ISD::SETUGT: Invert = true; [[fallthrough]];
23547 case ISD::SETULE: Opc = ISD::UMIN; break;
23548 case ISD::SETULT: Invert = true; [[fallthrough]];
23549 case ISD::SETUGE: Opc = ISD::UMAX; break;
23550 // clang-format on
23551 }
23552
23553 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
23554 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
23555
23556 // If the logical-not of the result is required, perform that now.
23557 if (Invert)
23558 Result = DAG.getNOT(dl, Result, VT);
23559
23560 return Result;
23561 }
23562
23563 // Try to use SUBUS and PCMPEQ.
23564 if (FlipSigns)
23565 if (SDValue V =
23566 LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG))
23567 return V;
23568
23569 // We are handling one of the integer comparisons here. Since SSE only has
23570 // GT and EQ comparisons for integer, swapping operands and multiple
23571 // operations may be required for some comparisons.
23572 unsigned Opc = (Cond == ISD::SETEQ || Cond == ISD::SETNE) ? X86ISD::PCMPEQ
23573 : X86ISD::PCMPGT;
23574 bool Swap = Cond == ISD::SETLT || Cond == ISD::SETULT ||
23575 Cond == ISD::SETGE || Cond == ISD::SETUGE;
23576 bool Invert = Cond == ISD::SETNE ||
23577 (Cond != ISD::SETEQ && ISD::isTrueWhenEqual(Cond));
23578
23579 if (Swap)
23580 std::swap(Op0, Op1);
23581
23582 // Check that the operation in question is available (most are plain SSE2,
23583 // but PCMPGTQ and PCMPEQQ have different requirements).
23584 if (VT == MVT::v2i64) {
23585 if (Opc == X86ISD::PCMPGT && !Subtarget.hasSSE42()) {
23586 assert(Subtarget.hasSSE2() && "Don't know how to lower!");
23587
23588 // Special case for sign bit test. We can use a v4i32 PCMPGT and shuffle
23589 // the odd elements over the even elements.
23590 if (!FlipSigns && !Invert && ISD::isBuildVectorAllZeros(Op0.getNode())) {
23591 Op0 = DAG.getConstant(0, dl, MVT::v4i32);
23592 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23593
23594 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23595 static const int MaskHi[] = { 1, 1, 3, 3 };
23596 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23597
23598 return DAG.getBitcast(VT, Result);
23599 }
23600
23601 if (!FlipSigns && !Invert && ISD::isBuildVectorAllOnes(Op1.getNode())) {
23602 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23603 Op1 = DAG.getAllOnesConstant(dl, MVT::v4i32);
23604
23605 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23606 static const int MaskHi[] = { 1, 1, 3, 3 };
23607 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23608
23609 return DAG.getBitcast(VT, Result);
23610 }
23611
23612 // If the i64 elements are sign-extended enough to be representable as i32
23613 // then we can compare the lower i32 bits and splat.
23614 if (!FlipSigns && !Invert && DAG.ComputeNumSignBits(Op0) > 32 &&
23615 DAG.ComputeNumSignBits(Op1) > 32) {
23616 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23617 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23618
23619 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23620 static const int MaskLo[] = {0, 0, 2, 2};
23621 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
23622
23623 return DAG.getBitcast(VT, Result);
23624 }
23625
23626 // Since SSE has no unsigned integer comparisons, we need to flip the sign
23627 // bits of the inputs before performing those operations. The lower
23628 // compare is always unsigned.
23629 SDValue SB = DAG.getConstant(FlipSigns ? 0x8000000080000000ULL
23630 : 0x0000000080000000ULL,
23631 dl, MVT::v2i64);
23632
23633 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op0, SB);
23634 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op1, SB);
23635
23636 // Cast everything to the right type.
23637 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23638 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23639
23640 // Emulate PCMPGTQ with (hi1 > hi2) | ((hi1 == hi2) & (lo1 > lo2))
23641 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
23642 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
23643
23644 // Create masks for only the low parts/high parts of the 64 bit integers.
23645 static const int MaskHi[] = { 1, 1, 3, 3 };
23646 static const int MaskLo[] = { 0, 0, 2, 2 };
23647 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
23648 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
23649 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
23650
23651 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
23652 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi);
23653
23654 if (Invert)
23655 Result = DAG.getNOT(dl, Result, MVT::v4i32);
23656
23657 return DAG.getBitcast(VT, Result);
23658 }
23659
23660 if (Opc == X86ISD::PCMPEQ && !Subtarget.hasSSE41()) {
23661 // If pcmpeqq is missing but pcmpeqd is available synthesize pcmpeqq with
23662 // pcmpeqd + pshufd + pand.
23663 assert(Subtarget.hasSSE2() && !FlipSigns && "Don't know how to lower!");
23664
23665 // First cast everything to the right type.
23666 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
23667 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
23668
23669 // Do the compare.
23670 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
23671
23672 // Make sure the lower and upper halves are both all-ones.
23673 static const int Mask[] = { 1, 0, 3, 2 };
23674 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
23675 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
23676
23677 if (Invert)
23678 Result = DAG.getNOT(dl, Result, MVT::v4i32);
23679
23680 return DAG.getBitcast(VT, Result);
23681 }
23682 }
23683
23684 // Since SSE has no unsigned integer comparisons, we need to flip the sign
23685 // bits of the inputs before performing those operations.
23686 if (FlipSigns) {
23687 MVT EltVT = VT.getVectorElementType();
23688 SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
23689 VT);
23690 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM);
23691 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM);
23692 }
23693
23694 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
23695
23696 // If the logical-not of the result is required, perform that now.
23697 if (Invert)
23698 Result = DAG.getNOT(dl, Result, VT);
23699
23700 return Result;
23701}
23702
23703// Try to select this as a KORTEST+SETCC or KTEST+SETCC if possible.
23704static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
23705 const SDLoc &dl, SelectionDAG &DAG,
23706 const X86Subtarget &Subtarget,
23707 SDValue &X86CC) {
23708 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
23709
23710 // Must be a bitcast from vXi1.
23711 if (Op0.getOpcode() != ISD::BITCAST)
23712 return SDValue();
23713
23714 Op0 = Op0.getOperand(0);
23715 MVT VT = Op0.getSimpleValueType();
23716 if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) &&
23717 !(Subtarget.hasDQI() && VT == MVT::v8i1) &&
23718 !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)))
23719 return SDValue();
23720
23721 X86::CondCode X86Cond;
23722 if (isNullConstant(Op1)) {
23723 X86Cond = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
23724 } else if (isAllOnesConstant(Op1)) {
23725 // C flag is set for all ones.
23726 X86Cond = CC == ISD::SETEQ ? X86::COND_B : X86::COND_AE;
23727 } else
23728 return SDValue();
23729
23730 // If the input is an AND, we can combine it's operands into the KTEST.
23731 bool KTestable = false;
23732 if (Subtarget.hasDQI() && (VT == MVT::v8i1 || VT == MVT::v16i1))
23733 KTestable = true;
23734 if (Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1))
23735 KTestable = true;
23736 if (!isNullConstant(Op1))
23737 KTestable = false;
23738 if (KTestable && Op0.getOpcode() == ISD::AND && Op0.hasOneUse()) {
23739 SDValue LHS = Op0.getOperand(0);
23740 SDValue RHS = Op0.getOperand(1);
23741 X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
23742 return DAG.getNode(X86ISD::KTEST, dl, MVT::i32, LHS, RHS);
23743 }
23744
23745 // If the input is an OR, we can combine it's operands into the KORTEST.
23746 SDValue LHS = Op0;
23747 SDValue RHS = Op0;
23748 if (Op0.getOpcode() == ISD::OR && Op0.hasOneUse()) {
23749 LHS = Op0.getOperand(0);
23750 RHS = Op0.getOperand(1);
23751 }
23752
23753 X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
23754 return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
23755}
23756
23757/// Emit flags for the given setcc condition and operands. Also returns the
23758/// corresponding X86 condition code constant in X86CC.
23759SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
23760 ISD::CondCode CC, const SDLoc &dl,
23761 SelectionDAG &DAG,
23762 SDValue &X86CC) const {
23763 // Equality Combines.
23764 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
23765 X86::CondCode X86CondCode;
23766
23767 // Optimize to BT if possible.
23768 // Lower (X & (1 << N)) == 0 to BT(X, N).
23769 // Lower ((X >>u N) & 1) != 0 to BT(X, N).
23770 // Lower ((X >>s N) & 1) != 0 to BT(X, N).
23771 if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() && isNullConstant(Op1)) {
23772 if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CondCode)) {
23773 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23774 return BT;
23775 }
23776 }
23777
23778 // Try to use PTEST/PMOVMSKB for a tree AND/ORs equality compared with -1/0.
23779 if (SDValue CmpZ = MatchVectorAllEqualTest(Op0, Op1, CC, dl, Subtarget, DAG,
23780 X86CondCode)) {
23781 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23782 return CmpZ;
23783 }
23784
23785 // Try to lower using KORTEST or KTEST.
23786 if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC))
23787 return Test;
23788
23789 // Look for X == 0, X == 1, X != 0, or X != 1. We can simplify some forms
23790 // of these.
23791 if (isOneConstant(Op1) || isNullConstant(Op1)) {
23792 // If the input is a setcc, then reuse the input setcc or use a new one
23793 // with the inverted condition.
23794 if (Op0.getOpcode() == X86ISD::SETCC) {
23795 bool Invert = (CC == ISD::SETNE) ^ isNullConstant(Op1);
23796
23797 X86CC = Op0.getOperand(0);
23798 if (Invert) {
23799 X86CondCode = (X86::CondCode)Op0.getConstantOperandVal(0);
23800 X86CondCode = X86::GetOppositeBranchCondition(X86CondCode);
23801 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23802 }
23803
23804 return Op0.getOperand(1);
23805 }
23806 }
23807
23808 // Look for X == INT_MIN or X != INT_MIN. We can use NEG and test for
23809 // overflow.
23810 if (isMinSignedConstant(Op1)) {
23811 EVT VT = Op0.getValueType();
23812 if (VT == MVT::i32 || VT == MVT::i64 || Op0->hasOneUse()) {
23813 SDVTList CmpVTs = DAG.getVTList(VT, MVT::i32);
23814 X86::CondCode CondCode = CC == ISD::SETEQ ? X86::COND_O : X86::COND_NO;
23815 X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
23816 SDValue Neg = DAG.getNode(X86ISD::SUB, dl, CmpVTs,
23817 DAG.getConstant(0, dl, VT), Op0);
23818 return SDValue(Neg.getNode(), 1);
23819 }
23820 }
23821
23822 // Try to use the carry flag from the add in place of an separate CMP for:
23823 // (seteq (add X, -1), -1). Similar for setne.
23824 if (isAllOnesConstant(Op1) && Op0.getOpcode() == ISD::ADD &&
23825 Op0.getOperand(1) == Op1) {
23826 if (isProfitableToUseFlagOp(Op0)) {
23827 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32);
23828
23829 SDValue New = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(0),
23830 Op0.getOperand(1));
23831 DAG.ReplaceAllUsesOfValueWith(SDValue(Op0.getNode(), 0), New);
23832 X86CondCode = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
23833 X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);
23834 return SDValue(New.getNode(), 1);
23835 }
23836 }
23837 }
23838
23839 X86::CondCode CondCode =
23840 TranslateX86CC(CC, dl, /*IsFP*/ false, Op0, Op1, DAG);
23841 assert(CondCode != X86::COND_INVALID && "Unexpected condition code!");
23842
23843 SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget);
23844 X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
23845 return EFLAGS;
23846}
23847
23848SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
23849
23850 bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC ||
23851 Op.getOpcode() == ISD::STRICT_FSETCCS;
23852 MVT VT = Op->getSimpleValueType(0);
23853
23854 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
23855
23856 assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");
23857 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
23858 SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
23859 SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
23860 SDLoc dl(Op);
23861 ISD::CondCode CC =
23862 cast<CondCodeSDNode>(Op.getOperand(IsStrict ? 3 : 2))->get();
23863
23864 if (isSoftF16(Op0.getValueType(), Subtarget))
23865 return SDValue();
23866
23867 // Handle f128 first, since one possible outcome is a normal integer
23868 // comparison which gets handled by emitFlagsForSetcc.
23869 if (Op0.getValueType() == MVT::f128) {
23870 softenSetCCOperands(DAG, MVT::f128, Op0, Op1, CC, dl, Op0, Op1, Chain,
23871 Op.getOpcode() == ISD::STRICT_FSETCCS);
23872
23873 // If softenSetCCOperands returned a scalar, use it.
23874 if (!Op1.getNode()) {
23875 assert(Op0.getValueType() == Op.getValueType() &&
23876 "Unexpected setcc expansion!");
23877 if (IsStrict)
23878 return DAG.getMergeValues({Op0, Chain}, dl);
23879 return Op0;
23880 }
23881 }
23882
23883 if (Op0.getSimpleValueType().isInteger()) {
23884 // Attempt to canonicalize SGT/UGT -> SGE/UGE compares with constant which
23885 // reduces the number of EFLAGs bit reads (the GE conditions don't read ZF),
23886 // this may translate to less uops depending on uarch implementation. The
23887 // equivalent for SLE/ULE -> SLT/ULT isn't likely to happen as we already
23888 // canonicalize to that CondCode.
23889 // NOTE: Only do this if incrementing the constant doesn't increase the bit
23890 // encoding size - so it must either already be a i8 or i32 immediate, or it
23891 // shrinks down to that. We don't do this for any i64's to avoid additional
23892 // constant materializations.
23893 // TODO: Can we move this to TranslateX86CC to handle jumps/branches too?
23894 if (auto *Op1C = dyn_cast<ConstantSDNode>(Op1)) {
23895 const APInt &Op1Val = Op1C->getAPIntValue();
23896 if (!Op1Val.isZero()) {
23897 // Ensure the constant+1 doesn't overflow.
23898 if ((CC == ISD::CondCode::SETGT && !Op1Val.isMaxSignedValue()) ||
23899 (CC == ISD::CondCode::SETUGT && !Op1Val.isMaxValue())) {
23900 APInt Op1ValPlusOne = Op1Val + 1;
23901 if (Op1ValPlusOne.isSignedIntN(32) &&
23902 (!Op1Val.isSignedIntN(8) || Op1ValPlusOne.isSignedIntN(8))) {
23903 Op1 = DAG.getConstant(Op1ValPlusOne, dl, Op0.getValueType());
23904 CC = CC == ISD::CondCode::SETGT ? ISD::CondCode::SETGE
23905 : ISD::CondCode::SETUGE;
23906 }
23907 }
23908 }
23909 }
23910
23911 SDValue X86CC;
23912 SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC);
23913 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
23914 return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
23915 }
23916
23917 // Handle floating point.
23918 X86::CondCode CondCode = TranslateX86CC(CC, dl, /*IsFP*/ true, Op0, Op1, DAG);
23919 if (CondCode == X86::COND_INVALID)
23920 return SDValue();
23921
23922 SDValue EFLAGS;
23923 if (IsStrict) {
23924 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
23925 EFLAGS =
23926 DAG.getNode(IsSignaling ? X86ISD::STRICT_FCMPS : X86ISD::STRICT_FCMP,
23927 dl, {MVT::i32, MVT::Other}, {Chain, Op0, Op1});
23928 Chain = EFLAGS.getValue(1);
23929 } else {
23930 EFLAGS = DAG.getNode(X86ISD::FCMP, dl, MVT::i32, Op0, Op1);
23931 }
23932
23933 SDValue X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
23934 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
23935 return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
23936}
23937
23938SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
23939 SDValue LHS = Op.getOperand(0);
23940 SDValue RHS = Op.getOperand(1);
23941 SDValue Carry = Op.getOperand(2);
23942 SDValue Cond = Op.getOperand(3);
23943 SDLoc DL(Op);
23944
23945 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
23946 X86::CondCode CC = TranslateIntegerX86CC(cast<CondCodeSDNode>(Cond)->get());
23947
23948 // Recreate the carry if needed.
23949 EVT CarryVT = Carry.getValueType();
23950 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
23951 Carry, DAG.getAllOnesConstant(DL, CarryVT));
23952
23953 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
23954 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
23955 return getSETCC(CC, Cmp.getValue(1), DL, DAG);
23956}
23957
23958// This function returns three things: the arithmetic computation itself
23959// (Value), an EFLAGS result (Overflow), and a condition code (Cond). The
23960// flag and the condition code define the case in which the arithmetic
23961// computation overflows.
23962static std::pair<SDValue, SDValue>
23963getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
23964 assert(Op.getResNo() == 0 && "Unexpected result number!");
23965 SDValue Value, Overflow;
23966 SDValue LHS = Op.getOperand(0);
23967 SDValue RHS = Op.getOperand(1);
23968 unsigned BaseOp = 0;
23969 SDLoc DL(Op);
23970 switch (Op.getOpcode()) {
23971 default: llvm_unreachable("Unknown ovf instruction!");
23972 case ISD::SADDO:
23973 BaseOp = X86ISD::ADD;
23974 Cond = X86::COND_O;
23975 break;
23976 case ISD::UADDO:
23977 BaseOp = X86ISD::ADD;
23978 Cond = isOneConstant(RHS) ? X86::COND_E : X86::COND_B;
23979 break;
23980 case ISD::SSUBO:
23981 BaseOp = X86ISD::SUB;
23982 Cond = X86::COND_O;
23983 break;
23984 case ISD::USUBO:
23985 BaseOp = X86ISD::SUB;
23986 Cond = X86::COND_B;
23987 break;
23988 case ISD::SMULO:
23989 BaseOp = X86ISD::SMUL;
23990 Cond = X86::COND_O;
23991 break;
23992 case ISD::UMULO:
23993 BaseOp = X86ISD::UMUL;
23994 Cond = X86::COND_O;
23995 break;
23996 }
23997
23998 if (BaseOp) {
23999 // Also sets EFLAGS.
24000 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
24001 Value = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
24002 Overflow = Value.getValue(1);
24003 }
24004
24005 return std::make_pair(Value, Overflow);
24006}
24007
24008static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
24009 // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
24010 // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
24011 // looks for this combo and may remove the "setcc" instruction if the "setcc"
24012 // has only one use.
24013 SDLoc DL(Op);
24014 X86::CondCode Cond;
24015 SDValue Value, Overflow;
24016 std::tie(Value, Overflow) = getX86XALUOOp(Cond, Op, DAG);
24017
24018 SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
24019 assert(Op->getValueType(1) == MVT::i8 && "Unexpected VT!");
24020 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC);
24021}
24022
24023/// Return true if opcode is a X86 logical comparison.
24024static bool isX86LogicalCmp(SDValue Op) {
24025 unsigned Opc = Op.getOpcode();
24026 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
24027 Opc == X86ISD::FCMP)
24028 return true;
24029 if (Op.getResNo() == 1 &&
24030 (Opc == X86ISD::ADD || Opc == X86ISD::SUB || Opc == X86ISD::ADC ||
24031 Opc == X86ISD::SBB || Opc == X86ISD::SMUL || Opc == X86ISD::UMUL ||
24032 Opc == X86ISD::OR || Opc == X86ISD::XOR || Opc == X86ISD::AND))
24033 return true;
24034
24035 return false;
24036}
24037
24038static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
24039 if (V.getOpcode() != ISD::TRUNCATE)
24040 return false;
24041
24042 SDValue VOp0 = V.getOperand(0);
24043 unsigned InBits = VOp0.getValueSizeInBits();
24044 unsigned Bits = V.getValueSizeInBits();
24045 return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
24046}
24047
24048SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
24049 bool AddTest = true;
24050 SDValue Cond = Op.getOperand(0);
24051 SDValue Op1 = Op.getOperand(1);
24052 SDValue Op2 = Op.getOperand(2);
24053 SDLoc DL(Op);
24054 MVT VT = Op1.getSimpleValueType();
24055 SDValue CC;
24056
24057 if (isSoftF16(VT, Subtarget)) {
24058 MVT NVT = VT.changeTypeToInteger();
24059 return DAG.getBitcast(VT, DAG.getNode(ISD::SELECT, DL, NVT, Cond,
24060 DAG.getBitcast(NVT, Op1),
24061 DAG.getBitcast(NVT, Op2)));
24062 }
24063
24064 // Lower FP selects into a CMP/AND/ANDN/OR sequence when the necessary SSE ops
24065 // are available or VBLENDV if AVX is available.
24066 // Otherwise FP cmovs get lowered into a less efficient branch sequence later.
24067 if (Cond.getOpcode() == ISD::SETCC && isScalarFPTypeInSSEReg(VT) &&
24068 VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) {
24069 SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
24070 bool IsAlwaysSignaling;
24071 unsigned SSECC =
24072 translateX86FSETCC(cast<CondCodeSDNode>(Cond.getOperand(2))->get(),
24073 CondOp0, CondOp1, IsAlwaysSignaling);
24074
24075 if (Subtarget.hasAVX512()) {
24076 SDValue Cmp =
24077 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1,
24078 DAG.getTargetConstant(SSECC, DL, MVT::i8));
24079 assert(!VT.isVector() && "Not a scalar type?");
24080 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
24081 }
24082
24083 if (SSECC < 8 || Subtarget.hasAVX()) {
24084 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
24085 DAG.getTargetConstant(SSECC, DL, MVT::i8));
24086
24087 // If we have AVX, we can use a variable vector select (VBLENDV) instead
24088 // of 3 logic instructions for size savings and potentially speed.
24089 // Unfortunately, there is no scalar form of VBLENDV.
24090
24091 // If either operand is a +0.0 constant, don't try this. We can expect to
24092 // optimize away at least one of the logic instructions later in that
24093 // case, so that sequence would be faster than a variable blend.
24094
24095 // BLENDV was introduced with SSE 4.1, but the 2 register form implicitly
24096 // uses XMM0 as the selection register. That may need just as many
24097 // instructions as the AND/ANDN/OR sequence due to register moves, so
24098 // don't bother.
24099 if (Subtarget.hasAVX() && !isNullFPConstant(Op1) &&
24100 !isNullFPConstant(Op2)) {
24101 // Convert to vectors, do a VSELECT, and convert back to scalar.
24102 // All of the conversions should be optimized away.
24103 MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64;
24104 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
24105 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2);
24106 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
24107
24108 MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64;
24109 VCmp = DAG.getBitcast(VCmpVT, VCmp);
24110
24111 SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2);
24112
24113 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
24114 VSel, DAG.getIntPtrConstant(0, DL));
24115 }
24116 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
24117 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
24118 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
24119 }
24120 }
24121
24122 // AVX512 fallback is to lower selects of scalar floats to masked moves.
24123 if (isScalarFPTypeInSSEReg(VT) && Subtarget.hasAVX512()) {
24124 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);
24125 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
24126 }
24127
24128 if (Cond.getOpcode() == ISD::SETCC &&
24129 !isSoftF16(Cond.getOperand(0).getSimpleValueType(), Subtarget)) {
24130 if (SDValue NewCond = LowerSETCC(Cond, DAG)) {
24131 Cond = NewCond;
24132 // If the condition was updated, it's possible that the operands of the
24133 // select were also updated (for example, EmitTest has a RAUW). Refresh
24134 // the local references to the select operands in case they got stale.
24135 Op1 = Op.getOperand(1);
24136 Op2 = Op.getOperand(2);
24137 }
24138 }
24139
24140 // (select (x == 0), -1, y) -> (sign_bit (x - 1)) | y
24141 // (select (x == 0), y, -1) -> ~(sign_bit (x - 1)) | y
24142 // (select (x != 0), y, -1) -> (sign_bit (x - 1)) | y
24143 // (select (x != 0), -1, y) -> ~(sign_bit (x - 1)) | y
24144 // (select (and (x , 0x1) == 0), y, (z ^ y) ) -> (-(and (x , 0x1)) & z ) ^ y
24145 // (select (and (x , 0x1) == 0), y, (z | y) ) -> (-(and (x , 0x1)) & z ) | y
24146 // (select (x > 0), x, 0) -> (~(x >> (size_in_bits(x)-1))) & x
24147 // (select (x < 0), x, 0) -> ((x >> (size_in_bits(x)-1))) & x
24148 if (Cond.getOpcode() == X86ISD::SETCC &&
24149 Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
24150 isNullConstant(Cond.getOperand(1).getOperand(1))) {
24151 SDValue Cmp = Cond.getOperand(1);
24152 SDValue CmpOp0 = Cmp.getOperand(0);
24153 unsigned CondCode = Cond.getConstantOperandVal(0);
24154
24155 // Special handling for __builtin_ffs(X) - 1 pattern which looks like
24156 // (select (seteq X, 0), -1, (cttz_zero_undef X)). Disable the special
24157 // handle to keep the CMP with 0. This should be removed by
24158 // optimizeCompareInst by using the flags from the BSR/TZCNT used for the
24159 // cttz_zero_undef.
24160 auto MatchFFSMinus1 = [&](SDValue Op1, SDValue Op2) {
24161 return (Op1.getOpcode() == ISD::CTTZ_ZERO_UNDEF && Op1.hasOneUse() &&
24162 Op1.getOperand(0) == CmpOp0 && isAllOnesConstant(Op2));
24163 };
24164 if (Subtarget.canUseCMOV() && (VT == MVT::i32 || VT == MVT::i64) &&
24165 ((CondCode == X86::COND_NE && MatchFFSMinus1(Op1, Op2)) ||
24166 (CondCode == X86::COND_E && MatchFFSMinus1(Op2, Op1)))) {
24167 // Keep Cmp.
24168 } else if ((isAllOnesConstant(Op1) || isAllOnesConstant(Op2)) &&
24169 (CondCode == X86::COND_E || CondCode == X86::COND_NE)) {
24170 SDValue Y = isAllOnesConstant(Op2) ? Op1 : Op2;
24171 SDVTList CmpVTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32);
24172
24173 // 'X - 1' sets the carry flag if X == 0.
24174 // '0 - X' sets the carry flag if X != 0.
24175 // Convert the carry flag to a -1/0 mask with sbb:
24176 // select (X != 0), -1, Y --> 0 - X; or (sbb), Y
24177 // select (X == 0), Y, -1 --> 0 - X; or (sbb), Y
24178 // select (X != 0), Y, -1 --> X - 1; or (sbb), Y
24179 // select (X == 0), -1, Y --> X - 1; or (sbb), Y
24180 SDValue Sub;
24181 if (isAllOnesConstant(Op1) == (CondCode == X86::COND_NE)) {
24182 SDValue Zero = DAG.getConstant(0, DL, CmpOp0.getValueType());
24183 Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, Zero, CmpOp0);
24184 } else {
24185 SDValue One = DAG.getConstant(1, DL, CmpOp0.getValueType());
24186 Sub = DAG.getNode(X86ISD::SUB, DL, CmpVTs, CmpOp0, One);
24187 }
24188 SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
24189 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
24190 Sub.getValue(1));
24191 return DAG.getNode(ISD::OR, DL, VT, SBB, Y);
24192 } else if (!Subtarget.canUseCMOV() && CondCode == X86::COND_E &&
24193 CmpOp0.getOpcode() == ISD::AND &&
24194 isOneConstant(CmpOp0.getOperand(1))) {
24195 SDValue Src1, Src2;
24196 // true if Op2 is XOR or OR operator and one of its operands
24197 // is equal to Op1
24198 // ( a , a op b) || ( b , a op b)
24199 auto isOrXorPattern = [&]() {
24200 if ((Op2.getOpcode() == ISD::XOR || Op2.getOpcode() == ISD::OR) &&
24201 (Op2.getOperand(0) == Op1 || Op2.getOperand(1) == Op1)) {
24202 Src1 =
24203 Op2.getOperand(0) == Op1 ? Op2.getOperand(1) : Op2.getOperand(0);
24204 Src2 = Op1;
24205 return true;
24206 }
24207 return false;
24208 };
24209
24210 if (isOrXorPattern()) {
24211 SDValue Neg;
24212 unsigned int CmpSz = CmpOp0.getSimpleValueType().getSizeInBits();
24213 // we need mask of all zeros or ones with same size of the other
24214 // operands.
24215 if (CmpSz > VT.getSizeInBits())
24216 Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0);
24217 else if (CmpSz < VT.getSizeInBits())
24218 Neg = DAG.getNode(ISD::AND, DL, VT,
24219 DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)),
24220 DAG.getConstant(1, DL, VT));
24221 else
24222 Neg = CmpOp0;
24223 SDValue Mask = DAG.getNegative(Neg, DL, VT); // -(and (x, 0x1))
24224 SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z
24225 return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2); // And Op y
24226 }
24227 } else if ((VT == MVT::i32 || VT == MVT::i64) && isNullConstant(Op2) &&
24228 Cmp.getNode()->hasOneUse() && (CmpOp0 == Op1) &&
24229 ((CondCode == X86::COND_S) || // smin(x, 0)
24230 (CondCode == X86::COND_G && hasAndNot(Op1)))) { // smax(x, 0)
24231 // (select (x < 0), x, 0) -> ((x >> (size_in_bits(x)-1))) & x
24232 //
24233 // If the comparison is testing for a positive value, we have to invert
24234 // the sign bit mask, so only do that transform if the target has a
24235 // bitwise 'and not' instruction (the invert is free).
24236 // (select (x > 0), x, 0) -> (~(x >> (size_in_bits(x)-1))) & x
24237 unsigned ShCt = VT.getSizeInBits() - 1;
24238 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, VT);
24239 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, Op1, ShiftAmt);
24240 if (CondCode == X86::COND_G)
24241 Shift = DAG.getNOT(DL, Shift, VT);
24242 return DAG.getNode(ISD::AND, DL, VT, Shift, Op1);
24243 }
24244 }
24245
24246 // Look past (and (setcc_carry (cmp ...)), 1).
24247 if (Cond.getOpcode() == ISD::AND &&
24248 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
24249 isOneConstant(Cond.getOperand(1)))
24250 Cond = Cond.getOperand(0);
24251
24252 // If condition flag is set by a X86ISD::CMP, then use it as the condition
24253 // setting operand in place of the X86ISD::SETCC.
24254 unsigned CondOpcode = Cond.getOpcode();
24255 if (CondOpcode == X86ISD::SETCC ||
24256 CondOpcode == X86ISD::SETCC_CARRY) {
24257 CC = Cond.getOperand(0);
24258
24259 SDValue Cmp = Cond.getOperand(1);
24260 bool IllegalFPCMov = false;
24261 if (VT.isFloatingPoint() && !VT.isVector() &&
24262 !isScalarFPTypeInSSEReg(VT) && Subtarget.canUseCMOV()) // FPStack?
24263 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
24264
24265 if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
24266 Cmp.getOpcode() == X86ISD::BT) { // FIXME
24267 Cond = Cmp;
24268 AddTest = false;
24269 }
24270 } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
24271 CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
24272 CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) {
24273 SDValue Value;
24274 X86::CondCode X86Cond;
24275 std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
24276
24277 CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);
24278 AddTest = false;
24279 }
24280
24281 if (AddTest) {
24282 // Look past the truncate if the high bits are known zero.
24283 if (isTruncWithZeroHighBitsInput(Cond, DAG))
24284 Cond = Cond.getOperand(0);
24285
24286 // We know the result of AND is compared against zero. Try to match
24287 // it to BT.
24288 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
24289 X86::CondCode X86CondCode;
24290 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, X86CondCode)) {
24291 CC = DAG.getTargetConstant(X86CondCode, DL, MVT::i8);
24292 Cond = BT;
24293 AddTest = false;
24294 }
24295 }
24296 }
24297
24298 if (AddTest) {
24299 CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
24300 Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
24301 }
24302
24303 // a < b ? -1 : 0 -> RES = ~setcc_carry
24304 // a < b ? 0 : -1 -> RES = setcc_carry
24305 // a >= b ? -1 : 0 -> RES = setcc_carry
24306 // a >= b ? 0 : -1 -> RES = ~setcc_carry
24307 if (Cond.getOpcode() == X86ISD::SUB) {
24308 unsigned CondCode = CC->getAsZExtVal();
24309
24310 if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) &&
24311 (isAllOnesConstant(Op1) || isAllOnesConstant(Op2)) &&
24312 (isNullConstant(Op1) || isNullConstant(Op2))) {
24313 SDValue Res =
24314 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
24315 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond);
24316 if (isAllOnesConstant(Op1) != (CondCode == X86::COND_B))
24317 return DAG.getNOT(DL, Res, Res.getValueType());
24318 return Res;
24319 }
24320 }
24321
24322 // X86 doesn't have an i8 cmov. If both operands are the result of a truncate
24323 // widen the cmov and push the truncate through. This avoids introducing a new
24324 // branch during isel and doesn't add any extensions.
24325 if (Op.getValueType() == MVT::i8 &&
24326 Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) {
24327 SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
24328 if (T1.getValueType() == T2.getValueType() &&
24329 // Exclude CopyFromReg to avoid partial register stalls.
24330 T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){
24331 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
24332 CC, Cond);
24333 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
24334 }
24335 }
24336
24337 // Or finally, promote i8 cmovs if we have CMOV,
24338 // or i16 cmovs if it won't prevent folding a load.
24339 // FIXME: we should not limit promotion of i8 case to only when the CMOV is
24340 // legal, but EmitLoweredSelect() can not deal with these extensions
24341 // being inserted between two CMOV's. (in i16 case too TBN)
24342 // https://bugs.llvm.org/show_bug.cgi?id=40974
24343 if ((Op.getValueType() == MVT::i8 && Subtarget.canUseCMOV()) ||
24344 (Op.getValueType() == MVT::i16 && !X86::mayFoldLoad(Op1, Subtarget) &&
24345 !X86::mayFoldLoad(Op2, Subtarget))) {
24346 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
24347 Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
24348 SDValue Ops[] = { Op2, Op1, CC, Cond };
24349 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
24350 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
24351 }
24352
24353 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
24354 // condition is true.
24355 SDValue Ops[] = { Op2, Op1, CC, Cond };
24356 return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops, Op->getFlags());
24357}
24358
24359static SDValue LowerSIGN_EXTEND_Mask(SDValue Op, const SDLoc &dl,
24360 const X86Subtarget &Subtarget,
24361 SelectionDAG &DAG) {
24362 MVT VT = Op->getSimpleValueType(0);
24363 SDValue In = Op->getOperand(0);
24364 MVT InVT = In.getSimpleValueType();
24365 assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!");
24366 MVT VTElt = VT.getVectorElementType();
24367 unsigned NumElts = VT.getVectorNumElements();
24368
24369 // Extend VT if the scalar type is i8/i16 and BWI is not supported.
24370 MVT ExtVT = VT;
24371 if (!Subtarget.hasBWI() && VTElt.getSizeInBits() <= 16) {
24372 // If v16i32 is to be avoided, we'll need to split and concatenate.
24373 if (NumElts == 16 && !Subtarget.canExtendTo512DQ())
24374 return SplitAndExtendv16i1(Op.getOpcode(), VT, In, dl, DAG);
24375
24376 ExtVT = MVT::getVectorVT(MVT::i32, NumElts);
24377 }
24378
24379 // Widen to 512-bits if VLX is not supported.
24380 MVT WideVT = ExtVT;
24381 if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) {
24382 NumElts *= 512 / ExtVT.getSizeInBits();
24383 InVT = MVT::getVectorVT(MVT::i1, NumElts);
24384 In = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, InVT, DAG.getUNDEF(InVT),
24385 In, DAG.getIntPtrConstant(0, dl));
24386 WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(), NumElts);
24387 }
24388
24389 SDValue V;
24390 MVT WideEltVT = WideVT.getVectorElementType();
24391 if ((Subtarget.hasDQI() && WideEltVT.getSizeInBits() >= 32) ||
24392 (Subtarget.hasBWI() && WideEltVT.getSizeInBits() <= 16)) {
24393 V = DAG.getNode(Op.getOpcode(), dl, WideVT, In);
24394 } else {
24395 SDValue NegOne = DAG.getAllOnesConstant(dl, WideVT);
24396 SDValue Zero = DAG.getConstant(0, dl, WideVT);
24397 V = DAG.getSelect(dl, WideVT, In, NegOne, Zero);
24398 }
24399
24400 // Truncate if we had to extend i16/i8 above.
24401 if (VT != ExtVT) {
24402 WideVT = MVT::getVectorVT(VTElt, NumElts);
24403 V = DAG.getNode(ISD::TRUNCATE, dl, WideVT, V);
24404 }
24405
24406 // Extract back to 128/256-bit if we widened.
24407 if (WideVT != VT)
24408 V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V,
24409 DAG.getIntPtrConstant(0, dl));
24410
24411 return V;
24412}
24413
24414static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
24415 SelectionDAG &DAG) {
24416 SDValue In = Op->getOperand(0);
24417 MVT InVT = In.getSimpleValueType();
24418 SDLoc DL(Op);
24419
24420 if (InVT.getVectorElementType() == MVT::i1)
24421 return LowerSIGN_EXTEND_Mask(Op, DL, Subtarget, DAG);
24422
24423 assert(Subtarget.hasAVX() && "Expected AVX support");
24424 return LowerAVXExtend(Op, DL, DAG, Subtarget);
24425}
24426
24427// Lowering for SIGN_EXTEND_VECTOR_INREG and ZERO_EXTEND_VECTOR_INREG.
24428// For sign extend this needs to handle all vector sizes and SSE4.1 and
24429// non-SSE4.1 targets. For zero extend this should only handle inputs of
24430// MVT::v64i8 when BWI is not supported, but AVX512 is.
24431static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
24432 const X86Subtarget &Subtarget,
24433 SelectionDAG &DAG) {
24434 SDValue In = Op->getOperand(0);
24435 MVT VT = Op->getSimpleValueType(0);
24436 MVT InVT = In.getSimpleValueType();
24437
24438 MVT SVT = VT.getVectorElementType();
24439 MVT InSVT = InVT.getVectorElementType();
24440 assert(SVT.getFixedSizeInBits() > InSVT.getFixedSizeInBits());
24441
24442 if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16)
24443 return SDValue();
24444 if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8)
24445 return SDValue();
24446 if (!(VT.is128BitVector() && Subtarget.hasSSE2()) &&
24447 !(VT.is256BitVector() && Subtarget.hasAVX()) &&
24448 !(VT.is512BitVector() && Subtarget.hasAVX512()))
24449 return SDValue();
24450
24451 SDLoc dl(Op);
24452 unsigned Opc = Op.getOpcode();
24453 unsigned NumElts = VT.getVectorNumElements();
24454
24455 // For 256-bit vectors, we only need the lower (128-bit) half of the input.
24456 // For 512-bit vectors, we need 128-bits or 256-bits.
24457 if (InVT.getSizeInBits() > 128) {
24458 // Input needs to be at least the same number of elements as output, and
24459 // at least 128-bits.
24460 int InSize = InSVT.getSizeInBits() * NumElts;
24461 In = extractSubVector(In, 0, DAG, dl, std::max(InSize, 128));
24462 InVT = In.getSimpleValueType();
24463 }
24464
24465 // SSE41 targets can use the pmov[sz]x* instructions directly for 128-bit results,
24466 // so are legal and shouldn't occur here. AVX2/AVX512 pmovsx* instructions still
24467 // need to be handled here for 256/512-bit results.
24468 if (Subtarget.hasInt256()) {
24469 assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension");
24470
24471 if (InVT.getVectorNumElements() != NumElts)
24472 return DAG.getNode(Op.getOpcode(), dl, VT, In);
24473
24474 // FIXME: Apparently we create inreg operations that could be regular
24475 // extends.
24476 unsigned ExtOpc =
24477 Opc == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SIGN_EXTEND
24478 : ISD::ZERO_EXTEND;
24479 return DAG.getNode(ExtOpc, dl, VT, In);
24480 }
24481
24482 // pre-AVX2 256-bit extensions need to be split into 128-bit instructions.
24483 if (Subtarget.hasAVX()) {
24484 assert(VT.is256BitVector() && "256-bit vector expected");
24485 MVT HalfVT = VT.getHalfNumVectorElementsVT();
24486 int HalfNumElts = HalfVT.getVectorNumElements();
24487
24488 unsigned NumSrcElts = InVT.getVectorNumElements();
24489 SmallVector<int, 16> HiMask(NumSrcElts, SM_SentinelUndef);
24490 for (int i = 0; i != HalfNumElts; ++i)
24491 HiMask[i] = HalfNumElts + i;
24492
24493 SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In);
24494 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, DAG.getUNDEF(InVT), HiMask);
24495 Hi = DAG.getNode(Opc, dl, HalfVT, Hi);
24496 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
24497 }
24498
24499 // We should only get here for sign extend.
24500 assert(Opc == ISD::SIGN_EXTEND_VECTOR_INREG && "Unexpected opcode!");
24501 assert(VT.is128BitVector() && InVT.is128BitVector() && "Unexpected VTs");
24502 unsigned InNumElts = InVT.getVectorNumElements();
24503
24504 // If the source elements are already all-signbits, we don't need to extend,
24505 // just splat the elements.
24506 APInt DemandedElts = APInt::getLowBitsSet(InNumElts, NumElts);
24507 if (DAG.ComputeNumSignBits(In, DemandedElts) == InVT.getScalarSizeInBits()) {
24508 unsigned Scale = InNumElts / NumElts;
24509 SmallVector<int, 16> ShuffleMask;
24510 for (unsigned I = 0; I != NumElts; ++I)
24511 ShuffleMask.append(Scale, I);
24512 return DAG.getBitcast(VT,
24513 DAG.getVectorShuffle(InVT, dl, In, In, ShuffleMask));
24514 }
24515
24516 // pre-SSE41 targets unpack lower lanes and then sign-extend using SRAI.
24517 SDValue Curr = In;
24518 SDValue SignExt = Curr;
24519
24520 // As SRAI is only available on i16/i32 types, we expand only up to i32
24521 // and handle i64 separately.
24522 if (InVT != MVT::v4i32) {
24523 MVT DestVT = VT == MVT::v2i64 ? MVT::v4i32 : VT;
24524
24525 unsigned DestWidth = DestVT.getScalarSizeInBits();
24526 unsigned Scale = DestWidth / InSVT.getSizeInBits();
24527 unsigned DestElts = DestVT.getVectorNumElements();
24528
24529 // Build a shuffle mask that takes each input element and places it in the
24530 // MSBs of the new element size.
24531 SmallVector<int, 16> Mask(InNumElts, SM_SentinelUndef);
24532 for (unsigned i = 0; i != DestElts; ++i)
24533 Mask[i * Scale + (Scale - 1)] = i;
24534
24535 Curr = DAG.getVectorShuffle(InVT, dl, In, In, Mask);
24536 Curr = DAG.getBitcast(DestVT, Curr);
24537
24538 unsigned SignExtShift = DestWidth - InSVT.getSizeInBits();
24539 SignExt = DAG.getNode(X86ISD::VSRAI, dl, DestVT, Curr,
24540 DAG.getTargetConstant(SignExtShift, dl, MVT::i8));
24541 }
24542
24543 if (VT == MVT::v2i64) {
24544 assert(Curr.getValueType() == MVT::v4i32 && "Unexpected input VT");
24545 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
24546 SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT);
24547 SignExt = DAG.getVectorShuffle(MVT::v4i32, dl, SignExt, Sign, {0, 4, 1, 5});
24548 SignExt = DAG.getBitcast(VT, SignExt);
24549 }
24550
24551 return SignExt;
24552}
24553
24554static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
24555 SelectionDAG &DAG) {
24556 MVT VT = Op->getSimpleValueType(0);
24557 SDValue In = Op->getOperand(0);
24558 MVT InVT = In.getSimpleValueType();
24559 SDLoc dl(Op);
24560
24561 if (InVT.getVectorElementType() == MVT::i1)
24562 return LowerSIGN_EXTEND_Mask(Op, dl, Subtarget, DAG);
24563
24564 assert(VT.isVector() && InVT.isVector() && "Expected vector type");
24565 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
24566 "Expected same number of elements");
24567 assert((VT.getVectorElementType() == MVT::i16 ||
24568 VT.getVectorElementType() == MVT::i32 ||
24569 VT.getVectorElementType() == MVT::i64) &&
24570 "Unexpected element type");
24571 assert((InVT.getVectorElementType() == MVT::i8 ||
24572 InVT.getVectorElementType() == MVT::i16 ||
24573 InVT.getVectorElementType() == MVT::i32) &&
24574 "Unexpected element type");
24575
24576 if (VT == MVT::v32i16 && !Subtarget.hasBWI()) {
24577 assert(InVT == MVT::v32i8 && "Unexpected VT!");
24578 return splitVectorIntUnary(Op, DAG, dl);
24579 }
24580
24581 if (Subtarget.hasInt256())
24582 return Op;
24583
24584 // Optimize vectors in AVX mode
24585 // Sign extend v8i16 to v8i32 and
24586 // v4i32 to v4i64
24587 //
24588 // Divide input vector into two parts
24589 // for v4i32 the high shuffle mask will be {2, 3, -1, -1}
24590 // use vpmovsx instruction to extend v4i32 -> v2i64; v8i16 -> v4i32
24591 // concat the vectors to original VT
24592 MVT HalfVT = VT.getHalfNumVectorElementsVT();
24593 SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
24594
24595 unsigned NumElems = InVT.getVectorNumElements();
24596 SmallVector<int,8> ShufMask(NumElems, -1);
24597 for (unsigned i = 0; i != NumElems/2; ++i)
24598 ShufMask[i] = i + NumElems/2;
24599
24600 SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
24601 OpHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, OpHi);
24602
24603 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
24604}
24605
24606/// Change a vector store into a pair of half-size vector stores.
24607static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) {
24608 SDValue StoredVal = Store->getValue();
24609 assert((StoredVal.getValueType().is256BitVector() ||
24610 StoredVal.getValueType().is512BitVector()) &&
24611 "Expecting 256/512-bit op");
24612
24613 // Splitting volatile memory ops is not allowed unless the operation was not
24614 // legal to begin with. Assume the input store is legal (this transform is
24615 // only used for targets with AVX). Note: It is possible that we have an
24616 // illegal type like v2i128, and so we could allow splitting a volatile store
24617 // in that case if that is important.
24618 if (!Store->isSimple())
24619 return SDValue();
24620
24621 SDLoc DL(Store);
24622 SDValue Value0, Value1;
24623 std::tie(Value0, Value1) = splitVector(StoredVal, DAG, DL);
24624 unsigned HalfOffset = Value0.getValueType().getStoreSize();
24625 SDValue Ptr0 = Store->getBasePtr();
24626 SDValue Ptr1 =
24627 DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(HalfOffset), DL);
24628 SDValue Ch0 =
24629 DAG.getStore(Store->getChain(), DL, Value0, Ptr0, Store->getPointerInfo(),
24630 Store->getOriginalAlign(),
24631 Store->getMemOperand()->getFlags());
24632 SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1,
24633 Store->getPointerInfo().getWithOffset(HalfOffset),
24634 Store->getOriginalAlign(),
24635 Store->getMemOperand()->getFlags());
24636 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Ch0, Ch1);
24637}
24638
24639/// Scalarize a vector store, bitcasting to TargetVT to determine the scalar
24640/// type.
24641static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT,
24642 SelectionDAG &DAG) {
24643 SDValue StoredVal = Store->getValue();
24644 assert(StoreVT.is128BitVector() &&
24645 StoredVal.getValueType().is128BitVector() && "Expecting 128-bit op");
24646 StoredVal = DAG.getBitcast(StoreVT, StoredVal);
24647
24648 // Splitting volatile memory ops is not allowed unless the operation was not
24649 // legal to begin with. We are assuming the input op is legal (this transform
24650 // is only used for targets with AVX).
24651 if (!Store->isSimple())
24652 return SDValue();
24653
24654 MVT StoreSVT = StoreVT.getScalarType();
24655 unsigned NumElems = StoreVT.getVectorNumElements();
24656 unsigned ScalarSize = StoreSVT.getStoreSize();
24657
24658 SDLoc DL(Store);
24659 SmallVector<SDValue, 4> Stores;
24660 for (unsigned i = 0; i != NumElems; ++i) {
24661 unsigned Offset = i * ScalarSize;
24662 SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(),
24663 TypeSize::getFixed(Offset), DL);
24664 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal,
24665 DAG.getIntPtrConstant(i, DL));
24666 SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr,
24667 Store->getPointerInfo().getWithOffset(Offset),
24668 Store->getOriginalAlign(),
24669 Store->getMemOperand()->getFlags());
24670 Stores.push_back(Ch);
24671 }
24672 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
24673}
24674
24675static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
24676 SelectionDAG &DAG) {
24677 StoreSDNode *St = cast<StoreSDNode>(Op.getNode());
24678 SDLoc dl(St);
24679 SDValue StoredVal = St->getValue();
24680
24681 // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 stores.
24682 if (StoredVal.getValueType().isVector() &&
24683 StoredVal.getValueType().getVectorElementType() == MVT::i1) {
24684 unsigned NumElts = StoredVal.getValueType().getVectorNumElements();
24685 assert(NumElts <= 8 && "Unexpected VT");
24686 assert(!St->isTruncatingStore() && "Expected non-truncating store");
24687 assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
24688 "Expected AVX512F without AVX512DQI");
24689
24690 // We must pad with zeros to ensure we store zeroes to any unused bits.
24691 StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
24692 DAG.getUNDEF(MVT::v16i1), StoredVal,
24693 DAG.getIntPtrConstant(0, dl));
24694 StoredVal = DAG.getBitcast(MVT::i16, StoredVal);
24695 StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal);
24696 // Make sure we store zeros in the extra bits.
24697 if (NumElts < 8)
24698 StoredVal = DAG.getZeroExtendInReg(
24699 StoredVal, dl, EVT::getIntegerVT(*DAG.getContext(), NumElts));
24700
24701 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
24702 St->getPointerInfo(), St->getOriginalAlign(),
24703 St->getMemOperand()->getFlags());
24704 }
24705
24706 if (St->isTruncatingStore())
24707 return SDValue();
24708
24709 // If this is a 256-bit store of concatenated ops, we are better off splitting
24710 // that store into two 128-bit stores. This avoids spurious use of 256-bit ops
24711 // and each half can execute independently. Some cores would split the op into
24712 // halves anyway, so the concat (vinsertf128) is purely an extra op.
24713 MVT StoreVT = StoredVal.getSimpleValueType();
24714 if (StoreVT.is256BitVector() ||
24715 ((StoreVT == MVT::v32i16 || StoreVT == MVT::v64i8) &&
24716 !Subtarget.hasBWI())) {
24717 if (StoredVal.hasOneUse() && isFreeToSplitVector(StoredVal.getNode(), DAG))
24718 return splitVectorStore(St, DAG);
24719 return SDValue();
24720 }
24721
24722 if (StoreVT.is32BitVector())
24723 return SDValue();
24724
24725 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24726 assert(StoreVT.is64BitVector() && "Unexpected VT");
24727 assert(TLI.getTypeAction(*DAG.getContext(), StoreVT) ==
24728 TargetLowering::TypeWidenVector &&
24729 "Unexpected type action!");
24730
24731 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StoreVT);
24732 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, StoredVal,
24733 DAG.getUNDEF(StoreVT));
24734
24735 if (Subtarget.hasSSE2()) {
24736 // Widen the vector, cast to a v2x64 type, extract the single 64-bit element
24737 // and store it.
24738 MVT StVT = Subtarget.is64Bit() && StoreVT.isInteger() ? MVT::i64 : MVT::f64;
24739 MVT CastVT = MVT::getVectorVT(StVT, 2);
24740 StoredVal = DAG.getBitcast(CastVT, StoredVal);
24741 StoredVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, StVT, StoredVal,
24742 DAG.getIntPtrConstant(0, dl));
24743
24744 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
24745 St->getPointerInfo(), St->getOriginalAlign(),
24746 St->getMemOperand()->getFlags());
24747 }
24748 assert(Subtarget.hasSSE1() && "Expected SSE");
24749 SDVTList Tys = DAG.getVTList(MVT::Other);
24750 SDValue Ops[] = {St->getChain(), StoredVal, St->getBasePtr()};
24751 return DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops, MVT::i64,
24752 St->getMemOperand());
24753}
24754
24755// Lower vector extended loads using a shuffle. If SSSE3 is not available we
24756// may emit an illegal shuffle but the expansion is still better than scalar
24757// code. We generate sext/sext_invec for SEXTLOADs if it's available, otherwise
24758// we'll emit a shuffle and a arithmetic shift.
24759// FIXME: Is the expansion actually better than scalar code? It doesn't seem so.
24760// TODO: It is possible to support ZExt by zeroing the undef values during
24761// the shuffle phase or after the shuffle.
24762static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
24763 SelectionDAG &DAG) {
24764 MVT RegVT = Op.getSimpleValueType();
24765 assert(RegVT.isVector() && "We only custom lower vector loads.");
24766 assert(RegVT.isInteger() &&
24767 "We only custom lower integer vector loads.");
24768
24769 LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
24770 SDLoc dl(Ld);
24771
24772 // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 loads.
24773 if (RegVT.getVectorElementType() == MVT::i1) {
24774 assert(EVT(RegVT) == Ld->getMemoryVT() && "Expected non-extending load");
24775 assert(RegVT.getVectorNumElements() <= 8 && "Unexpected VT");
24776 assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
24777 "Expected AVX512F without AVX512DQI");
24778
24779 SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(),
24780 Ld->getPointerInfo(), Ld->getOriginalAlign(),
24781 Ld->getMemOperand()->getFlags());
24782
24783 // Replace chain users with the new chain.
24784 assert(NewLd->getNumValues() == 2 && "Loads must carry a chain!");
24785
24786 SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);
24787 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RegVT,
24788 DAG.getBitcast(MVT::v16i1, Val),
24789 DAG.getIntPtrConstant(0, dl));
24790 return DAG.getMergeValues({Val, NewLd.getValue(1)}, dl);
24791 }
24792
24793 return SDValue();
24794}
24795
24796/// Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes
24797/// each of which has no other use apart from the AND / OR.
24798static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
24799 Opc = Op.getOpcode();
24800 if (Opc != ISD::OR && Opc != ISD::AND)
24801 return false;
24802 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
24803 Op.getOperand(0).hasOneUse() &&
24804 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
24805 Op.getOperand(1).hasOneUse());
24806}
24807
24808SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
24809 SDValue Chain = Op.getOperand(0);
24810 SDValue Cond = Op.getOperand(1);
24811 SDValue Dest = Op.getOperand(2);
24812 SDLoc dl(Op);
24813
24814 // Bail out when we don't have native compare instructions.
24815 if (Cond.getOpcode() == ISD::SETCC &&
24816 Cond.getOperand(0).getValueType() != MVT::f128 &&
24817 !isSoftF16(Cond.getOperand(0).getValueType(), Subtarget)) {
24818 SDValue LHS = Cond.getOperand(0);
24819 SDValue RHS = Cond.getOperand(1);
24820 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
24821
24822 // Special case for
24823 // setcc([su]{add,sub,mul}o == 0)
24824 // setcc([su]{add,sub,mul}o != 1)
24825 if (ISD::isOverflowIntrOpRes(LHS) &&
24826 (CC == ISD::SETEQ || CC == ISD::SETNE) &&
24827 (isNullConstant(RHS) || isOneConstant(RHS))) {
24828 SDValue Value, Overflow;
24829 X86::CondCode X86Cond;
24830 std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, LHS.getValue(0), DAG);
24831
24832 if ((CC == ISD::SETEQ) == isNullConstant(RHS))
24833 X86Cond = X86::GetOppositeBranchCondition(X86Cond);
24834
24835 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24836 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24837 Overflow, Op->getFlags());
24838 }
24839
24840 if (LHS.getSimpleValueType().isInteger()) {
24841 SDValue CCVal;
24842 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal);
24843 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24844 EFLAGS, Op->getFlags());
24845 }
24846
24847 if (CC == ISD::SETOEQ) {
24848 // For FCMP_OEQ, we can emit
24849 // two branches instead of an explicit AND instruction with a
24850 // separate test. However, we only do this if this block doesn't
24851 // have a fall-through edge, because this requires an explicit
24852 // jmp when the condition is false.
24853 if (Op.getNode()->hasOneUse()) {
24854 SDNode *User = *Op.getNode()->use_begin();
24855 // Look for an unconditional branch following this conditional branch.
24856 // We need this because we need to reverse the successors in order
24857 // to implement FCMP_OEQ.
24858 if (User->getOpcode() == ISD::BR) {
24859 SDValue FalseBB = User->getOperand(1);
24860 SDNode *NewBR =
24861 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
24862 assert(NewBR == User);
24863 (void)NewBR;
24864 Dest = FalseBB;
24865
24866 SDValue Cmp =
24867 DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
24868 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
24869 Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest,
24870 CCVal, Cmp, Op->getFlags());
24871 CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
24872 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24873 Cmp, Op->getFlags());
24874 }
24875 }
24876 } else if (CC == ISD::SETUNE) {
24877 // For FCMP_UNE, we can emit
24878 // two branches instead of an explicit OR instruction with a
24879 // separate test.
24880 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
24881 SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
24882 Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24883 Cmp, Op->getFlags());
24884 CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
24885 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24886 Cmp, Op->getFlags());
24887 } else {
24888 X86::CondCode X86Cond =
24889 TranslateX86CC(CC, dl, /*IsFP*/ true, LHS, RHS, DAG);
24890 SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
24891 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24892 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24893 Cmp, Op->getFlags());
24894 }
24895 }
24896
24897 if (ISD::isOverflowIntrOpRes(Cond)) {
24898 SDValue Value, Overflow;
24899 X86::CondCode X86Cond;
24900 std::tie(Value, Overflow) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
24901
24902 SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
24903 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24904 Overflow, Op->getFlags());
24905 }
24906
24907 // Look past the truncate if the high bits are known zero.
24908 if (isTruncWithZeroHighBitsInput(Cond, DAG))
24909 Cond = Cond.getOperand(0);
24910
24911 EVT CondVT = Cond.getValueType();
24912
24913 // Add an AND with 1 if we don't already have one.
24914 if (!(Cond.getOpcode() == ISD::AND && isOneConstant(Cond.getOperand(1))))
24915 Cond =
24916 DAG.getNode(ISD::AND, dl, CondVT, Cond, DAG.getConstant(1, dl, CondVT));
24917
24918 SDValue LHS = Cond;
24919 SDValue RHS = DAG.getConstant(0, dl, CondVT);
24920
24921 SDValue CCVal;
24922 SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal);
24923 return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, EFLAGS,
24924 Op->getFlags());
24925}
24926
24927// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
24928// Calls to _alloca are needed to probe the stack when allocating more than 4k
24929// bytes in one go. Touching the stack at 4K increments is necessary to ensure
24930// that the guard pages used by the OS virtual memory manager are allocated in
24931// correct sequence.
24932SDValue
24933X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
24934 SelectionDAG &DAG) const {
24935 MachineFunction &MF = DAG.getMachineFunction();
24936 bool SplitStack = MF.shouldSplitStack();
24937 bool EmitStackProbeCall = hasStackProbeSymbol(MF);
24938 bool Lower = (Subtarget.isOSWindows() && !Subtarget.isTargetMachO()) ||
24939 SplitStack || EmitStackProbeCall;
24940 SDLoc dl(Op);
24941
24942 // Get the inputs.
24943 SDNode *Node = Op.getNode();
24944 SDValue Chain = Op.getOperand(0);
24945 SDValue Size = Op.getOperand(1);
24946 MaybeAlign Alignment(Op.getConstantOperandVal(2));
24947 EVT VT = Node->getValueType(0);
24948
24949 // Chain the dynamic stack allocation so that it doesn't modify the stack
24950 // pointer when other instructions are using the stack.
24951 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
24952
24953 bool Is64Bit = Subtarget.is64Bit();
24954 MVT SPTy = getPointerTy(DAG.getDataLayout());
24955
24956 SDValue Result;
24957 if (!Lower) {
24958 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24959 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
24960 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
24961 " not tell us which reg is the stack pointer!");
24962
24963 const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
24964 const Align StackAlign = TFI.getStackAlign();
24965 if (hasInlineStackProbe(MF)) {
24966 MachineRegisterInfo &MRI = MF.getRegInfo();
24967
24968 const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
24969 Register Vreg = MRI.createVirtualRegister(AddrRegClass);
24970 Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
24971 Result = DAG.getNode(X86ISD::PROBED_ALLOCA, dl, SPTy, Chain,
24972 DAG.getRegister(Vreg, SPTy));
24973 } else {
24974 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
24975 Chain = SP.getValue(1);
24976 Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
24977 }
24978 if (Alignment && *Alignment > StackAlign)
24979 Result = DAG.getNode(
24980 ISD::AND, dl, VT, Result,
24981 DAG.getSignedConstant(~(Alignment->value() - 1ULL), dl, VT));
24982 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Result); // Output chain
24983 } else if (SplitStack) {
24984 MachineRegisterInfo &MRI = MF.getRegInfo();
24985
24986 if (Is64Bit) {
24987 // The 64 bit implementation of segmented stacks needs to clobber both r10
24988 // r11. This makes it impossible to use it along with nested parameters.
24989 const Function &F = MF.getFunction();
24990 for (const auto &A : F.args()) {
24991 if (A.hasNestAttr())
24992 report_fatal_error("Cannot use segmented stacks with functions that "
24993 "have nested arguments.");
24994 }
24995 }
24996
24997 const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
24998 Register Vreg = MRI.createVirtualRegister(AddrRegClass);
24999 Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
25000 Result = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
25001 DAG.getRegister(Vreg, SPTy));
25002 } else {
25003 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
25004 Chain = DAG.getNode(X86ISD::DYN_ALLOCA, dl, NodeTys, Chain, Size);
25005 MF.getInfo<X86MachineFunctionInfo>()->setHasDynAlloca(true);
25006
25007 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
25008 Register SPReg = RegInfo->getStackRegister();
25009 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
25010 Chain = SP.getValue(1);
25011
25012 if (Alignment) {
25013 SP = DAG.getNode(
25014 ISD::AND, dl, VT, SP.getValue(0),
25015 DAG.getSignedConstant(~(Alignment->value() - 1ULL), dl, VT));
25016 Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
25017 }
25018
25019 Result = SP;
25020 }
25021
25022 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
25023
25024 SDValue Ops[2] = {Result, Chain};
25025 return DAG.getMergeValues(Ops, dl);
25026}
25027
25028SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
25029 MachineFunction &MF = DAG.getMachineFunction();
25030 auto PtrVT = getPointerTy(MF.getDataLayout());
25031 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
25032
25033 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
25034 SDLoc DL(Op);
25035
25036 if (!Subtarget.is64Bit() ||
25037 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv())) {
25038 // vastart just stores the address of the VarArgsFrameIndex slot into the
25039 // memory location argument.
25040 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
25041 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
25042 MachinePointerInfo(SV));
25043 }
25044
25045 // __va_list_tag:
25046 // gp_offset (0 - 6 * 8)
25047 // fp_offset (48 - 48 + 8 * 16)
25048 // overflow_arg_area (point to parameters coming in memory).
25049 // reg_save_area
25050 SmallVector<SDValue, 8> MemOps;
25051 SDValue FIN = Op.getOperand(1);
25052 // Store gp_offset
25053 SDValue Store = DAG.getStore(
25054 Op.getOperand(0), DL,
25055 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN,
25056 MachinePointerInfo(SV));
25057 MemOps.push_back(Store);
25058
25059 // Store fp_offset
25060 FIN = DAG.getMemBasePlusOffset(FIN, TypeSize::getFixed(4), DL);
25061 Store = DAG.getStore(
25062 Op.getOperand(0), DL,
25063 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN,
25064 MachinePointerInfo(SV, 4));
25065 MemOps.push_back(Store);
25066
25067 // Store ptr to overflow_arg_area
25068 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL));
25069 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
25070 Store =
25071 DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, MachinePointerInfo(SV, 8));
25072 MemOps.push_back(Store);
25073
25074 // Store ptr to reg_save_area.
25075 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(
25076 Subtarget.isTarget64BitLP64() ? 8 : 4, DL));
25077 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
25078 Store = DAG.getStore(
25079 Op.getOperand(0), DL, RSFIN, FIN,
25080 MachinePointerInfo(SV, Subtarget.isTarget64BitLP64() ? 16 : 12));
25081 MemOps.push_back(Store);
25082 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
25083}
25084
25085SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
25086 assert(Subtarget.is64Bit() &&
25087 "LowerVAARG only handles 64-bit va_arg!");
25088 assert(Op.getNumOperands() == 4);
25089
25090 MachineFunction &MF = DAG.getMachineFunction();
25091 if (Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()))
25092 // The Win64 ABI uses char* instead of a structure.
25093 return DAG.expandVAArg(Op.getNode());
25094
25095 SDValue Chain = Op.getOperand(0);
25096 SDValue SrcPtr = Op.getOperand(1);
25097 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
25098 unsigned Align = Op.getConstantOperandVal(3);
25099 SDLoc dl(Op);
25100
25101 EVT ArgVT = Op.getNode()->getValueType(0);
25102 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
25103 uint32_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
25104 uint8_t ArgMode;
25105
25106 // Decide which area this value should be read from.
25107 // TODO: Implement the AMD64 ABI in its entirety. This simple
25108 // selection mechanism works only for the basic types.
25109 assert(ArgVT != MVT::f80 && "va_arg for f80 not yet implemented");
25110 if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) {
25111 ArgMode = 2; // Argument passed in XMM register. Use fp_offset.
25112 } else {
25113 assert(ArgVT.isInteger() && ArgSize <= 32 /*bytes*/ &&
25114 "Unhandled argument type in LowerVAARG");
25115 ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset.
25116 }
25117
25118 if (ArgMode == 2) {
25119 // Make sure using fp_offset makes sense.
25120 assert(!Subtarget.useSoftFloat() &&
25121 !(MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) &&
25122 Subtarget.hasSSE1());
25123 }
25124
25125 // Insert VAARG node into the DAG
25126 // VAARG returns two values: Variable Argument Address, Chain
25127 SDValue InstOps[] = {Chain, SrcPtr,
25128 DAG.getTargetConstant(ArgSize, dl, MVT::i32),
25129 DAG.getTargetConstant(ArgMode, dl, MVT::i8),
25130 DAG.getTargetConstant(Align, dl, MVT::i32)};
25131 SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);
25132 SDValue VAARG = DAG.getMemIntrinsicNode(
25133 Subtarget.isTarget64BitLP64() ? X86ISD::VAARG_64 : X86ISD::VAARG_X32, dl,
25134 VTs, InstOps, MVT::i64, MachinePointerInfo(SV),
25135 /*Alignment=*/std::nullopt,
25136 MachineMemOperand::MOLoad | MachineMemOperand::MOStore);
25137 Chain = VAARG.getValue(1);
25138
25139 // Load the next argument and return it
25140 return DAG.getLoad(ArgVT, dl, Chain, VAARG, MachinePointerInfo());
25141}
25142
25143static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
25144 SelectionDAG &DAG) {
25145 // X86-64 va_list is a struct { i32, i32, i8*, i8* }, except on Windows,
25146 // where a va_list is still an i8*.
25147 assert(Subtarget.is64Bit() && "This code only handles 64-bit va_copy!");
25148 if (Subtarget.isCallingConvWin64(
25149 DAG.getMachineFunction().getFunction().getCallingConv()))
25150 // Probably a Win64 va_copy.
25151 return DAG.expandVACopy(Op.getNode());
25152
25153 SDValue Chain = Op.getOperand(0);
25154 SDValue DstPtr = Op.getOperand(1);
25155 SDValue SrcPtr = Op.getOperand(2);
25156 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
25157 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
25158 SDLoc DL(Op);
25159
25160 return DAG.getMemcpy(
25161 Chain, DL, DstPtr, SrcPtr,
25162 DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL),
25163 Align(Subtarget.isTarget64BitLP64() ? 8 : 4), /*isVolatile*/ false, false,
25164 /*CI=*/nullptr, std::nullopt, MachinePointerInfo(DstSV),
25165 MachinePointerInfo(SrcSV));
25166}
25167
25168// Helper to get immediate/variable SSE shift opcode from other shift opcodes.
25169static unsigned getTargetVShiftUniformOpcode(unsigned Opc, bool IsVariable) {
25170 switch (Opc) {
25171 case ISD::SHL:
25172 case X86ISD::VSHL:
25173 case X86ISD::VSHLI:
25174 return IsVariable ? X86ISD::VSHL : X86ISD::VSHLI;
25175 case ISD::SRL:
25176 case X86ISD::VSRL:
25177 case X86ISD::VSRLI:
25178 return IsVariable ? X86ISD::VSRL : X86ISD::VSRLI;
25179 case ISD::SRA:
25180 case X86ISD::VSRA:
25181 case X86ISD::VSRAI:
25182 return IsVariable ? X86ISD::VSRA : X86ISD::VSRAI;
25183 }
25184 llvm_unreachable("Unknown target vector shift node");
25185}
25186
25187/// Handle vector element shifts where the shift amount is a constant.
25188/// Takes immediate version of shift as input.
25189static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
25190 SDValue SrcOp, uint64_t ShiftAmt,
25191 SelectionDAG &DAG) {
25192 MVT ElementType = VT.getVectorElementType();
25193
25194 // Bitcast the source vector to the output type, this is mainly necessary for
25195 // vXi8/vXi64 shifts.
25196 if (VT != SrcOp.getSimpleValueType())
25197 SrcOp = DAG.getBitcast(VT, SrcOp);
25198
25199 // Fold this packed shift into its first operand if ShiftAmt is 0.
25200 if (ShiftAmt == 0)
25201 return SrcOp;
25202
25203 // Check for ShiftAmt >= element width
25204 if (ShiftAmt >= ElementType.getSizeInBits()) {
25205 if (Opc == X86ISD::VSRAI)
25206 ShiftAmt = ElementType.getSizeInBits() - 1;
25207 else
25208 return DAG.getConstant(0, dl, VT);
25209 }
25210
25211 assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
25212 && "Unknown target vector shift-by-constant node");
25213
25214 // Fold this packed vector shift into a build vector if SrcOp is a
25215 // vector of Constants or UNDEFs.
25216 if (ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
25217 unsigned ShiftOpc;
25218 switch (Opc) {
25219 default: llvm_unreachable("Unknown opcode!");
25220 case X86ISD::VSHLI:
25221 ShiftOpc = ISD::SHL;
25222 break;
25223 case X86ISD::VSRLI:
25224 ShiftOpc = ISD::SRL;
25225 break;
25226 case X86ISD::VSRAI:
25227 ShiftOpc = ISD::SRA;
25228 break;
25229 }
25230
25231 SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT);
25232 if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt}))
25233 return C;
25234 }
25235
25236 return DAG.getNode(Opc, dl, VT, SrcOp,
25237 DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));
25238}
25239
25240/// Handle vector element shifts by a splat shift amount
25241static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
25242 SDValue SrcOp, SDValue ShAmt, int ShAmtIdx,
25243 const X86Subtarget &Subtarget,
25244 SelectionDAG &DAG) {
25245 MVT AmtVT = ShAmt.getSimpleValueType();
25246 assert(AmtVT.isVector() && "Vector shift type mismatch");
25247 assert(0 <= ShAmtIdx && ShAmtIdx < (int)AmtVT.getVectorNumElements() &&
25248 "Illegal vector splat index");
25249
25250 // Move the splat element to the bottom element.
25251 if (ShAmtIdx != 0) {
25252 SmallVector<int> Mask(AmtVT.getVectorNumElements(), -1);
25253 Mask[0] = ShAmtIdx;
25254 ShAmt = DAG.getVectorShuffle(AmtVT, dl, ShAmt, DAG.getUNDEF(AmtVT), Mask);
25255 }
25256
25257 // Peek through any zext node if we can get back to a 128-bit source.
25258 if (AmtVT.getScalarSizeInBits() == 64 &&
25259 (ShAmt.getOpcode() == ISD::ZERO_EXTEND ||
25260 ShAmt.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) &&
25261 ShAmt.getOperand(0).getValueType().isSimple() &&
25262 ShAmt.getOperand(0).getValueType().is128BitVector()) {
25263 ShAmt = ShAmt.getOperand(0);
25264 AmtVT = ShAmt.getSimpleValueType();
25265 }
25266
25267 // See if we can mask off the upper elements using the existing source node.
25268 // The shift uses the entire lower 64-bits of the amount vector, so no need to
25269 // do this for vXi64 types.
25270 bool IsMasked = false;
25271 if (AmtVT.getScalarSizeInBits() < 64) {
25272 if (ShAmt.getOpcode() == ISD::BUILD_VECTOR ||
25273 ShAmt.getOpcode() == ISD::SCALAR_TO_VECTOR) {
25274 // If the shift amount has come from a scalar, then zero-extend the scalar
25275 // before moving to the vector.
25276 ShAmt = DAG.getZExtOrTrunc(ShAmt.getOperand(0), dl, MVT::i32);
25277 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
25278 ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt);
25279 AmtVT = MVT::v4i32;
25280 IsMasked = true;
25281 } else if (ShAmt.getOpcode() == ISD::AND) {
25282 // See if the shift amount is already masked (e.g. for rotation modulo),
25283 // then we can zero-extend it by setting all the other mask elements to
25284 // zero.
25285 SmallVector<SDValue> MaskElts(
25286 AmtVT.getVectorNumElements(),
25287 DAG.getConstant(0, dl, AmtVT.getScalarType()));
25288 MaskElts[0] = DAG.getAllOnesConstant(dl, AmtVT.getScalarType());
25289 SDValue Mask = DAG.getBuildVector(AmtVT, dl, MaskElts);
25290 if ((Mask = DAG.FoldConstantArithmetic(ISD::AND, dl, AmtVT,
25291 {ShAmt.getOperand(1), Mask}))) {
25292 ShAmt = DAG.getNode(ISD::AND, dl, AmtVT, ShAmt.getOperand(0), Mask);
25293 IsMasked = true;
25294 }
25295 }
25296 }
25297
25298 // Extract if the shift amount vector is larger than 128-bits.
25299 if (AmtVT.getSizeInBits() > 128) {
25300 ShAmt = extract128BitVector(ShAmt, 0, DAG, dl);
25301 AmtVT = ShAmt.getSimpleValueType();
25302 }
25303
25304 // Zero-extend bottom element to v2i64 vector type, either by extension or
25305 // shuffle masking.
25306 if (!IsMasked && AmtVT.getScalarSizeInBits() < 64) {
25307 if (AmtVT == MVT::v4i32 && (ShAmt.getOpcode() == X86ISD::VBROADCAST ||
25308 ShAmt.getOpcode() == X86ISD::VBROADCAST_LOAD)) {
25309 ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt);
25310 } else if (Subtarget.hasSSE41()) {
25311 ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
25312 MVT::v2i64, ShAmt);
25313 } else {
25314 SDValue ByteShift = DAG.getTargetConstant(
25315 (128 - AmtVT.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8);
25316 ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);
25317 ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
25318 ByteShift);
25319 ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
25320 ByteShift);
25321 }
25322 }
25323
25324 // Change opcode to non-immediate version.
25325 Opc = getTargetVShiftUniformOpcode(Opc, true);
25326
25327 // The return type has to be a 128-bit type with the same element
25328 // type as the input type.
25329 MVT EltVT = VT.getVectorElementType();
25330 MVT ShVT = MVT::getVectorVT(EltVT, 128 / EltVT.getSizeInBits());
25331
25332 ShAmt = DAG.getBitcast(ShVT, ShAmt);
25333 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
25334}
25335
25336/// Return Mask with the necessary casting or extending
25337/// for \p Mask according to \p MaskVT when lowering masking intrinsics
25338static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
25339 const X86Subtarget &Subtarget, SelectionDAG &DAG,
25340 const SDLoc &dl) {
25341
25342 if (isAllOnesConstant(Mask))
25343 return DAG.getConstant(1, dl, MaskVT);
25344 if (X86::isZeroNode(Mask))
25345 return DAG.getConstant(0, dl, MaskVT);
25346
25347 assert(MaskVT.bitsLE(Mask.getSimpleValueType()) && "Unexpected mask size!");
25348
25349 if (Mask.getSimpleValueType() == MVT::i64 && Subtarget.is32Bit()) {
25350 assert(MaskVT == MVT::v64i1 && "Expected v64i1 mask!");
25351 assert(Subtarget.hasBWI() && "Expected AVX512BW target!");
25352 // In case 32bit mode, bitcast i64 is illegal, extend/split it.
25353 SDValue Lo, Hi;
25354 std::tie(Lo, Hi) = DAG.SplitScalar(Mask, dl, MVT::i32, MVT::i32);
25355 Lo = DAG.getBitcast(MVT::v32i1, Lo);
25356 Hi = DAG.getBitcast(MVT::v32i1, Hi);
25357 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
25358 } else {
25359 MVT BitcastVT = MVT::getVectorVT(MVT::i1,
25360 Mask.getSimpleValueType().getSizeInBits());
25361 // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
25362 // are extracted by EXTRACT_SUBVECTOR.
25363 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
25364 DAG.getBitcast(BitcastVT, Mask),
25365 DAG.getIntPtrConstant(0, dl));
25366 }
25367}
25368
25369/// Return (and \p Op, \p Mask) for compare instructions or
25370/// (vselect \p Mask, \p Op, \p PreservedSrc) for others along with the
25371/// necessary casting or extending for \p Mask when lowering masking intrinsics
25372static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
25373 SDValue PreservedSrc,
25374 const X86Subtarget &Subtarget,
25375 SelectionDAG &DAG) {
25376 MVT VT = Op.getSimpleValueType();
25377 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
25378 unsigned OpcodeSelect = ISD::VSELECT;
25379 SDLoc dl(Op);
25380
25381 if (isAllOnesConstant(Mask))
25382 return Op;
25383
25384 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
25385
25386 if (PreservedSrc.isUndef())
25387 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
25388 return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc);
25389}
25390
25391/// Creates an SDNode for a predicated scalar operation.
25392/// \returns (X86vselect \p Mask, \p Op, \p PreservedSrc).
25393/// The mask is coming as MVT::i8 and it should be transformed
25394/// to MVT::v1i1 while lowering masking intrinsics.
25395/// The main difference between ScalarMaskingNode and VectorMaskingNode is using
25396/// "X86select" instead of "vselect". We just can't create the "vselect" node
25397/// for a scalar instruction.
25398static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
25399 SDValue PreservedSrc,
25400 const X86Subtarget &Subtarget,
25401 SelectionDAG &DAG) {
25402
25403 if (auto *MaskConst = dyn_cast<ConstantSDNode>(Mask))
25404 if (MaskConst->getZExtValue() & 0x1)
25405 return Op;
25406
25407 MVT VT = Op.getSimpleValueType();
25408 SDLoc dl(Op);
25409
25410 assert(Mask.getValueType() == MVT::i8 && "Unexpect type");
25411 SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1,
25412 DAG.getBitcast(MVT::v8i1, Mask),
25413 DAG.getIntPtrConstant(0, dl));
25414 if (Op.getOpcode() == X86ISD::FSETCCM ||
25415 Op.getOpcode() == X86ISD::FSETCCM_SAE ||
25416 Op.getOpcode() == X86ISD::VFPCLASSS)
25417 return DAG.getNode(ISD::AND, dl, VT, Op, IMask);
25418
25419 if (PreservedSrc.isUndef())
25420 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
25421 return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc);
25422}
25423
25424static int getSEHRegistrationNodeSize(const Function *Fn) {
25425 if (!Fn->hasPersonalityFn())
25426 report_fatal_error(
25427 "querying registration node size for function without personality");
25428 // The RegNodeSize is 6 32-bit words for SEH and 4 for C++ EH. See
25429 // WinEHStatePass for the full struct definition.
25430 switch (classifyEHPersonality(Fn->getPersonalityFn())) {
25431 case EHPersonality::MSVC_X86SEH: return 24;
25432 case EHPersonality::MSVC_CXX: return 16;
25433 default: break;
25434 }
25435 report_fatal_error(
25436 "can only recover FP for 32-bit MSVC EH personality functions");
25437}
25438
25439/// When the MSVC runtime transfers control to us, either to an outlined
25440/// function or when returning to a parent frame after catching an exception, we
25441/// recover the parent frame pointer by doing arithmetic on the incoming EBP.
25442/// Here's the math:
25443/// RegNodeBase = EntryEBP - RegNodeSize
25444/// ParentFP = RegNodeBase - ParentFrameOffset
25445/// Subtracting RegNodeSize takes us to the offset of the registration node, and
25446/// subtracting the offset (negative on x86) takes us back to the parent FP.
25447static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
25448 SDValue EntryEBP) {
25449 MachineFunction &MF = DAG.getMachineFunction();
25450 SDLoc dl;
25451
25452 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25453 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
25454
25455 // It's possible that the parent function no longer has a personality function
25456 // if the exceptional code was optimized away, in which case we just return
25457 // the incoming EBP.
25458 if (!Fn->hasPersonalityFn())
25459 return EntryEBP;
25460
25461 // Get an MCSymbol that will ultimately resolve to the frame offset of the EH
25462 // registration, or the .set_setframe offset.
25463 MCSymbol *OffsetSym = MF.getContext().getOrCreateParentFrameOffsetSymbol(
25464 GlobalValue::dropLLVMManglingEscape(Fn->getName()));
25465 SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
25466 SDValue ParentFrameOffset =
25467 DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal);
25468
25469 // Return EntryEBP + ParentFrameOffset for x64. This adjusts from RSP after
25470 // prologue to RBP in the parent function.
25471 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
25472 if (Subtarget.is64Bit())
25473 return DAG.getNode(ISD::ADD, dl, PtrVT, EntryEBP, ParentFrameOffset);
25474
25475 int RegNodeSize = getSEHRegistrationNodeSize(Fn);
25476 // RegNodeBase = EntryEBP - RegNodeSize
25477 // ParentFP = RegNodeBase - ParentFrameOffset
25478 SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP,
25479 DAG.getConstant(RegNodeSize, dl, PtrVT));
25480 return DAG.getNode(ISD::SUB, dl, PtrVT, RegNodeBase, ParentFrameOffset);
25481}
25482
25483SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
25484 SelectionDAG &DAG) const {
25485 // Helper to detect if the operand is CUR_DIRECTION rounding mode.
25486 auto isRoundModeCurDirection = [](SDValue Rnd) {
25487 if (auto *C = dyn_cast<ConstantSDNode>(Rnd))
25488 return C->getAPIntValue() == X86::STATIC_ROUNDING::CUR_DIRECTION;
25489
25490 return false;
25491 };
25492 auto isRoundModeSAE = [](SDValue Rnd) {
25493 if (auto *C = dyn_cast<ConstantSDNode>(Rnd)) {
25494 unsigned RC = C->getZExtValue();
25495 if (RC & X86::STATIC_ROUNDING::NO_EXC) {
25496 // Clear the NO_EXC bit and check remaining bits.
25497 RC ^= X86::STATIC_ROUNDING::NO_EXC;
25498 // As a convenience we allow no other bits or explicitly
25499 // current direction.
25500 return RC == 0 || RC == X86::STATIC_ROUNDING::CUR_DIRECTION;
25501 }
25502 }
25503
25504 return false;
25505 };
25506 auto isRoundModeSAEToX = [](SDValue Rnd, unsigned &RC) {
25507 if (auto *C = dyn_cast<ConstantSDNode>(Rnd)) {
25508 RC = C->getZExtValue();
25509 if (RC & X86::STATIC_ROUNDING::NO_EXC) {
25510 // Clear the NO_EXC bit and check remaining bits.
25511 RC ^= X86::STATIC_ROUNDING::NO_EXC;
25512 return RC == X86::STATIC_ROUNDING::TO_NEAREST_INT ||
25513 RC == X86::STATIC_ROUNDING::TO_NEG_INF ||
25514 RC == X86::STATIC_ROUNDING::TO_POS_INF ||
25515 RC == X86::STATIC_ROUNDING::TO_ZERO;
25516 }
25517 }
25518
25519 return false;
25520 };
25521
25522 SDLoc dl(Op);
25523 unsigned IntNo = Op.getConstantOperandVal(0);
25524 MVT VT = Op.getSimpleValueType();
25525 const IntrinsicData* IntrData = getIntrinsicWithoutChain(IntNo);
25526
25527 // Propagate flags from original node to transformed node(s).
25528 SelectionDAG::FlagInserter FlagsInserter(DAG, Op->getFlags());
25529
25530 if (IntrData) {
25531 switch(IntrData->Type) {
25532 case INTR_TYPE_1OP: {
25533 // We specify 2 possible opcodes for intrinsics with rounding modes.
25534 // First, we check if the intrinsic may have non-default rounding mode,
25535 // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25536 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25537 if (IntrWithRoundingModeOpcode != 0) {
25538 SDValue Rnd = Op.getOperand(2);
25539 unsigned RC = 0;
25540 if (isRoundModeSAEToX(Rnd, RC))
25541 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25542 Op.getOperand(1),
25543 DAG.getTargetConstant(RC, dl, MVT::i32));
25544 if (!isRoundModeCurDirection(Rnd))
25545 return SDValue();
25546 }
25547 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25548 Op.getOperand(1));
25549 }
25550 case INTR_TYPE_1OP_SAE: {
25551 SDValue Sae = Op.getOperand(2);
25552
25553 unsigned Opc;
25554 if (isRoundModeCurDirection(Sae))
25555 Opc = IntrData->Opc0;
25556 else if (isRoundModeSAE(Sae))
25557 Opc = IntrData->Opc1;
25558 else
25559 return SDValue();
25560
25561 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1));
25562 }
25563 case INTR_TYPE_2OP: {
25564 SDValue Src2 = Op.getOperand(2);
25565
25566 // We specify 2 possible opcodes for intrinsics with rounding modes.
25567 // First, we check if the intrinsic may have non-default rounding mode,
25568 // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25569 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25570 if (IntrWithRoundingModeOpcode != 0) {
25571 SDValue Rnd = Op.getOperand(3);
25572 unsigned RC = 0;
25573 if (isRoundModeSAEToX(Rnd, RC))
25574 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25575 Op.getOperand(1), Src2,
25576 DAG.getTargetConstant(RC, dl, MVT::i32));
25577 if (!isRoundModeCurDirection(Rnd))
25578 return SDValue();
25579 }
25580
25581 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25582 Op.getOperand(1), Src2);
25583 }
25584 case INTR_TYPE_2OP_SAE: {
25585 SDValue Sae = Op.getOperand(3);
25586
25587 unsigned Opc;
25588 if (isRoundModeCurDirection(Sae))
25589 Opc = IntrData->Opc0;
25590 else if (isRoundModeSAE(Sae))
25591 Opc = IntrData->Opc1;
25592 else
25593 return SDValue();
25594
25595 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1),
25596 Op.getOperand(2));
25597 }
25598 case INTR_TYPE_3OP:
25599 case INTR_TYPE_3OP_IMM8: {
25600 SDValue Src1 = Op.getOperand(1);
25601 SDValue Src2 = Op.getOperand(2);
25602 SDValue Src3 = Op.getOperand(3);
25603
25604 if (IntrData->Type == INTR_TYPE_3OP_IMM8 &&
25605 Src3.getValueType() != MVT::i8) {
25606 Src3 = DAG.getTargetConstant(Src3->getAsZExtVal() & 0xff, dl, MVT::i8);
25607 }
25608
25609 // We specify 2 possible opcodes for intrinsics with rounding modes.
25610 // First, we check if the intrinsic may have non-default rounding mode,
25611 // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25612 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25613 if (IntrWithRoundingModeOpcode != 0) {
25614 SDValue Rnd = Op.getOperand(4);
25615 unsigned RC = 0;
25616 if (isRoundModeSAEToX(Rnd, RC))
25617 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25618 Src1, Src2, Src3,
25619 DAG.getTargetConstant(RC, dl, MVT::i32));
25620 if (!isRoundModeCurDirection(Rnd))
25621 return SDValue();
25622 }
25623
25624 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25625 {Src1, Src2, Src3});
25626 }
25627 case INTR_TYPE_4OP_IMM8: {
25628 assert(Op.getOperand(4)->getOpcode() == ISD::TargetConstant);
25629 SDValue Src4 = Op.getOperand(4);
25630 if (Src4.getValueType() != MVT::i8) {
25631 Src4 = DAG.getTargetConstant(Src4->getAsZExtVal() & 0xff, dl, MVT::i8);
25632 }
25633
25634 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25635 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3),
25636 Src4);
25637 }
25638 case INTR_TYPE_1OP_MASK: {
25639 SDValue Src = Op.getOperand(1);
25640 SDValue PassThru = Op.getOperand(2);
25641 SDValue Mask = Op.getOperand(3);
25642 // We add rounding mode to the Node when
25643 // - RC Opcode is specified and
25644 // - RC is not "current direction".
25645 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25646 if (IntrWithRoundingModeOpcode != 0) {
25647 SDValue Rnd = Op.getOperand(4);
25648 unsigned RC = 0;
25649 if (isRoundModeSAEToX(Rnd, RC))
25650 return getVectorMaskingNode(
25651 DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
25652 Src, DAG.getTargetConstant(RC, dl, MVT::i32)),
25653 Mask, PassThru, Subtarget, DAG);
25654 if (!isRoundModeCurDirection(Rnd))
25655 return SDValue();
25656 }
25657 return getVectorMaskingNode(
25658 DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru,
25659 Subtarget, DAG);
25660 }
25661 case INTR_TYPE_1OP_MASK_SAE: {
25662 SDValue Src = Op.getOperand(1);
25663 SDValue PassThru = Op.getOperand(2);
25664 SDValue Mask = Op.getOperand(3);
25665 SDValue Rnd = Op.getOperand(4);
25666
25667 unsigned Opc;
25668 if (isRoundModeCurDirection(Rnd))
25669 Opc = IntrData->Opc0;
25670 else if (isRoundModeSAE(Rnd))
25671 Opc = IntrData->Opc1;
25672 else
25673 return SDValue();
25674
25675 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src), Mask, PassThru,
25676 Subtarget, DAG);
25677 }
25678 case INTR_TYPE_SCALAR_MASK: {
25679 SDValue Src1 = Op.getOperand(1);
25680 SDValue Src2 = Op.getOperand(2);
25681 SDValue passThru = Op.getOperand(3);
25682 SDValue Mask = Op.getOperand(4);
25683 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
25684 // There are 2 kinds of intrinsics in this group:
25685 // (1) With suppress-all-exceptions (sae) or rounding mode- 6 operands
25686 // (2) With rounding mode and sae - 7 operands.
25687 bool HasRounding = IntrWithRoundingModeOpcode != 0;
25688 if (Op.getNumOperands() == (5U + HasRounding)) {
25689 if (HasRounding) {
25690 SDValue Rnd = Op.getOperand(5);
25691 unsigned RC = 0;
25692 if (isRoundModeSAEToX(Rnd, RC))
25693 return getScalarMaskingNode(
25694 DAG.getNode(IntrWithRoundingModeOpcode, dl, VT, Src1, Src2,
25695 DAG.getTargetConstant(RC, dl, MVT::i32)),
25696 Mask, passThru, Subtarget, DAG);
25697 if (!isRoundModeCurDirection(Rnd))
25698 return SDValue();
25699 }
25700 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,
25701 Src2),
25702 Mask, passThru, Subtarget, DAG);
25703 }
25704
25705 assert(Op.getNumOperands() == (6U + HasRounding) &&
25706 "Unexpected intrinsic form");
25707 SDValue RoundingMode = Op.getOperand(5);
25708 unsigned Opc = IntrData->Opc0;
25709 if (HasRounding) {
25710 SDValue Sae = Op.getOperand(6);
25711 if (isRoundModeSAE(Sae))
25712 Opc = IntrWithRoundingModeOpcode;
25713 else if (!isRoundModeCurDirection(Sae))
25714 return SDValue();
25715 }
25716 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1,
25717 Src2, RoundingMode),
25718 Mask, passThru, Subtarget, DAG);
25719 }
25720 case INTR_TYPE_SCALAR_MASK_RND: {
25721 SDValue Src1 = Op.getOperand(1);
25722 SDValue Src2 = Op.getOperand(2);
25723 SDValue passThru = Op.getOperand(3);
25724 SDValue Mask = Op.getOperand(4);
25725 SDValue Rnd = Op.getOperand(5);
25726
25727 SDValue NewOp;
25728 unsigned RC = 0;
25729 if (isRoundModeCurDirection(Rnd))
25730 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
25731 else if (isRoundModeSAEToX(Rnd, RC))
25732 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
25733 DAG.getTargetConstant(RC, dl, MVT::i32));
25734 else
25735 return SDValue();
25736
25737 return getScalarMaskingNode(NewOp, Mask, passThru, Subtarget, DAG);
25738 }
25739 case INTR_TYPE_SCALAR_MASK_SAE: {
25740 SDValue Src1 = Op.getOperand(1);
25741 SDValue Src2 = Op.getOperand(2);
25742 SDValue passThru = Op.getOperand(3);
25743 SDValue Mask = Op.getOperand(4);
25744 SDValue Sae = Op.getOperand(5);
25745 unsigned Opc;
25746 if (isRoundModeCurDirection(Sae))
25747 Opc = IntrData->Opc0;
25748 else if (isRoundModeSAE(Sae))
25749 Opc = IntrData->Opc1;
25750 else
25751 return SDValue();
25752
25753 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
25754 Mask, passThru, Subtarget, DAG);
25755 }
25756 case INTR_TYPE_2OP_MASK: {
25757 SDValue Src1 = Op.getOperand(1);
25758 SDValue Src2 = Op.getOperand(2);
25759 SDValue PassThru = Op.getOperand(3);
25760 SDValue Mask = Op.getOperand(4);
25761 SDValue NewOp;
25762 if (IntrData->Opc1 != 0) {
25763 SDValue Rnd = Op.getOperand(5);
25764 unsigned RC = 0;
25765 if (isRoundModeSAEToX(Rnd, RC))
25766 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
25767 DAG.getTargetConstant(RC, dl, MVT::i32));
25768 else if (!isRoundModeCurDirection(Rnd))
25769 return SDValue();
25770 }
25771 if (!NewOp)
25772 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
25773 return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
25774 }
25775 case INTR_TYPE_2OP_MASK_SAE: {
25776 SDValue Src1 = Op.getOperand(1);
25777 SDValue Src2 = Op.getOperand(2);
25778 SDValue PassThru = Op.getOperand(3);
25779 SDValue Mask = Op.getOperand(4);
25780
25781 unsigned Opc = IntrData->Opc0;
25782 if (IntrData->Opc1 != 0) {
25783 SDValue Sae = Op.getOperand(5);
25784 if (isRoundModeSAE(Sae))
25785 Opc = IntrData->Opc1;
25786 else if (!isRoundModeCurDirection(Sae))
25787 return SDValue();
25788 }
25789
25790 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
25791 Mask, PassThru, Subtarget, DAG);
25792 }
25793 case INTR_TYPE_3OP_SCALAR_MASK_SAE: {
25794 SDValue Src1 = Op.getOperand(1);
25795 SDValue Src2 = Op.getOperand(2);
25796 SDValue Src3 = Op.getOperand(3);
25797 SDValue PassThru = Op.getOperand(4);
25798 SDValue Mask = Op.getOperand(5);
25799 SDValue Sae = Op.getOperand(6);
25800 unsigned Opc;
25801 if (isRoundModeCurDirection(Sae))
25802 Opc = IntrData->Opc0;
25803 else if (isRoundModeSAE(Sae))
25804 Opc = IntrData->Opc1;
25805 else
25806 return SDValue();
25807
25808 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
25809 Mask, PassThru, Subtarget, DAG);
25810 }
25811 case INTR_TYPE_3OP_MASK_SAE: {
25812 SDValue Src1 = Op.getOperand(1);
25813 SDValue Src2 = Op.getOperand(2);
25814 SDValue Src3 = Op.getOperand(3);
25815 SDValue PassThru = Op.getOperand(4);
25816 SDValue Mask = Op.getOperand(5);
25817
25818 unsigned Opc = IntrData->Opc0;
25819 if (IntrData->Opc1 != 0) {
25820 SDValue Sae = Op.getOperand(6);
25821 if (isRoundModeSAE(Sae))
25822 Opc = IntrData->Opc1;
25823 else if (!isRoundModeCurDirection(Sae))
25824 return SDValue();
25825 }
25826 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
25827 Mask, PassThru, Subtarget, DAG);
25828 }
25829 case BLENDV: {
25830 SDValue Src1 = Op.getOperand(1);
25831 SDValue Src2 = Op.getOperand(2);
25832 SDValue Src3 = Op.getOperand(3);
25833
25834 EVT MaskVT = Src3.getValueType().changeVectorElementTypeToInteger();
25835 Src3 = DAG.getBitcast(MaskVT, Src3);
25836
25837 // Reverse the operands to match VSELECT order.
25838 return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1);
25839 }
25840 case VPERM_2OP : {
25841 SDValue Src1 = Op.getOperand(1);
25842 SDValue Src2 = Op.getOperand(2);
25843
25844 // Swap Src1 and Src2 in the node creation
25845 return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);
25846 }
25847 case CFMA_OP_MASKZ:
25848 case CFMA_OP_MASK: {
25849 SDValue Src1 = Op.getOperand(1);
25850 SDValue Src2 = Op.getOperand(2);
25851 SDValue Src3 = Op.getOperand(3);
25852 SDValue Mask = Op.getOperand(4);
25853 MVT VT = Op.getSimpleValueType();
25854
25855 SDValue PassThru = Src3;
25856 if (IntrData->Type == CFMA_OP_MASKZ)
25857 PassThru = getZeroVector(VT, Subtarget, DAG, dl);
25858
25859 // We add rounding mode to the Node when
25860 // - RC Opcode is specified and
25861 // - RC is not "current direction".
25862 SDValue NewOp;
25863 if (IntrData->Opc1 != 0) {
25864 SDValue Rnd = Op.getOperand(5);
25865 unsigned RC = 0;
25866 if (isRoundModeSAEToX(Rnd, RC))
25867 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, Src3,
25868 DAG.getTargetConstant(RC, dl, MVT::i32));
25869 else if (!isRoundModeCurDirection(Rnd))
25870 return SDValue();
25871 }
25872 if (!NewOp)
25873 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3);
25874 return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
25875 }
25876 case IFMA_OP:
25877 // NOTE: We need to swizzle the operands to pass the multiply operands
25878 // first.
25879 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
25880 Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
25881 case FPCLASSS: {
25882 SDValue Src1 = Op.getOperand(1);
25883 SDValue Imm = Op.getOperand(2);
25884 SDValue Mask = Op.getOperand(3);
25885 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
25886 SDValue FPclassMask = getScalarMaskingNode(FPclass, Mask, SDValue(),
25887 Subtarget, DAG);
25888 // Need to fill with zeros to ensure the bitcast will produce zeroes
25889 // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
25890 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
25891 DAG.getConstant(0, dl, MVT::v8i1),
25892 FPclassMask, DAG.getIntPtrConstant(0, dl));
25893 return DAG.getBitcast(MVT::i8, Ins);
25894 }
25895
25896 case CMP_MASK_CC: {
25897 MVT MaskVT = Op.getSimpleValueType();
25898 SDValue CC = Op.getOperand(3);
25899 SDValue Mask = Op.getOperand(4);
25900 // We specify 2 possible opcodes for intrinsics with rounding modes.
25901 // First, we check if the intrinsic may have non-default rounding mode,
25902 // (IntrData->Opc1 != 0), then we check the rounding mode operand.
25903 if (IntrData->Opc1 != 0) {
25904 SDValue Sae = Op.getOperand(5);
25905 if (isRoundModeSAE(Sae))
25906 return DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1),
25907 Op.getOperand(2), CC, Mask, Sae);
25908 if (!isRoundModeCurDirection(Sae))
25909 return SDValue();
25910 }
25911 //default rounding mode
25912 return DAG.getNode(IntrData->Opc0, dl, MaskVT,
25913 {Op.getOperand(1), Op.getOperand(2), CC, Mask});
25914 }
25915 case CMP_MASK_SCALAR_CC: {
25916 SDValue Src1 = Op.getOperand(1);
25917 SDValue Src2 = Op.getOperand(2);
25918 SDValue CC = Op.getOperand(3);
25919 SDValue Mask = Op.getOperand(4);
25920
25921 SDValue Cmp;
25922 if (IntrData->Opc1 != 0) {
25923 SDValue Sae = Op.getOperand(5);
25924 if (isRoundModeSAE(Sae))
25925 Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Sae);
25926 else if (!isRoundModeCurDirection(Sae))
25927 return SDValue();
25928 }
25929 //default rounding mode
25930 if (!Cmp.getNode())
25931 Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC);
25932
25933 SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
25934 Subtarget, DAG);
25935 // Need to fill with zeros to ensure the bitcast will produce zeroes
25936 // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
25937 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
25938 DAG.getConstant(0, dl, MVT::v8i1),
25939 CmpMask, DAG.getIntPtrConstant(0, dl));
25940 return DAG.getBitcast(MVT::i8, Ins);
25941 }
25942 case COMI: { // Comparison intrinsics
25943 ISD::CondCode CC = (ISD::CondCode)IntrData->Opc1;
25944 SDValue LHS = Op.getOperand(1);
25945 SDValue RHS = Op.getOperand(2);
25946 // Some conditions require the operands to be swapped.
25947 if (CC == ISD::SETLT || CC == ISD::SETLE)
25948 std::swap(LHS, RHS);
25949
25950 SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);
25951 SDValue SetCC;
25952 switch (CC) {
25953 case ISD::SETEQ: { // (ZF = 0 and PF = 0)
25954 SetCC = getSETCC(X86::COND_E, Comi, dl, DAG);
25955 SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG);
25956 SetCC = DAG.getNode(ISD::AND, dl, MVT::i8, SetCC, SetNP);
25957 break;
25958 }
25959 case ISD::SETNE: { // (ZF = 1 or PF = 1)
25960 SetCC = getSETCC(X86::COND_NE, Comi, dl, DAG);
25961 SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG);
25962 SetCC = DAG.getNode(ISD::OR, dl, MVT::i8, SetCC, SetP);
25963 break;
25964 }
25965 case ISD::SETGT: // (CF = 0 and ZF = 0)
25966 case ISD::SETLT: { // Condition opposite to GT. Operands swapped above.
25967 SetCC = getSETCC(X86::COND_A, Comi, dl, DAG);
25968 break;
25969 }
25970 case ISD::SETGE: // CF = 0
25971 case ISD::SETLE: // Condition opposite to GE. Operands swapped above.
25972 SetCC = getSETCC(X86::COND_AE, Comi, dl, DAG);
25973 break;
25974 default:
25975 llvm_unreachable("Unexpected illegal condition!");
25976 }
25977 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
25978 }
25979 case COMI_RM: { // Comparison intrinsics with Sae
25980 SDValue LHS = Op.getOperand(1);
25981 SDValue RHS = Op.getOperand(2);
25982 unsigned CondVal = Op.getConstantOperandVal(3);
25983 SDValue Sae = Op.getOperand(4);
25984
25985 SDValue FCmp;
25986 if (isRoundModeCurDirection(Sae))
25987 FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS,
25988 DAG.getTargetConstant(CondVal, dl, MVT::i8));
25989 else if (isRoundModeSAE(Sae))
25990 FCmp = DAG.getNode(X86ISD::FSETCCM_SAE, dl, MVT::v1i1, LHS, RHS,
25991 DAG.getTargetConstant(CondVal, dl, MVT::i8), Sae);
25992 else
25993 return SDValue();
25994 // Need to fill with zeros to ensure the bitcast will produce zeroes
25995 // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
25996 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
25997 DAG.getConstant(0, dl, MVT::v16i1),
25998 FCmp, DAG.getIntPtrConstant(0, dl));
25999 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32,
26000 DAG.getBitcast(MVT::i16, Ins));
26001 }
26002 case VSHIFT: {
26003 SDValue SrcOp = Op.getOperand(1);
26004 SDValue ShAmt = Op.getOperand(2);
26005 assert(ShAmt.getValueType() == MVT::i32 &&
26006 "Unexpected VSHIFT amount type");
26007
26008 // Catch shift-by-constant.
26009 if (auto *CShAmt = dyn_cast<ConstantSDNode>(ShAmt))
26010 return getTargetVShiftByConstNode(IntrData->Opc0, dl,
26011 Op.getSimpleValueType(), SrcOp,
26012 CShAmt->getZExtValue(), DAG);
26013
26014 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);
26015 return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(),
26016 SrcOp, ShAmt, 0, Subtarget, DAG);
26017 }
26018 case COMPRESS_EXPAND_IN_REG: {
26019 SDValue Mask = Op.getOperand(3);
26020 SDValue DataToCompress = Op.getOperand(1);
26021 SDValue PassThru = Op.getOperand(2);
26022 if (ISD::isBuildVectorAllOnes(Mask.getNode())) // return data as is
26023 return Op.getOperand(1);
26024
26025 // Avoid false dependency.
26026 if (PassThru.isUndef())
26027 PassThru = getZeroVector(VT, Subtarget, DAG, dl);
26028
26029 return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru,
26030 Mask);
26031 }
26032 case FIXUPIMM:
26033 case FIXUPIMM_MASKZ: {
26034 SDValue Src1 = Op.getOperand(1);
26035 SDValue Src2 = Op.getOperand(2);
26036 SDValue Src3 = Op.getOperand(3);
26037 SDValue Imm = Op.getOperand(4);
26038 SDValue Mask = Op.getOperand(5);
26039 SDValue Passthru = (IntrData->Type == FIXUPIMM)
26040 ? Src1
26041 : getZeroVector(VT, Subtarget, DAG, dl);
26042
26043 unsigned Opc = IntrData->Opc0;
26044 if (IntrData->Opc1 != 0) {
26045 SDValue Sae = Op.getOperand(6);
26046 if (isRoundModeSAE(Sae))
26047 Opc = IntrData->Opc1;
26048 else if (!isRoundModeCurDirection(Sae))
26049 return SDValue();
26050 }
26051
26052 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm);
26053
26054 if (Opc == X86ISD::VFIXUPIMM || Opc == X86ISD::VFIXUPIMM_SAE)
26055 return getVectorMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
26056
26057 return getScalarMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
26058 }
26059 case ROUNDP: {
26060 assert(IntrData->Opc0 == X86ISD::VRNDSCALE && "Unexpected opcode");
26061 // Clear the upper bits of the rounding immediate so that the legacy
26062 // intrinsic can't trigger the scaling behavior of VRNDSCALE.
26063 uint64_t Round = Op.getConstantOperandVal(2);
26064 SDValue RoundingMode = DAG.getTargetConstant(Round & 0xf, dl, MVT::i32);
26065 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26066 Op.getOperand(1), RoundingMode);
26067 }
26068 case ROUNDS: {
26069 assert(IntrData->Opc0 == X86ISD::VRNDSCALES && "Unexpected opcode");
26070 // Clear the upper bits of the rounding immediate so that the legacy
26071 // intrinsic can't trigger the scaling behavior of VRNDSCALE.
26072 uint64_t Round = Op.getConstantOperandVal(3);
26073 SDValue RoundingMode = DAG.getTargetConstant(Round & 0xf, dl, MVT::i32);
26074 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26075 Op.getOperand(1), Op.getOperand(2), RoundingMode);
26076 }
26077 case BEXTRI: {
26078 assert(IntrData->Opc0 == X86ISD::BEXTRI && "Unexpected opcode");
26079
26080 uint64_t Imm = Op.getConstantOperandVal(2);
26081 SDValue Control = DAG.getTargetConstant(Imm & 0xffff, dl,
26082 Op.getValueType());
26083 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
26084 Op.getOperand(1), Control);
26085 }
26086 // ADC/SBB
26087 case ADX: {
26088 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
26089 SDVTList VTs = DAG.getVTList(Op.getOperand(2).getValueType(), MVT::i32);
26090
26091 SDValue Res;
26092 // If the carry in is zero, then we should just use ADD/SUB instead of
26093 // ADC/SBB.
26094 if (isNullConstant(Op.getOperand(1))) {
26095 Res = DAG.getNode(IntrData->Opc1, dl, VTs, Op.getOperand(2),
26096 Op.getOperand(3));
26097 } else {
26098 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
26099 DAG.getAllOnesConstant(dl, MVT::i8));
26100 Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2),
26101 Op.getOperand(3), GenCF.getValue(1));
26102 }
26103 SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
26104 SDValue Results[] = { SetCC, Res };
26105 return DAG.getMergeValues(Results, dl);
26106 }
26107 case CVTPD2PS_MASK:
26108 case CVTPD2DQ_MASK:
26109 case CVTQQ2PS_MASK:
26110 case TRUNCATE_TO_REG: {
26111 SDValue Src = Op.getOperand(1);
26112 SDValue PassThru = Op.getOperand(2);
26113 SDValue Mask = Op.getOperand(3);
26114
26115 if (isAllOnesConstant(Mask))
26116 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
26117
26118 MVT SrcVT = Src.getSimpleValueType();
26119 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
26120 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26121 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(),
26122 {Src, PassThru, Mask});
26123 }
26124 case TRUNCATE2_TO_REG: {
26125 SDValue Src = Op.getOperand(1);
26126 SDValue Src2 = Op.getOperand(2);
26127 SDValue PassThru = Op.getOperand(3);
26128 SDValue Mask = Op.getOperand(4);
26129
26130 if (isAllOnesConstant(Mask))
26131 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), {Src, Src2});
26132
26133 MVT Src2VT = Src2.getSimpleValueType();
26134 MVT MaskVT = MVT::getVectorVT(MVT::i1, Src2VT.getVectorNumElements());
26135 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26136 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(),
26137 {Src, Src2, PassThru, Mask});
26138 }
26139 case CVTPS2PH_MASK: {
26140 SDValue Src = Op.getOperand(1);
26141 SDValue Rnd = Op.getOperand(2);
26142 SDValue PassThru = Op.getOperand(3);
26143 SDValue Mask = Op.getOperand(4);
26144
26145 unsigned RC = 0;
26146 unsigned Opc = IntrData->Opc0;
26147 bool SAE = Src.getValueType().is512BitVector() &&
26148 (isRoundModeSAEToX(Rnd, RC) || isRoundModeSAE(Rnd));
26149 if (SAE) {
26150 Opc = X86ISD::CVTPS2PH_SAE;
26151 Rnd = DAG.getTargetConstant(RC, dl, MVT::i32);
26152 }
26153
26154 if (isAllOnesConstant(Mask))
26155 return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd);
26156
26157 if (SAE)
26158 Opc = X86ISD::MCVTPS2PH_SAE;
26159 else
26160 Opc = IntrData->Opc1;
26161 MVT SrcVT = Src.getSimpleValueType();
26162 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
26163 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26164 return DAG.getNode(Opc, dl, Op.getValueType(), Src, Rnd, PassThru, Mask);
26165 }
26166 case CVTNEPS2BF16_MASK: {
26167 SDValue Src = Op.getOperand(1);
26168 SDValue PassThru = Op.getOperand(2);
26169 SDValue Mask = Op.getOperand(3);
26170
26171 if (ISD::isBuildVectorAllOnes(Mask.getNode()))
26172 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
26173
26174 // Break false dependency.
26175 if (PassThru.isUndef())
26176 PassThru = DAG.getConstant(0, dl, PassThru.getValueType());
26177
26178 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, PassThru,
26179 Mask);
26180 }
26181 default:
26182 break;
26183 }
26184 }
26185
26186 switch (IntNo) {
26187 default: return SDValue(); // Don't custom lower most intrinsics.
26188
26189 // ptest and testp intrinsics. The intrinsic these come from are designed to
26190 // return an integer value, not just an instruction so lower it to the ptest
26191 // or testp pattern and a setcc for the result.
26192 case Intrinsic::x86_avx512_ktestc_b:
26193 case Intrinsic::x86_avx512_ktestc_w:
26194 case Intrinsic::x86_avx512_ktestc_d:
26195 case Intrinsic::x86_avx512_ktestc_q:
26196 case Intrinsic::x86_avx512_ktestz_b:
26197 case Intrinsic::x86_avx512_ktestz_w:
26198 case Intrinsic::x86_avx512_ktestz_d:
26199 case Intrinsic::x86_avx512_ktestz_q:
26200 case Intrinsic::x86_sse41_ptestz:
26201 case Intrinsic::x86_sse41_ptestc:
26202 case Intrinsic::x86_sse41_ptestnzc:
26203 case Intrinsic::x86_avx_ptestz_256:
26204 case Intrinsic::x86_avx_ptestc_256:
26205 case Intrinsic::x86_avx_ptestnzc_256:
26206 case Intrinsic::x86_avx_vtestz_ps:
26207 case Intrinsic::x86_avx_vtestc_ps:
26208 case Intrinsic::x86_avx_vtestnzc_ps:
26209 case Intrinsic::x86_avx_vtestz_pd:
26210 case Intrinsic::x86_avx_vtestc_pd:
26211 case Intrinsic::x86_avx_vtestnzc_pd:
26212 case Intrinsic::x86_avx_vtestz_ps_256:
26213 case Intrinsic::x86_avx_vtestc_ps_256:
26214 case Intrinsic::x86_avx_vtestnzc_ps_256:
26215 case Intrinsic::x86_avx_vtestz_pd_256:
26216 case Intrinsic::x86_avx_vtestc_pd_256:
26217 case Intrinsic::x86_avx_vtestnzc_pd_256: {
26218 unsigned TestOpc = X86ISD::PTEST;
26219 X86::CondCode X86CC;
26220 switch (IntNo) {
26221 default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
26222 case Intrinsic::x86_avx512_ktestc_b:
26223 case Intrinsic::x86_avx512_ktestc_w:
26224 case Intrinsic::x86_avx512_ktestc_d:
26225 case Intrinsic::x86_avx512_ktestc_q:
26226 // CF = 1
26227 TestOpc = X86ISD::KTEST;
26228 X86CC = X86::COND_B;
26229 break;
26230 case Intrinsic::x86_avx512_ktestz_b:
26231 case Intrinsic::x86_avx512_ktestz_w:
26232 case Intrinsic::x86_avx512_ktestz_d:
26233 case Intrinsic::x86_avx512_ktestz_q:
26234 TestOpc = X86ISD::KTEST;
26235 X86CC = X86::COND_E;
26236 break;
26237 case Intrinsic::x86_avx_vtestz_ps:
26238 case Intrinsic::x86_avx_vtestz_pd:
26239 case Intrinsic::x86_avx_vtestz_ps_256:
26240 case Intrinsic::x86_avx_vtestz_pd_256:
26241 TestOpc = X86ISD::TESTP;
26242 [[fallthrough]];
26243 case Intrinsic::x86_sse41_ptestz:
26244 case Intrinsic::x86_avx_ptestz_256:
26245 // ZF = 1
26246 X86CC = X86::COND_E;
26247 break;
26248 case Intrinsic::x86_avx_vtestc_ps:
26249 case Intrinsic::x86_avx_vtestc_pd:
26250 case Intrinsic::x86_avx_vtestc_ps_256:
26251 case Intrinsic::x86_avx_vtestc_pd_256:
26252 TestOpc = X86ISD::TESTP;
26253 [[fallthrough]];
26254 case Intrinsic::x86_sse41_ptestc:
26255 case Intrinsic::x86_avx_ptestc_256:
26256 // CF = 1
26257 X86CC = X86::COND_B;
26258 break;
26259 case Intrinsic::x86_avx_vtestnzc_ps:
26260 case Intrinsic::x86_avx_vtestnzc_pd:
26261 case Intrinsic::x86_avx_vtestnzc_ps_256:
26262 case Intrinsic::x86_avx_vtestnzc_pd_256:
26263 TestOpc = X86ISD::TESTP;
26264 [[fallthrough]];
26265 case Intrinsic::x86_sse41_ptestnzc:
26266 case Intrinsic::x86_avx_ptestnzc_256:
26267 // ZF and CF = 0
26268 X86CC = X86::COND_A;
26269 break;
26270 }
26271
26272 SDValue LHS = Op.getOperand(1);
26273 SDValue RHS = Op.getOperand(2);
26274 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
26275 SDValue SetCC = getSETCC(X86CC, Test, dl, DAG);
26276 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
26277 }
26278
26279 case Intrinsic::x86_sse42_pcmpistria128:
26280 case Intrinsic::x86_sse42_pcmpestria128:
26281 case Intrinsic::x86_sse42_pcmpistric128:
26282 case Intrinsic::x86_sse42_pcmpestric128:
26283 case Intrinsic::x86_sse42_pcmpistrio128:
26284 case Intrinsic::x86_sse42_pcmpestrio128:
26285 case Intrinsic::x86_sse42_pcmpistris128:
26286 case Intrinsic::x86_sse42_pcmpestris128:
26287 case Intrinsic::x86_sse42_pcmpistriz128:
26288 case Intrinsic::x86_sse42_pcmpestriz128: {
26289 unsigned Opcode;
26290 X86::CondCode X86CC;
26291 switch (IntNo) {
26292 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
26293 case Intrinsic::x86_sse42_pcmpistria128:
26294 Opcode = X86ISD::PCMPISTR;
26295 X86CC = X86::COND_A;
26296 break;
26297 case Intrinsic::x86_sse42_pcmpestria128:
26298 Opcode = X86ISD::PCMPESTR;
26299 X86CC = X86::COND_A;
26300 break;
26301 case Intrinsic::x86_sse42_pcmpistric128:
26302 Opcode = X86ISD::PCMPISTR;
26303 X86CC = X86::COND_B;
26304 break;
26305 case Intrinsic::x86_sse42_pcmpestric128:
26306 Opcode = X86ISD::PCMPESTR;
26307 X86CC = X86::COND_B;
26308 break;
26309 case Intrinsic::x86_sse42_pcmpistrio128:
26310 Opcode = X86ISD::PCMPISTR;
26311 X86CC = X86::COND_O;
26312 break;
26313 case Intrinsic::x86_sse42_pcmpestrio128:
26314 Opcode = X86ISD::PCMPESTR;
26315 X86CC = X86::COND_O;
26316 break;
26317 case Intrinsic::x86_sse42_pcmpistris128:
26318 Opcode = X86ISD::PCMPISTR;
26319 X86CC = X86::COND_S;
26320 break;
26321 case Intrinsic::x86_sse42_pcmpestris128:
26322 Opcode = X86ISD::PCMPESTR;
26323 X86CC = X86::COND_S;
26324 break;
26325 case Intrinsic::x86_sse42_pcmpistriz128:
26326 Opcode = X86ISD::PCMPISTR;
26327 X86CC = X86::COND_E;
26328 break;
26329 case Intrinsic::x86_sse42_pcmpestriz128:
26330 Opcode = X86ISD::PCMPESTR;
26331 X86CC = X86::COND_E;
26332 break;
26333 }
26334 SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26335 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26336 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2);
26337 SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG);
26338 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
26339 }
26340
26341 case Intrinsic::x86_sse42_pcmpistri128:
26342 case Intrinsic::x86_sse42_pcmpestri128: {
26343 unsigned Opcode;
26344 if (IntNo == Intrinsic::x86_sse42_pcmpistri128)
26345 Opcode = X86ISD::PCMPISTR;
26346 else
26347 Opcode = X86ISD::PCMPESTR;
26348
26349 SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26350 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26351 return DAG.getNode(Opcode, dl, VTs, NewOps);
26352 }
26353
26354 case Intrinsic::x86_sse42_pcmpistrm128:
26355 case Intrinsic::x86_sse42_pcmpestrm128: {
26356 unsigned Opcode;
26357 if (IntNo == Intrinsic::x86_sse42_pcmpistrm128)
26358 Opcode = X86ISD::PCMPISTR;
26359 else
26360 Opcode = X86ISD::PCMPESTR;
26361
26362 SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops()));
26363 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
26364 return DAG.getNode(Opcode, dl, VTs, NewOps).getValue(1);
26365 }
26366
26367 case Intrinsic::eh_sjlj_lsda: {
26368 MachineFunction &MF = DAG.getMachineFunction();
26369 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26370 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
26371 auto &Context = MF.getContext();
26372 MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
26373 Twine(MF.getFunctionNumber()));
26374 return DAG.getNode(getGlobalWrapperKind(nullptr, /*OpFlags=*/0), dl, VT,
26375 DAG.getMCSymbol(S, PtrVT));
26376 }
26377
26378 case Intrinsic::x86_seh_lsda: {
26379 // Compute the symbol for the LSDA. We know it'll get emitted later.
26380 MachineFunction &MF = DAG.getMachineFunction();
26381 SDValue Op1 = Op.getOperand(1);
26382 auto *Fn = cast<Function>(cast<GlobalAddressSDNode>(Op1)->getGlobal());
26383 MCSymbol *LSDASym = MF.getContext().getOrCreateLSDASymbol(
26384 GlobalValue::dropLLVMManglingEscape(Fn->getName()));
26385
26386 // Generate a simple absolute symbol reference. This intrinsic is only
26387 // supported on 32-bit Windows, which isn't PIC.
26388 SDValue Result = DAG.getMCSymbol(LSDASym, VT);
26389 return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);
26390 }
26391
26392 case Intrinsic::eh_recoverfp: {
26393 SDValue FnOp = Op.getOperand(1);
26394 SDValue IncomingFPOp = Op.getOperand(2);
26395 GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
26396 auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
26397 if (!Fn)
26398 report_fatal_error(
26399 "llvm.eh.recoverfp must take a function as the first argument");
26400 return recoverFramePointer(DAG, Fn, IncomingFPOp);
26401 }
26402
26403 case Intrinsic::localaddress: {
26404 // Returns one of the stack, base, or frame pointer registers, depending on
26405 // which is used to reference local variables.
26406 MachineFunction &MF = DAG.getMachineFunction();
26407 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
26408 unsigned Reg;
26409 if (RegInfo->hasBasePointer(MF))
26410 Reg = RegInfo->getBaseRegister();
26411 else { // Handles the SP or FP case.
26412 bool CantUseFP = RegInfo->hasStackRealignment(MF);
26413 if (CantUseFP)
26414 Reg = RegInfo->getPtrSizedStackRegister(MF);
26415 else
26416 Reg = RegInfo->getPtrSizedFrameRegister(MF);
26417 }
26418 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
26419 }
26420 case Intrinsic::x86_avx512_vp2intersect_q_512:
26421 case Intrinsic::x86_avx512_vp2intersect_q_256:
26422 case Intrinsic::x86_avx512_vp2intersect_q_128:
26423 case Intrinsic::x86_avx512_vp2intersect_d_512:
26424 case Intrinsic::x86_avx512_vp2intersect_d_256:
26425 case Intrinsic::x86_avx512_vp2intersect_d_128: {
26426 MVT MaskVT = Op.getSimpleValueType();
26427
26428 SDVTList VTs = DAG.getVTList(MVT::Untyped, MVT::Other);
26429 SDLoc DL(Op);
26430
26431 SDValue Operation =
26432 DAG.getNode(X86ISD::VP2INTERSECT, DL, VTs,
26433 Op->getOperand(1), Op->getOperand(2));
26434
26435 SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
26436 MaskVT, Operation);
26437 SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
26438 MaskVT, Operation);
26439 return DAG.getMergeValues({Result0, Result1}, DL);
26440 }
26441 case Intrinsic::x86_mmx_pslli_w:
26442 case Intrinsic::x86_mmx_pslli_d:
26443 case Intrinsic::x86_mmx_pslli_q:
26444 case Intrinsic::x86_mmx_psrli_w:
26445 case Intrinsic::x86_mmx_psrli_d:
26446 case Intrinsic::x86_mmx_psrli_q:
26447 case Intrinsic::x86_mmx_psrai_w:
26448 case Intrinsic::x86_mmx_psrai_d: {
26449 SDLoc DL(Op);
26450 SDValue ShAmt = Op.getOperand(2);
26451 // If the argument is a constant, convert it to a target constant.
26452 if (auto *C = dyn_cast<ConstantSDNode>(ShAmt)) {
26453 // Clamp out of bounds shift amounts since they will otherwise be masked
26454 // to 8-bits which may make it no longer out of bounds.
26455 unsigned ShiftAmount = C->getAPIntValue().getLimitedValue(255);
26456 if (ShiftAmount == 0)
26457 return Op.getOperand(1);
26458
26459 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
26460 Op.getOperand(0), Op.getOperand(1),
26461 DAG.getTargetConstant(ShiftAmount, DL, MVT::i32));
26462 }
26463
26464 unsigned NewIntrinsic;
26465 switch (IntNo) {
26466 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
26467 case Intrinsic::x86_mmx_pslli_w:
26468 NewIntrinsic = Intrinsic::x86_mmx_psll_w;
26469 break;
26470 case Intrinsic::x86_mmx_pslli_d:
26471 NewIntrinsic = Intrinsic::x86_mmx_psll_d;
26472 break;
26473 case Intrinsic::x86_mmx_pslli_q:
26474 NewIntrinsic = Intrinsic::x86_mmx_psll_q;
26475 break;
26476 case Intrinsic::x86_mmx_psrli_w:
26477 NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
26478 break;
26479 case Intrinsic::x86_mmx_psrli_d:
26480 NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
26481 break;
26482 case Intrinsic::x86_mmx_psrli_q:
26483 NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
26484 break;
26485 case Intrinsic::x86_mmx_psrai_w:
26486 NewIntrinsic = Intrinsic::x86_mmx_psra_w;
26487 break;
26488 case Intrinsic::x86_mmx_psrai_d:
26489 NewIntrinsic = Intrinsic::x86_mmx_psra_d;
26490 break;
26491 }
26492
26493 // The vector shift intrinsics with scalars uses 32b shift amounts but
26494 // the sse2/mmx shift instructions reads 64 bits. Copy the 32 bits to an
26495 // MMX register.
26496 ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt);
26497 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
26498 DAG.getTargetConstant(NewIntrinsic, DL,
26499 getPointerTy(DAG.getDataLayout())),
26500 Op.getOperand(1), ShAmt);
26501 }
26502 case Intrinsic::thread_pointer: {
26503 if (Subtarget.isTargetELF()) {
26504 SDLoc dl(Op);
26505 EVT PtrVT = getPointerTy(DAG.getDataLayout());
26506 // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
26507 Value *Ptr = Constant::getNullValue(PointerType::get(
26508 *DAG.getContext(), Subtarget.is64Bit() ? X86AS::FS : X86AS::GS));
26509 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
26510 DAG.getIntPtrConstant(0, dl), MachinePointerInfo(Ptr));
26511 }
26512 report_fatal_error(
26513 "Target OS doesn't support __builtin_thread_pointer() yet.");
26514 }
26515 }
26516}
26517
26518static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
26519 SDValue Src, SDValue Mask, SDValue Base,
26520 SDValue Index, SDValue ScaleOp, SDValue Chain,
26521 const X86Subtarget &Subtarget) {
26522 SDLoc dl(Op);
26523 auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26524 // Scale must be constant.
26525 if (!C)
26526 return SDValue();
26527 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26528 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26529 TLI.getPointerTy(DAG.getDataLayout()));
26530 EVT MaskVT = Mask.getValueType().changeVectorElementTypeToInteger();
26531 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);
26532 // If source is undef or we know it won't be used, use a zero vector
26533 // to break register dependency.
26534 // TODO: use undef instead and let BreakFalseDeps deal with it?
26535 if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
26536 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
26537
26538 // Cast mask to an integer type.
26539 Mask = DAG.getBitcast(MaskVT, Mask);
26540
26541 MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26542
26543 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
26544 SDValue Res =
26545 DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
26546 MemIntr->getMemoryVT(), MemIntr->getMemOperand());
26547 return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
26548}
26549
26550static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
26551 SDValue Src, SDValue Mask, SDValue Base,
26552 SDValue Index, SDValue ScaleOp, SDValue Chain,
26553 const X86Subtarget &Subtarget) {
26554 MVT VT = Op.getSimpleValueType();
26555 SDLoc dl(Op);
26556 auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26557 // Scale must be constant.
26558 if (!C)
26559 return SDValue();
26560 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26561 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26562 TLI.getPointerTy(DAG.getDataLayout()));
26563 unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
26564 VT.getVectorNumElements());
26565 MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
26566
26567 // We support two versions of the gather intrinsics. One with scalar mask and
26568 // one with vXi1 mask. Convert scalar to vXi1 if necessary.
26569 if (Mask.getValueType() != MaskVT)
26570 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26571
26572 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);
26573 // If source is undef or we know it won't be used, use a zero vector
26574 // to break register dependency.
26575 // TODO: use undef instead and let BreakFalseDeps deal with it?
26576 if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
26577 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
26578
26579 MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26580
26581 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
26582 SDValue Res =
26583 DAG.getMemIntrinsicNode(X86ISD::MGATHER, dl, VTs, Ops,
26584 MemIntr->getMemoryVT(), MemIntr->getMemOperand());
26585 return DAG.getMergeValues({Res, Res.getValue(1)}, dl);
26586}
26587
26588static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
26589 SDValue Src, SDValue Mask, SDValue Base,
26590 SDValue Index, SDValue ScaleOp, SDValue Chain,
26591 const X86Subtarget &Subtarget) {
26592 SDLoc dl(Op);
26593 auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26594 // Scale must be constant.
26595 if (!C)
26596 return SDValue();
26597 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26598 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26599 TLI.getPointerTy(DAG.getDataLayout()));
26600 unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
26601 Src.getSimpleValueType().getVectorNumElements());
26602 MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
26603
26604 // We support two versions of the scatter intrinsics. One with scalar mask and
26605 // one with vXi1 mask. Convert scalar to vXi1 if necessary.
26606 if (Mask.getValueType() != MaskVT)
26607 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26608
26609 MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26610
26611 SDVTList VTs = DAG.getVTList(MVT::Other);
26612 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
26613 SDValue Res =
26614 DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
26615 MemIntr->getMemoryVT(), MemIntr->getMemOperand());
26616 return Res;
26617}
26618
26619static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
26620 SDValue Mask, SDValue Base, SDValue Index,
26621 SDValue ScaleOp, SDValue Chain,
26622 const X86Subtarget &Subtarget) {
26623 SDLoc dl(Op);
26624 auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
26625 // Scale must be constant.
26626 if (!C)
26627 return SDValue();
26628 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26629 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
26630 TLI.getPointerTy(DAG.getDataLayout()));
26631 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
26632 SDValue Segment = DAG.getRegister(0, MVT::i32);
26633 MVT MaskVT =
26634 MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
26635 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
26636 SDValue Ops[] = {VMask, Base, Scale, Index, Disp, Segment, Chain};
26637 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
26638 return SDValue(Res, 0);
26639}
26640
26641/// Handles the lowering of builtin intrinsics with chain that return their
26642/// value into registers EDX:EAX.
26643/// If operand ScrReg is a valid register identifier, then operand 2 of N is
26644/// copied to SrcReg. The assumption is that SrcReg is an implicit input to
26645/// TargetOpcode.
26646/// Returns a Glue value which can be used to add extra copy-from-reg if the
26647/// expanded intrinsics implicitly defines extra registers (i.e. not just
26648/// EDX:EAX).
26649static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
26650 SelectionDAG &DAG,
26651 unsigned TargetOpcode,
26652 unsigned SrcReg,
26653 const X86Subtarget &Subtarget,
26654 SmallVectorImpl<SDValue> &Results) {
26655 SDValue Chain = N->getOperand(0);
26656 SDValue Glue;
26657
26658 if (SrcReg) {
26659 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
26660 Chain = DAG.getCopyToReg(Chain, DL, SrcReg, N->getOperand(2), Glue);
26661 Glue = Chain.getValue(1);
26662 }
26663
26664 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
26665 SDValue N1Ops[] = {Chain, Glue};
26666 SDNode *N1 = DAG.getMachineNode(
26667 TargetOpcode, DL, Tys, ArrayRef<SDValue>(N1Ops, Glue.getNode() ? 2 : 1));
26668 Chain = SDValue(N1, 0);
26669
26670 // Reads the content of XCR and returns it in registers EDX:EAX.
26671 SDValue LO, HI;
26672 if (Subtarget.is64Bit()) {
26673 LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));
26674 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
26675 LO.getValue(2));
26676 } else {
26677 LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));
26678 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
26679 LO.getValue(2));
26680 }
26681 Chain = HI.getValue(1);
26682 Glue = HI.getValue(2);
26683
26684 if (Subtarget.is64Bit()) {
26685 // Merge the two 32-bit values into a 64-bit one.
26686 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
26687 DAG.getConstant(32, DL, MVT::i8));
26688 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
26689 Results.push_back(Chain);
26690 return Glue;
26691 }
26692
26693 // Use a buildpair to merge the two 32-bit values into a 64-bit one.
26694 SDValue Ops[] = { LO, HI };
26695 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
26696 Results.push_back(Pair);
26697 Results.push_back(Chain);
26698 return Glue;
26699}
26700
26701/// Handles the lowering of builtin intrinsics that read the time stamp counter
26702/// (x86_rdtsc and x86_rdtscp). This function is also used to custom lower
26703/// READCYCLECOUNTER nodes.
26704static void getReadTimeStampCounter(SDNode *N, const SDLoc &DL, unsigned Opcode,
26705 SelectionDAG &DAG,
26706 const X86Subtarget &Subtarget,
26707 SmallVectorImpl<SDValue> &Results) {
26708 // The processor's time-stamp counter (a 64-bit MSR) is stored into the
26709 // EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
26710 // and the EAX register is loaded with the low-order 32 bits.
26711 SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
26712 /* NoRegister */0, Subtarget,
26713 Results);
26714 if (Opcode != X86::RDTSCP)
26715 return;
26716
26717 SDValue Chain = Results[1];
26718 // Instruction RDTSCP loads the IA32:TSC_AUX_MSR (address C000_0103H) into
26719 // the ECX register. Add 'ecx' explicitly to the chain.
26720 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
26721 Results[1] = ecx;
26722 Results.push_back(ecx.getValue(1));
26723}
26724
26725static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget,
26726 SelectionDAG &DAG) {
26727 SmallVector<SDValue, 3> Results;
26728 SDLoc DL(Op);
26729 getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget,
26730 Results);
26731 return DAG.getMergeValues(Results, DL);
26732}
26733
26734static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
26735 MachineFunction &MF = DAG.getMachineFunction();
26736 SDValue Chain = Op.getOperand(0);
26737 SDValue RegNode = Op.getOperand(2);
26738 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
26739 if (!EHInfo)
26740 report_fatal_error("EH registrations only live in functions using WinEH");
26741
26742 // Cast the operand to an alloca, and remember the frame index.
26743 auto *FINode = dyn_cast<FrameIndexSDNode>(RegNode);
26744 if (!FINode)
26745 report_fatal_error("llvm.x86.seh.ehregnode expects a static alloca");
26746 EHInfo->EHRegNodeFrameIndex = FINode->getIndex();
26747
26748 // Return the chain operand without making any DAG nodes.
26749 return Chain;
26750}
26751
26752static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
26753 MachineFunction &MF = DAG.getMachineFunction();
26754 SDValue Chain = Op.getOperand(0);
26755 SDValue EHGuard = Op.getOperand(2);
26756 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
26757 if (!EHInfo)
26758 report_fatal_error("EHGuard only live in functions using WinEH");
26759
26760 // Cast the operand to an alloca, and remember the frame index.
26761 auto *FINode = dyn_cast<FrameIndexSDNode>(EHGuard);
26762 if (!FINode)
26763 report_fatal_error("llvm.x86.seh.ehguard expects a static alloca");
26764 EHInfo->EHGuardFrameIndex = FINode->getIndex();
26765
26766 // Return the chain operand without making any DAG nodes.
26767 return Chain;
26768}
26769
26770/// Emit Truncating Store with signed or unsigned saturation.
26771static SDValue
26772EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val,
26773 SDValue Ptr, EVT MemVT, MachineMemOperand *MMO,
26774 SelectionDAG &DAG) {
26775 SDVTList VTs = DAG.getVTList(MVT::Other);
26776 SDValue Undef = DAG.getUNDEF(Ptr.getValueType());
26777 SDValue Ops[] = { Chain, Val, Ptr, Undef };
26778 unsigned Opc = SignedSat ? X86ISD::VTRUNCSTORES : X86ISD::VTRUNCSTOREUS;
26779 return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
26780}
26781
26782/// Emit Masked Truncating Store with signed or unsigned saturation.
26783static SDValue EmitMaskedTruncSStore(bool SignedSat, SDValue Chain,
26784 const SDLoc &DL,
26785 SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
26786 MachineMemOperand *MMO, SelectionDAG &DAG) {
26787 SDVTList VTs = DAG.getVTList(MVT::Other);
26788 SDValue Ops[] = { Chain, Val, Ptr, Mask };
26789 unsigned Opc = SignedSat ? X86ISD::VMTRUNCSTORES : X86ISD::VMTRUNCSTOREUS;
26790 return DAG.getMemIntrinsicNode(Opc, DL, VTs, Ops, MemVT, MMO);
26791}
26792
26793bool X86::isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget,
26794 const MachineFunction &MF) {
26795 if (!Subtarget.is64Bit())
26796 return false;
26797 // 64-bit targets support extended Swift async frame setup,
26798 // except for targets that use the windows 64 prologue.
26799 return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
26800}
26801
26802static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
26803 SelectionDAG &DAG) {
26804 unsigned IntNo = Op.getConstantOperandVal(1);
26805 const IntrinsicData *IntrData = getIntrinsicWithChain(IntNo);
26806 if (!IntrData) {
26807 switch (IntNo) {
26808
26809 case Intrinsic::swift_async_context_addr: {
26810 SDLoc dl(Op);
26811 auto &MF = DAG.getMachineFunction();
26812 auto *X86FI = MF.getInfo<X86MachineFunctionInfo>();
26813 if (X86::isExtendedSwiftAsyncFrameSupported(Subtarget, MF)) {
26814 MF.getFrameInfo().setFrameAddressIsTaken(true);
26815 X86FI->setHasSwiftAsyncContext(true);
26816 SDValue Chain = Op->getOperand(0);
26817 SDValue CopyRBP = DAG.getCopyFromReg(Chain, dl, X86::RBP, MVT::i64);
26818 SDValue Result =
26819 SDValue(DAG.getMachineNode(X86::SUB64ri32, dl, MVT::i64, CopyRBP,
26820 DAG.getTargetConstant(8, dl, MVT::i32)),
26821 0);
26822 // Return { result, chain }.
26823 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
26824 CopyRBP.getValue(1));
26825 } else {
26826 // No special extended frame, create or reuse an existing stack slot.
26827 int PtrSize = Subtarget.is64Bit() ? 8 : 4;
26828 if (!X86FI->getSwiftAsyncContextFrameIdx())
26829 X86FI->setSwiftAsyncContextFrameIdx(
26830 MF.getFrameInfo().CreateStackObject(PtrSize, Align(PtrSize),
26831 false));
26832 SDValue Result =
26833 DAG.getFrameIndex(*X86FI->getSwiftAsyncContextFrameIdx(),
26834 PtrSize == 8 ? MVT::i64 : MVT::i32);
26835 // Return { result, chain }.
26836 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
26837 Op->getOperand(0));
26838 }
26839 }
26840
26841 case llvm::Intrinsic::x86_seh_ehregnode:
26842 return MarkEHRegistrationNode(Op, DAG);
26843 case llvm::Intrinsic::x86_seh_ehguard:
26844 return MarkEHGuard(Op, DAG);
26845 case llvm::Intrinsic::x86_rdpkru: {
26846 SDLoc dl(Op);
26847 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26848 // Create a RDPKRU node and pass 0 to the ECX parameter.
26849 return DAG.getNode(X86ISD::RDPKRU, dl, VTs, Op.getOperand(0),
26850 DAG.getConstant(0, dl, MVT::i32));
26851 }
26852 case llvm::Intrinsic::x86_wrpkru: {
26853 SDLoc dl(Op);
26854 // Create a WRPKRU node, pass the input to the EAX parameter, and pass 0
26855 // to the EDX and ECX parameters.
26856 return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other,
26857 Op.getOperand(0), Op.getOperand(2),
26858 DAG.getConstant(0, dl, MVT::i32),
26859 DAG.getConstant(0, dl, MVT::i32));
26860 }
26861 case llvm::Intrinsic::asan_check_memaccess: {
26862 // Mark this as adjustsStack because it will be lowered to a call.
26863 DAG.getMachineFunction().getFrameInfo().setAdjustsStack(true);
26864 // Don't do anything here, we will expand these intrinsics out later.
26865 return Op;
26866 }
26867 case llvm::Intrinsic::x86_flags_read_u32:
26868 case llvm::Intrinsic::x86_flags_read_u64:
26869 case llvm::Intrinsic::x86_flags_write_u32:
26870 case llvm::Intrinsic::x86_flags_write_u64: {
26871 // We need a frame pointer because this will get lowered to a PUSH/POP
26872 // sequence.
26873 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
26874 MFI.setHasCopyImplyingStackAdjustment(true);
26875 // Don't do anything here, we will expand these intrinsics out later
26876 // during FinalizeISel in EmitInstrWithCustomInserter.
26877 return Op;
26878 }
26879 case Intrinsic::x86_lwpins32:
26880 case Intrinsic::x86_lwpins64:
26881 case Intrinsic::x86_umwait:
26882 case Intrinsic::x86_tpause: {
26883 SDLoc dl(Op);
26884 SDValue Chain = Op->getOperand(0);
26885 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26886 unsigned Opcode;
26887
26888 switch (IntNo) {
26889 default: llvm_unreachable("Impossible intrinsic");
26890 case Intrinsic::x86_umwait:
26891 Opcode = X86ISD::UMWAIT;
26892 break;
26893 case Intrinsic::x86_tpause:
26894 Opcode = X86ISD::TPAUSE;
26895 break;
26896 case Intrinsic::x86_lwpins32:
26897 case Intrinsic::x86_lwpins64:
26898 Opcode = X86ISD::LWPINS;
26899 break;
26900 }
26901
26902 SDValue Operation =
26903 DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
26904 Op->getOperand(3), Op->getOperand(4));
26905 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
26906 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
26907 Operation.getValue(1));
26908 }
26909 case Intrinsic::x86_enqcmd:
26910 case Intrinsic::x86_enqcmds: {
26911 SDLoc dl(Op);
26912 SDValue Chain = Op.getOperand(0);
26913 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
26914 unsigned Opcode;
26915 switch (IntNo) {
26916 default: llvm_unreachable("Impossible intrinsic!");
26917 case Intrinsic::x86_enqcmd:
26918 Opcode = X86ISD::ENQCMD;
26919 break;
26920 case Intrinsic::x86_enqcmds:
26921 Opcode = X86ISD::ENQCMDS;
26922 break;
26923 }
26924 SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2),
26925 Op.getOperand(3));
26926 SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG);
26927 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
26928 Operation.getValue(1));
26929 }
26930 case Intrinsic::x86_aesenc128kl:
26931 case Intrinsic::x86_aesdec128kl:
26932 case Intrinsic::x86_aesenc256kl:
26933 case Intrinsic::x86_aesdec256kl: {
26934 SDLoc DL(Op);
26935 SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::i32, MVT::Other);
26936 SDValue Chain = Op.getOperand(0);
26937 unsigned Opcode;
26938
26939 switch (IntNo) {
26940 default: llvm_unreachable("Impossible intrinsic");
26941 case Intrinsic::x86_aesenc128kl:
26942 Opcode = X86ISD::AESENC128KL;
26943 break;
26944 case Intrinsic::x86_aesdec128kl:
26945 Opcode = X86ISD::AESDEC128KL;
26946 break;
26947 case Intrinsic::x86_aesenc256kl:
26948 Opcode = X86ISD::AESENC256KL;
26949 break;
26950 case Intrinsic::x86_aesdec256kl:
26951 Opcode = X86ISD::AESDEC256KL;
26952 break;
26953 }
26954
26955 MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26956 MachineMemOperand *MMO = MemIntr->getMemOperand();
26957 EVT MemVT = MemIntr->getMemoryVT();
26958 SDValue Operation = DAG.getMemIntrinsicNode(
26959 Opcode, DL, VTs, {Chain, Op.getOperand(2), Op.getOperand(3)}, MemVT,
26960 MMO);
26961 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(1), DL, DAG);
26962
26963 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
26964 {ZF, Operation.getValue(0), Operation.getValue(2)});
26965 }
26966 case Intrinsic::x86_aesencwide128kl:
26967 case Intrinsic::x86_aesdecwide128kl:
26968 case Intrinsic::x86_aesencwide256kl:
26969 case Intrinsic::x86_aesdecwide256kl: {
26970 SDLoc DL(Op);
26971 SDVTList VTs = DAG.getVTList(
26972 {MVT::i32, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64,
26973 MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::Other});
26974 SDValue Chain = Op.getOperand(0);
26975 unsigned Opcode;
26976
26977 switch (IntNo) {
26978 default: llvm_unreachable("Impossible intrinsic");
26979 case Intrinsic::x86_aesencwide128kl:
26980 Opcode = X86ISD::AESENCWIDE128KL;
26981 break;
26982 case Intrinsic::x86_aesdecwide128kl:
26983 Opcode = X86ISD::AESDECWIDE128KL;
26984 break;
26985 case Intrinsic::x86_aesencwide256kl:
26986 Opcode = X86ISD::AESENCWIDE256KL;
26987 break;
26988 case Intrinsic::x86_aesdecwide256kl:
26989 Opcode = X86ISD::AESDECWIDE256KL;
26990 break;
26991 }
26992
26993 MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
26994 MachineMemOperand *MMO = MemIntr->getMemOperand();
26995 EVT MemVT = MemIntr->getMemoryVT();
26996 SDValue Operation = DAG.getMemIntrinsicNode(
26997 Opcode, DL, VTs,
26998 {Chain, Op.getOperand(2), Op.getOperand(3), Op.getOperand(4),
26999 Op.getOperand(5), Op.getOperand(6), Op.getOperand(7),
27000 Op.getOperand(8), Op.getOperand(9), Op.getOperand(10)},
27001 MemVT, MMO);
27002 SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(0), DL, DAG);
27003
27004 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
27005 {ZF, Operation.getValue(1), Operation.getValue(2),
27006 Operation.getValue(3), Operation.getValue(4),
27007 Operation.getValue(5), Operation.getValue(6),
27008 Operation.getValue(7), Operation.getValue(8),
27009 Operation.getValue(9)});
27010 }
27011 case Intrinsic::x86_testui: {
27012 SDLoc dl(Op);
27013 SDValue Chain = Op.getOperand(0);
27014 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
27015 SDValue Operation = DAG.getNode(X86ISD::TESTUI, dl, VTs, Chain);
27016 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
27017 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
27018 Operation.getValue(1));
27019 }
27020 case Intrinsic::x86_atomic_bts_rm:
27021 case Intrinsic::x86_atomic_btc_rm:
27022 case Intrinsic::x86_atomic_btr_rm: {
27023 SDLoc DL(Op);
27024 MVT VT = Op.getSimpleValueType();
27025 SDValue Chain = Op.getOperand(0);
27026 SDValue Op1 = Op.getOperand(2);
27027 SDValue Op2 = Op.getOperand(3);
27028 unsigned Opc = IntNo == Intrinsic::x86_atomic_bts_rm ? X86ISD::LBTS_RM
27029 : IntNo == Intrinsic::x86_atomic_btc_rm ? X86ISD::LBTC_RM
27030 : X86ISD::LBTR_RM;
27031 MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
27032 SDValue Res =
27033 DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
27034 {Chain, Op1, Op2}, VT, MMO);
27035 Chain = Res.getValue(1);
27036 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
27037 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain);
27038 }
27039 case Intrinsic::x86_atomic_bts:
27040 case Intrinsic::x86_atomic_btc:
27041 case Intrinsic::x86_atomic_btr: {
27042 SDLoc DL(Op);
27043 MVT VT = Op.getSimpleValueType();
27044 SDValue Chain = Op.getOperand(0);
27045 SDValue Op1 = Op.getOperand(2);
27046 SDValue Op2 = Op.getOperand(3);
27047 unsigned Opc = IntNo == Intrinsic::x86_atomic_bts ? X86ISD::LBTS
27048 : IntNo == Intrinsic::x86_atomic_btc ? X86ISD::LBTC
27049 : X86ISD::LBTR;
27050 SDValue Size = DAG.getConstant(VT.getScalarSizeInBits(), DL, MVT::i32);
27051 MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
27052 SDValue Res =
27053 DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
27054 {Chain, Op1, Op2, Size}, VT, MMO);
27055 Chain = Res.getValue(1);
27056 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT);
27057 unsigned Imm = Op2->getAsZExtVal();
27058 if (Imm)
27059 Res = DAG.getNode(ISD::SHL, DL, VT, Res,
27060 DAG.getShiftAmountConstant(Imm, VT, DL));
27061 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Res, Chain);
27062 }
27063 case Intrinsic::x86_cmpccxadd32:
27064 case Intrinsic::x86_cmpccxadd64: {
27065 SDLoc DL(Op);
27066 SDValue Chain = Op.getOperand(0);
27067 SDValue Addr = Op.getOperand(2);
27068 SDValue Src1 = Op.getOperand(3);
27069 SDValue Src2 = Op.getOperand(4);
27070 SDValue CC = Op.getOperand(5);
27071 MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
27072 SDValue Operation = DAG.getMemIntrinsicNode(
27073 X86ISD::CMPCCXADD, DL, Op->getVTList(), {Chain, Addr, Src1, Src2, CC},
27074 MVT::i32, MMO);
27075 return Operation;
27076 }
27077 case Intrinsic::x86_aadd32:
27078 case Intrinsic::x86_aadd64:
27079 case Intrinsic::x86_aand32:
27080 case Intrinsic::x86_aand64:
27081 case Intrinsic::x86_aor32:
27082 case Intrinsic::x86_aor64:
27083 case Intrinsic::x86_axor32:
27084 case Intrinsic::x86_axor64: {
27085 SDLoc DL(Op);
27086 SDValue Chain = Op.getOperand(0);
27087 SDValue Op1 = Op.getOperand(2);
27088 SDValue Op2 = Op.getOperand(3);
27089 MVT VT = Op2.getSimpleValueType();
27090 unsigned Opc = 0;
27091 switch (IntNo) {
27092 default:
27093 llvm_unreachable("Unknown Intrinsic");
27094 case Intrinsic::x86_aadd32:
27095 case Intrinsic::x86_aadd64:
27096 Opc = X86ISD::AADD;
27097 break;
27098 case Intrinsic::x86_aand32:
27099 case Intrinsic::x86_aand64:
27100 Opc = X86ISD::AAND;
27101 break;
27102 case Intrinsic::x86_aor32:
27103 case Intrinsic::x86_aor64:
27104 Opc = X86ISD::AOR;
27105 break;
27106 case Intrinsic::x86_axor32:
27107 case Intrinsic::x86_axor64:
27108 Opc = X86ISD::AXOR;
27109 break;
27110 }
27111 MachineMemOperand *MMO = cast<MemSDNode>(Op)->getMemOperand();
27112 return DAG.getMemIntrinsicNode(Opc, DL, Op->getVTList(),
27113 {Chain, Op1, Op2}, VT, MMO);
27114 }
27115 case Intrinsic::x86_atomic_add_cc:
27116 case Intrinsic::x86_atomic_sub_cc:
27117 case Intrinsic::x86_atomic_or_cc:
27118 case Intrinsic::x86_atomic_and_cc:
27119 case Intrinsic::x86_atomic_xor_cc: {
27120 SDLoc DL(Op);
27121 SDValue Chain = Op.getOperand(0);
27122 SDValue Op1 = Op.getOperand(2);
27123 SDValue Op2 = Op.getOperand(3);
27124 X86::CondCode CC = (X86::CondCode)Op.getConstantOperandVal(4);
27125 MVT VT = Op2.getSimpleValueType();
27126 unsigned Opc = 0;
27127 switch (IntNo) {
27128 default:
27129 llvm_unreachable("Unknown Intrinsic");
27130 case Intrinsic::x86_atomic_add_cc:
27131 Opc = X86ISD::LADD;
27132 break;
27133 case Intrinsic::x86_atomic_sub_cc:
27134 Opc = X86ISD::LSUB;
27135 break;
27136 case Intrinsic::x86_atomic_or_cc:
27137 Opc = X86ISD::LOR;
27138 break;
27139 case Intrinsic::x86_atomic_and_cc:
27140 Opc = X86ISD::LAND;
27141 break;
27142 case Intrinsic::x86_atomic_xor_cc:
27143 Opc = X86ISD::LXOR;
27144 break;
27145 }
27146 MachineMemOperand *MMO = cast<MemIntrinsicSDNode>(Op)->getMemOperand();
27147 SDValue LockArith =
27148 DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),
27149 {Chain, Op1, Op2}, VT, MMO);
27150 Chain = LockArith.getValue(1);
27151 return DAG.getMergeValues({getSETCC(CC, LockArith, DL, DAG), Chain}, DL);
27152 }
27153 }
27154 return SDValue();
27155 }
27156
27157 SDLoc dl(Op);
27158 switch(IntrData->Type) {
27159 default: llvm_unreachable("Unknown Intrinsic Type");
27160 case RDSEED:
27161 case RDRAND: {
27162 // Emit the node with the right value type.
27163 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32, MVT::Other);
27164 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
27165
27166 // If the value returned by RDRAND/RDSEED was valid (CF=1), return 1.
27167 // Otherwise return the value from Rand, which is always 0, casted to i32.
27168 SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
27169 DAG.getConstant(1, dl, Op->getValueType(1)),
27170 DAG.getTargetConstant(X86::COND_B, dl, MVT::i8),
27171 SDValue(Result.getNode(), 1)};
27172 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
27173
27174 // Return { result, isValid, chain }.
27175 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
27176 SDValue(Result.getNode(), 2));
27177 }
27178 case GATHER_AVX2: {
27179 SDValue Chain = Op.getOperand(0);
27180 SDValue Src = Op.getOperand(2);
27181 SDValue Base = Op.getOperand(3);
27182 SDValue Index = Op.getOperand(4);
27183 SDValue Mask = Op.getOperand(5);
27184 SDValue Scale = Op.getOperand(6);
27185 return getAVX2GatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
27186 Scale, Chain, Subtarget);
27187 }
27188 case GATHER: {
27189 //gather(v1, mask, index, base, scale);
27190 SDValue Chain = Op.getOperand(0);
27191 SDValue Src = Op.getOperand(2);
27192 SDValue Base = Op.getOperand(3);
27193 SDValue Index = Op.getOperand(4);
27194 SDValue Mask = Op.getOperand(5);
27195 SDValue Scale = Op.getOperand(6);
27196 return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale,
27197 Chain, Subtarget);
27198 }
27199 case SCATTER: {
27200 //scatter(base, mask, index, v1, scale);
27201 SDValue Chain = Op.getOperand(0);
27202 SDValue Base = Op.getOperand(2);
27203 SDValue Mask = Op.getOperand(3);
27204 SDValue Index = Op.getOperand(4);
27205 SDValue Src = Op.getOperand(5);
27206 SDValue Scale = Op.getOperand(6);
27207 return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
27208 Scale, Chain, Subtarget);
27209 }
27210 case PREFETCH: {
27211 const APInt &HintVal = Op.getConstantOperandAPInt(6);
27212 assert((HintVal == 2 || HintVal == 3) &&
27213 "Wrong prefetch hint in intrinsic: should be 2 or 3");
27214 unsigned Opcode = (HintVal == 2 ? IntrData->Opc1 : IntrData->Opc0);
27215 SDValue Chain = Op.getOperand(0);
27216 SDValue Mask = Op.getOperand(2);
27217 SDValue Index = Op.getOperand(3);
27218 SDValue Base = Op.getOperand(4);
27219 SDValue Scale = Op.getOperand(5);
27220 return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain,
27221 Subtarget);
27222 }
27223 // Read Time Stamp Counter (RDTSC) and Processor ID (RDTSCP).
27224 case RDTSC: {
27225 SmallVector<SDValue, 2> Results;
27226 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget,
27227 Results);
27228 return DAG.getMergeValues(Results, dl);
27229 }
27230 // Read Performance Monitoring Counters.
27231 case RDPMC:
27232 // Read Processor Register.
27233 case RDPRU:
27234 // GetExtended Control Register.
27235 case XGETBV: {
27236 SmallVector<SDValue, 2> Results;
27237
27238 // RDPMC uses ECX to select the index of the performance counter to read.
27239 // RDPRU uses ECX to select the processor register to read.
27240 // XGETBV uses ECX to select the index of the XCR register to return.
27241 // The result is stored into registers EDX:EAX.
27242 expandIntrinsicWChainHelper(Op.getNode(), dl, DAG, IntrData->Opc0, X86::ECX,
27243 Subtarget, Results);
27244 return DAG.getMergeValues(Results, dl);
27245 }
27246 // XTEST intrinsics.
27247 case XTEST: {
27248 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
27249 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
27250
27251 SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
27252 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
27253 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
27254 Ret, SDValue(InTrans.getNode(), 1));
27255 }
27256 case TRUNCATE_TO_MEM_VI8:
27257 case TRUNCATE_TO_MEM_VI16:
27258 case TRUNCATE_TO_MEM_VI32: {
27259 SDValue Mask = Op.getOperand(4);
27260 SDValue DataToTruncate = Op.getOperand(3);
27261 SDValue Addr = Op.getOperand(2);
27262 SDValue Chain = Op.getOperand(0);
27263
27264 MemIntrinsicSDNode *MemIntr = dyn_cast<MemIntrinsicSDNode>(Op);
27265 assert(MemIntr && "Expected MemIntrinsicSDNode!");
27266
27267 EVT MemVT = MemIntr->getMemoryVT();
27268
27269 uint16_t TruncationOp = IntrData->Opc0;
27270 switch (TruncationOp) {
27271 case X86ISD::VTRUNC: {
27272 if (isAllOnesConstant(Mask)) // return just a truncate store
27273 return DAG.getTruncStore(Chain, dl, DataToTruncate, Addr, MemVT,
27274 MemIntr->getMemOperand());
27275
27276 MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());
27277 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27278 SDValue Offset = DAG.getUNDEF(VMask.getValueType());
27279
27280 return DAG.getMaskedStore(Chain, dl, DataToTruncate, Addr, Offset, VMask,
27281 MemVT, MemIntr->getMemOperand(), ISD::UNINDEXED,
27282 true /* truncating */);
27283 }
27284 case X86ISD::VTRUNCUS:
27285 case X86ISD::VTRUNCS: {
27286 bool IsSigned = (TruncationOp == X86ISD::VTRUNCS);
27287 if (isAllOnesConstant(Mask))
27288 return EmitTruncSStore(IsSigned, Chain, dl, DataToTruncate, Addr, MemVT,
27289 MemIntr->getMemOperand(), DAG);
27290
27291 MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());
27292 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
27293
27294 return EmitMaskedTruncSStore(IsSigned, Chain, dl, DataToTruncate, Addr,
27295 VMask, MemVT, MemIntr->getMemOperand(), DAG);
27296 }
27297 default:
27298 llvm_unreachable("Unsupported truncstore intrinsic");
27299 }
27300 }
27301 case INTR_TYPE_CAST_MMX:
27302 return SDValue(); // handled in combineINTRINSIC_*
27303 }
27304}
27305
27306SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
27307 SelectionDAG &DAG) const {
27308 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
27309 MFI.setReturnAddressIsTaken(true);
27310
27311 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
27312 return SDValue();
27313
27314 unsigned Depth = Op.getConstantOperandVal(0);
27315 SDLoc dl(Op);
27316 EVT PtrVT = getPointerTy(DAG.getDataLayout());
27317
27318 if (Depth > 0) {
27319 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
27320 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27321 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT);
27322 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
27323 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset),
27324 MachinePointerInfo());
27325 }
27326
27327 // Just load the return address.
27328 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
27329 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI,
27330 MachinePointerInfo());
27331}
27332
27333SDValue X86TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
27334 SelectionDAG &DAG) const {
27335 DAG.getMachineFunction().getFrameInfo().setReturnAddressIsTaken(true);
27336 return getReturnAddressFrameIndex(DAG);
27337}
27338
27339SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
27340 MachineFunction &MF = DAG.getMachineFunction();
27341 MachineFrameInfo &MFI = MF.getFrameInfo();
27342 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
27343 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27344 EVT VT = Op.getValueType();
27345
27346 MFI.setFrameAddressIsTaken(true);
27347
27348 if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
27349 // Depth > 0 makes no sense on targets which use Windows unwind codes. It
27350 // is not possible to crawl up the stack without looking at the unwind codes
27351 // simultaneously.
27352 int FrameAddrIndex = FuncInfo->getFAIndex();
27353 if (!FrameAddrIndex) {
27354 // Set up a frame object for the return address.
27355 unsigned SlotSize = RegInfo->getSlotSize();
27356 FrameAddrIndex = MF.getFrameInfo().CreateFixedObject(
27357 SlotSize, /*SPOffset=*/0, /*IsImmutable=*/false);
27358 FuncInfo->setFAIndex(FrameAddrIndex);
27359 }
27360 return DAG.getFrameIndex(FrameAddrIndex, VT);
27361 }
27362
27363 unsigned FrameReg =
27364 RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
27365 SDLoc dl(Op); // FIXME probably not meaningful
27366 unsigned Depth = Op.getConstantOperandVal(0);
27367 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
27368 (FrameReg == X86::EBP && VT == MVT::i32)) &&
27369 "Invalid Frame Register!");
27370 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
27371 while (Depth--)
27372 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
27373 MachinePointerInfo());
27374 return FrameAddr;
27375}
27376
27377// FIXME? Maybe this could be a TableGen attribute on some registers and
27378// this table could be generated automatically from RegInfo.
27379Register X86TargetLowering::getRegisterByName(const char* RegName, LLT VT,
27380 const MachineFunction &MF) const {
27381 const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
27382
27383 Register Reg = StringSwitch<unsigned>(RegName)
27384 .Case("esp", X86::ESP)
27385 .Case("rsp", X86::RSP)
27386 .Case("ebp", X86::EBP)
27387 .Case("rbp", X86::RBP)
27388 .Case("r14", X86::R14)
27389 .Case("r15", X86::R15)
27390 .Default(0);
27391
27392 if (Reg == X86::EBP || Reg == X86::RBP) {
27393 if (!TFI.hasFP(MF))
27394 report_fatal_error("register " + StringRef(RegName) +
27395 " is allocatable: function has no frame pointer");
27396#ifndef NDEBUG
27397 else {
27398 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27399 Register FrameReg = RegInfo->getPtrSizedFrameRegister(MF);
27400 assert((FrameReg == X86::EBP || FrameReg == X86::RBP) &&
27401 "Invalid Frame Register!");
27402 }
27403#endif
27404 }
27405
27406 if (Reg)
27407 return Reg;
27408
27409 report_fatal_error("Invalid register name global variable");
27410}
27411
27412SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
27413 SelectionDAG &DAG) const {
27414 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27415 return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize(), SDLoc(Op));
27416}
27417
27418Register X86TargetLowering::getExceptionPointerRegister(
27419 const Constant *PersonalityFn) const {
27420 if (classifyEHPersonality(PersonalityFn) == EHPersonality::CoreCLR)
27421 return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
27422
27423 return Subtarget.isTarget64BitLP64() ? X86::RAX : X86::EAX;
27424}
27425
27426Register X86TargetLowering::getExceptionSelectorRegister(
27427 const Constant *PersonalityFn) const {
27428 // Funclet personalities don't use selectors (the runtime does the selection).
27429 if (isFuncletEHPersonality(classifyEHPersonality(PersonalityFn)))
27430 return X86::NoRegister;
27431 return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
27432}
27433
27434bool X86TargetLowering::needsFixedCatchObjects() const {
27435 return Subtarget.isTargetWin64();
27436}
27437
27438SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
27439 SDValue Chain = Op.getOperand(0);
27440 SDValue Offset = Op.getOperand(1);
27441 SDValue Handler = Op.getOperand(2);
27442 SDLoc dl (Op);
27443
27444 EVT PtrVT = getPointerTy(DAG.getDataLayout());
27445 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
27446 Register FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction());
27447 assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) ||
27448 (FrameReg == X86::EBP && PtrVT == MVT::i32)) &&
27449 "Invalid Frame Register!");
27450 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
27451 Register StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
27452
27453 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
27454 DAG.getIntPtrConstant(RegInfo->getSlotSize(),
27455 dl));
27456 StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
27457 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
27458 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
27459
27460 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
27461 DAG.getRegister(StoreAddrReg, PtrVT));
27462}
27463
27464SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
27465 SelectionDAG &DAG) const {
27466 SDLoc DL(Op);
27467 // If the subtarget is not 64bit, we may need the global base reg
27468 // after isel expand pseudo, i.e., after CGBR pass ran.
27469 // Therefore, ask for the GlobalBaseReg now, so that the pass
27470 // inserts the code for us in case we need it.
27471 // Otherwise, we will end up in a situation where we will
27472 // reference a virtual register that is not defined!
27473 if (!Subtarget.is64Bit()) {
27474 const X86InstrInfo *TII = Subtarget.getInstrInfo();
27475 (void)TII->getGlobalBaseReg(&DAG.getMachineFunction());
27476 }
27477 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
27478 DAG.getVTList(MVT::i32, MVT::Other),
27479 Op.getOperand(0), Op.getOperand(1));
27480}
27481
27482SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
27483 SelectionDAG &DAG) const {
27484 SDLoc DL(Op);
27485 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
27486 Op.getOperand(0), Op.getOperand(1));
27487}
27488
27489SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
27490 SelectionDAG &DAG) const {
27491 SDLoc DL(Op);
27492 return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
27493 Op.getOperand(0));
27494}
27495
27496static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
27497 return Op.getOperand(0);
27498}
27499
27500SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
27501 SelectionDAG &DAG) const {
27502 SDValue Root = Op.getOperand(0);
27503 SDValue Trmp = Op.getOperand(1); // trampoline
27504 SDValue FPtr = Op.getOperand(2); // nested function
27505 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
27506 SDLoc dl (Op);
27507
27508 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
27509 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
27510
27511 if (Subtarget.is64Bit()) {
27512 SDValue OutChains[6];
27513
27514 // Large code-model.
27515 const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode.
27516 const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
27517
27518 const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
27519 const unsigned char N86R11 = TRI->getEncodingValue(X86::R11) & 0x7;
27520
27521 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
27522
27523 // Load the pointer to the nested function into R11.
27524 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11
27525 SDValue Addr = Trmp;
27526 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
27527 Addr, MachinePointerInfo(TrmpAddr));
27528
27529 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27530 DAG.getConstant(2, dl, MVT::i64));
27531 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr,
27532 MachinePointerInfo(TrmpAddr, 2), Align(2));
27533
27534 // Load the 'nest' parameter value into R10.
27535 // R10 is specified in X86CallingConv.td
27536 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10
27537 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27538 DAG.getConstant(10, dl, MVT::i64));
27539 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
27540 Addr, MachinePointerInfo(TrmpAddr, 10));
27541
27542 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27543 DAG.getConstant(12, dl, MVT::i64));
27544 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr,
27545 MachinePointerInfo(TrmpAddr, 12), Align(2));
27546
27547 // Jump to the nested function.
27548 OpCode = (JMP64r << 8) | REX_WB; // jmpq *...
27549 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27550 DAG.getConstant(20, dl, MVT::i64));
27551 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
27552 Addr, MachinePointerInfo(TrmpAddr, 20));
27553
27554 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11
27555 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
27556 DAG.getConstant(22, dl, MVT::i64));
27557 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8),
27558 Addr, MachinePointerInfo(TrmpAddr, 22));
27559
27560 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
27561 } else {
27562 const Function *Func =
27563 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
27564 CallingConv::ID CC = Func->getCallingConv();
27565 unsigned NestReg;
27566
27567 switch (CC) {
27568 default:
27569 llvm_unreachable("Unsupported calling convention");
27570 case CallingConv::C:
27571 case CallingConv::X86_StdCall: {
27572 // Pass 'nest' parameter in ECX.
27573 // Must be kept in sync with X86CallingConv.td
27574 NestReg = X86::ECX;
27575
27576 // Check that ECX wasn't needed by an 'inreg' parameter.
27577 FunctionType *FTy = Func->getFunctionType();
27578 const AttributeList &Attrs = Func->getAttributes();
27579
27580 if (!Attrs.isEmpty() && !Func->isVarArg()) {
27581 unsigned InRegCount = 0;
27582 unsigned Idx = 0;
27583
27584 for (FunctionType::param_iterator I = FTy->param_begin(),
27585 E = FTy->param_end(); I != E; ++I, ++Idx)
27586 if (Attrs.hasParamAttr(Idx, Attribute::InReg)) {
27587 const DataLayout &DL = DAG.getDataLayout();
27588 // FIXME: should only count parameters that are lowered to integers.
27589 InRegCount += (DL.getTypeSizeInBits(*I) + 31) / 32;
27590 }
27591
27592 if (InRegCount > 2) {
27593 report_fatal_error("Nest register in use - reduce number of inreg"
27594 " parameters!");
27595 }
27596 }
27597 break;
27598 }
27599 case CallingConv::X86_FastCall:
27600 case CallingConv::X86_ThisCall:
27601 case CallingConv::Fast:
27602 case CallingConv::Tail:
27603 case CallingConv::SwiftTail:
27604 // Pass 'nest' parameter in EAX.
27605 // Must be kept in sync with X86CallingConv.td
27606 NestReg = X86::EAX;
27607 break;
27608 }
27609
27610 SDValue OutChains[4];
27611 SDValue Addr, Disp;
27612
27613 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27614 DAG.getConstant(10, dl, MVT::i32));
27615 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
27616
27617 // This is storing the opcode for MOV32ri.
27618 const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
27619 const unsigned char N86Reg = TRI->getEncodingValue(NestReg) & 0x7;
27620 OutChains[0] =
27621 DAG.getStore(Root, dl, DAG.getConstant(MOV32ri | N86Reg, dl, MVT::i8),
27622 Trmp, MachinePointerInfo(TrmpAddr));
27623
27624 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27625 DAG.getConstant(1, dl, MVT::i32));
27626 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr,
27627 MachinePointerInfo(TrmpAddr, 1), Align(1));
27628
27629 const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode.
27630 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27631 DAG.getConstant(5, dl, MVT::i32));
27632 OutChains[2] =
27633 DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), Addr,
27634 MachinePointerInfo(TrmpAddr, 5), Align(1));
27635
27636 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
27637 DAG.getConstant(6, dl, MVT::i32));
27638 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr,
27639 MachinePointerInfo(TrmpAddr, 6), Align(1));
27640
27641 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
27642 }
27643}
27644
27645SDValue X86TargetLowering::LowerGET_ROUNDING(SDValue Op,
27646 SelectionDAG &DAG) const {
27647 /*
27648 The rounding mode is in bits 11:10 of FPSR, and has the following
27649 settings:
27650 00 Round to nearest
27651 01 Round to -inf
27652 10 Round to +inf
27653 11 Round to 0
27654
27655 GET_ROUNDING, on the other hand, expects the following:
27656 -1 Undefined
27657 0 Round to 0
27658 1 Round to nearest
27659 2 Round to +inf
27660 3 Round to -inf
27661
27662 To perform the conversion, we use a packed lookup table of the four 2-bit
27663 values that we can index by FPSP[11:10]
27664 0x2d --> (0b00,10,11,01) --> (0,2,3,1) >> FPSR[11:10]
27665
27666 (0x2d >> ((FPSR & 0xc00) >> 9)) & 3
27667 */
27668
27669 MachineFunction &MF = DAG.getMachineFunction();
27670 MVT VT = Op.getSimpleValueType();
27671 SDLoc DL(Op);
27672
27673 // Save FP Control Word to stack slot
27674 int SSFI = MF.getFrameInfo().CreateStackObject(2, Align(2), false);
27675 SDValue StackSlot =
27676 DAG.getFrameIndex(SSFI, getPointerTy(DAG.getDataLayout()));
27677
27678 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, SSFI);
27679
27680 SDValue Chain = Op.getOperand(0);
27681 SDValue Ops[] = {Chain, StackSlot};
27682 Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
27683 DAG.getVTList(MVT::Other), Ops, MVT::i16, MPI,
27684 Align(2), MachineMemOperand::MOStore);
27685
27686 // Load FP Control Word from stack slot
27687 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2));
27688 Chain = CWD.getValue(1);
27689
27690 // Mask and turn the control bits into a shift for the lookup table.
27691 SDValue Shift =
27692 DAG.getNode(ISD::SRL, DL, MVT::i16,
27693 DAG.getNode(ISD::AND, DL, MVT::i16,
27694 CWD, DAG.getConstant(0xc00, DL, MVT::i16)),
27695 DAG.getConstant(9, DL, MVT::i8));
27696 Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift);
27697
27698 SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);
27699 SDValue RetVal =
27700 DAG.getNode(ISD::AND, DL, MVT::i32,
27701 DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),
27702 DAG.getConstant(3, DL, MVT::i32));
27703
27704 RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT);
27705
27706 return DAG.getMergeValues({RetVal, Chain}, DL);
27707}
27708
27709SDValue X86TargetLowering::LowerSET_ROUNDING(SDValue Op,
27710 SelectionDAG &DAG) const {
27711 MachineFunction &MF = DAG.getMachineFunction();
27712 SDLoc DL(Op);
27713 SDValue Chain = Op.getNode()->getOperand(0);
27714
27715 // FP control word may be set only from data in memory. So we need to allocate
27716 // stack space to save/load FP control word.
27717 int OldCWFrameIdx = MF.getFrameInfo().CreateStackObject(4, Align(4), false);
27718 SDValue StackSlot =
27719 DAG.getFrameIndex(OldCWFrameIdx, getPointerTy(DAG.getDataLayout()));
27720 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, OldCWFrameIdx);
27721 MachineMemOperand *MMO =
27722 MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 2, Align(2));
27723
27724 // Store FP control word into memory.
27725 SDValue Ops[] = {Chain, StackSlot};
27726 Chain = DAG.getMemIntrinsicNode(
27727 X86ISD::FNSTCW16m, DL, DAG.getVTList(MVT::Other), Ops, MVT::i16, MMO);
27728
27729 // Load FP Control Word from stack slot and clear RM field (bits 11:10).
27730 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI);
27731 Chain = CWD.getValue(1);
27732 CWD = DAG.getNode(ISD::AND, DL, MVT::i16, CWD.getValue(0),
27733 DAG.getConstant(0xf3ff, DL, MVT::i16));
27734
27735 // Calculate new rounding mode.
27736 SDValue NewRM = Op.getNode()->getOperand(1);
27737 SDValue RMBits;
27738 if (auto *CVal = dyn_cast<ConstantSDNode>(NewRM)) {
27739 uint64_t RM = CVal->getZExtValue();
27740 int FieldVal;
27741 switch (static_cast<RoundingMode>(RM)) {
27742 // clang-format off
27743 case RoundingMode::NearestTiesToEven: FieldVal = X86::rmToNearest; break;
27744 case RoundingMode::TowardNegative: FieldVal = X86::rmDownward; break;
27745 case RoundingMode::TowardPositive: FieldVal = X86::rmUpward; break;
27746 case RoundingMode::TowardZero: FieldVal = X86::rmTowardZero; break;
27747 default:
27748 llvm_unreachable("rounding mode is not supported by X86 hardware");
27749 // clang-format on
27750 }
27751 RMBits = DAG.getConstant(FieldVal, DL, MVT::i16);
27752 } else {
27753 // Need to convert argument into bits of control word:
27754 // 0 Round to 0 -> 11
27755 // 1 Round to nearest -> 00
27756 // 2 Round to +inf -> 10
27757 // 3 Round to -inf -> 01
27758 // The 2-bit value needs then to be shifted so that it occupies bits 11:10.
27759 // To make the conversion, put all these values into a value 0xc9 and shift
27760 // it left depending on the rounding mode:
27761 // (0xc9 << 4) & 0xc00 = X86::rmTowardZero
27762 // (0xc9 << 6) & 0xc00 = X86::rmToNearest
27763 // ...
27764 // (0xc9 << (2 * NewRM + 4)) & 0xc00
27765 SDValue ShiftValue =
27766 DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
27767 DAG.getNode(ISD::ADD, DL, MVT::i32,
27768 DAG.getNode(ISD::SHL, DL, MVT::i32, NewRM,
27769 DAG.getConstant(1, DL, MVT::i8)),
27770 DAG.getConstant(4, DL, MVT::i32)));
27771 SDValue Shifted =
27772 DAG.getNode(ISD::SHL, DL, MVT::i16, DAG.getConstant(0xc9, DL, MVT::i16),
27773 ShiftValue);
27774 RMBits = DAG.getNode(ISD::AND, DL, MVT::i16, Shifted,
27775 DAG.getConstant(0xc00, DL, MVT::i16));
27776 }
27777
27778 // Update rounding mode bits and store the new FP Control Word into stack.
27779 CWD = DAG.getNode(ISD::OR, DL, MVT::i16, CWD, RMBits);
27780 Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(2));
27781
27782 // Load FP control word from the slot.
27783 SDValue OpsLD[] = {Chain, StackSlot};
27784 MachineMemOperand *MMOL =
27785 MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 2, Align(2));
27786 Chain = DAG.getMemIntrinsicNode(
27787 X86ISD::FLDCW16m, DL, DAG.getVTList(MVT::Other), OpsLD, MVT::i16, MMOL);
27788
27789 // If target supports SSE, set MXCSR as well. Rounding mode is encoded in the
27790 // same way but in bits 14:13.
27791 if (Subtarget.hasSSE1()) {
27792 // Store MXCSR into memory.
27793 Chain = DAG.getNode(
27794 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27795 DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),
27796 StackSlot);
27797
27798 // Load MXCSR from stack slot and clear RM field (bits 14:13).
27799 SDValue CWD = DAG.getLoad(MVT::i32, DL, Chain, StackSlot, MPI);
27800 Chain = CWD.getValue(1);
27801 CWD = DAG.getNode(ISD::AND, DL, MVT::i32, CWD.getValue(0),
27802 DAG.getConstant(0xffff9fff, DL, MVT::i32));
27803
27804 // Shift X87 RM bits from 11:10 to 14:13.
27805 RMBits = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, RMBits);
27806 RMBits = DAG.getNode(ISD::SHL, DL, MVT::i32, RMBits,
27807 DAG.getConstant(3, DL, MVT::i8));
27808
27809 // Update rounding mode bits and store the new FP Control Word into stack.
27810 CWD = DAG.getNode(ISD::OR, DL, MVT::i32, CWD, RMBits);
27811 Chain = DAG.getStore(Chain, DL, CWD, StackSlot, MPI, Align(4));
27812
27813 // Load MXCSR from the slot.
27814 Chain = DAG.getNode(
27815 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27816 DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),
27817 StackSlot);
27818 }
27819
27820 return Chain;
27821}
27822
27823const unsigned X87StateSize = 28;
27824const unsigned FPStateSize = 32;
27825[[maybe_unused]] const unsigned FPStateSizeInBits = FPStateSize * 8;
27826
27827SDValue X86TargetLowering::LowerGET_FPENV_MEM(SDValue Op,
27828 SelectionDAG &DAG) const {
27829 MachineFunction &MF = DAG.getMachineFunction();
27830 SDLoc DL(Op);
27831 SDValue Chain = Op->getOperand(0);
27832 SDValue Ptr = Op->getOperand(1);
27833 auto *Node = cast<FPStateAccessSDNode>(Op);
27834 EVT MemVT = Node->getMemoryVT();
27835 assert(MemVT.getSizeInBits() == FPStateSizeInBits);
27836 MachineMemOperand *MMO = cast<FPStateAccessSDNode>(Op)->getMemOperand();
27837
27838 // Get x87 state, if it presents.
27839 if (Subtarget.hasX87()) {
27840 Chain =
27841 DAG.getMemIntrinsicNode(X86ISD::FNSTENVm, DL, DAG.getVTList(MVT::Other),
27842 {Chain, Ptr}, MemVT, MMO);
27843
27844 // FNSTENV changes the exception mask, so load back the stored environment.
27845 MachineMemOperand::Flags NewFlags =
27846 MachineMemOperand::MOLoad |
27847 (MMO->getFlags() & ~MachineMemOperand::MOStore);
27848 MMO = MF.getMachineMemOperand(MMO, NewFlags);
27849 Chain =
27850 DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other),
27851 {Chain, Ptr}, MemVT, MMO);
27852 }
27853
27854 // If target supports SSE, get MXCSR as well.
27855 if (Subtarget.hasSSE1()) {
27856 // Get pointer to the MXCSR location in memory.
27857 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
27858 SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
27859 DAG.getConstant(X87StateSize, DL, PtrVT));
27860 // Store MXCSR into memory.
27861 Chain = DAG.getNode(
27862 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27863 DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),
27864 MXCSRAddr);
27865 }
27866
27867 return Chain;
27868}
27869
27870static SDValue createSetFPEnvNodes(SDValue Ptr, SDValue Chain, const SDLoc &DL,
27871 EVT MemVT, MachineMemOperand *MMO,
27872 SelectionDAG &DAG,
27873 const X86Subtarget &Subtarget) {
27874 // Set x87 state, if it presents.
27875 if (Subtarget.hasX87())
27876 Chain =
27877 DAG.getMemIntrinsicNode(X86ISD::FLDENVm, DL, DAG.getVTList(MVT::Other),
27878 {Chain, Ptr}, MemVT, MMO);
27879 // If target supports SSE, set MXCSR as well.
27880 if (Subtarget.hasSSE1()) {
27881 // Get pointer to the MXCSR location in memory.
27882 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
27883 SDValue MXCSRAddr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr,
27884 DAG.getConstant(X87StateSize, DL, PtrVT));
27885 // Load MXCSR from memory.
27886 Chain = DAG.getNode(
27887 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,
27888 DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),
27889 MXCSRAddr);
27890 }
27891 return Chain;
27892}
27893
27894SDValue X86TargetLowering::LowerSET_FPENV_MEM(SDValue Op,
27895 SelectionDAG &DAG) const {
27896 SDLoc DL(Op);
27897 SDValue Chain = Op->getOperand(0);
27898 SDValue Ptr = Op->getOperand(1);
27899 auto *Node = cast<FPStateAccessSDNode>(Op);
27900 EVT MemVT = Node->getMemoryVT();
27901 assert(MemVT.getSizeInBits() == FPStateSizeInBits);
27902 MachineMemOperand *MMO = cast<FPStateAccessSDNode>(Op)->getMemOperand();
27903 return createSetFPEnvNodes(Ptr, Chain, DL, MemVT, MMO, DAG, Subtarget);
27904}
27905
27906SDValue X86TargetLowering::LowerRESET_FPENV(SDValue Op,
27907 SelectionDAG &DAG) const {
27908 MachineFunction &MF = DAG.getMachineFunction();
27909 SDLoc DL(Op);
27910 SDValue Chain = Op.getNode()->getOperand(0);
27911
27912 IntegerType *ItemTy = Type::getInt32Ty(*DAG.getContext());
27913 ArrayType *FPEnvTy = ArrayType::get(ItemTy, 8);
27914 SmallVector<Constant *, 8> FPEnvVals;
27915
27916 // x87 FPU Control Word: mask all floating-point exceptions, sets rounding to
27917 // nearest. FPU precision is set to 53 bits on Windows and 64 bits otherwise
27918 // for compatibility with glibc.
27919 unsigned X87CW = Subtarget.isTargetWindowsMSVC() ? 0x27F : 0x37F;
27920 FPEnvVals.push_back(ConstantInt::get(ItemTy, X87CW));
27921 Constant *Zero = ConstantInt::get(ItemTy, 0);
27922 for (unsigned I = 0; I < 6; ++I)
27923 FPEnvVals.push_back(Zero);
27924
27925 // MXCSR: mask all floating-point exceptions, sets rounding to nearest, clear
27926 // all exceptions, sets DAZ and FTZ to 0.
27927 FPEnvVals.push_back(ConstantInt::get(ItemTy, 0x1F80));
27928 Constant *FPEnvBits = ConstantArray::get(FPEnvTy, FPEnvVals);
27929 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
27930 SDValue Env = DAG.getConstantPool(FPEnvBits, PtrVT);
27931 MachinePointerInfo MPI =
27932 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
27933 MachineMemOperand *MMO = MF.getMachineMemOperand(
27934 MPI, MachineMemOperand::MOStore, X87StateSize, Align(4));
27935
27936 return createSetFPEnvNodes(Env, Chain, DL, MVT::i32, MMO, DAG, Subtarget);
27937}
27938
27939/// Lower a vector CTLZ using native supported vector CTLZ instruction.
27940//
27941// i8/i16 vector implemented using dword LZCNT vector instruction
27942// ( sub(trunc(lzcnt(zext32(x)))) ). In case zext32(x) is illegal,
27943// split the vector, perform operation on it's Lo a Hi part and
27944// concatenate the results.
27945static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
27946 const X86Subtarget &Subtarget) {
27947 assert(Op.getOpcode() == ISD::CTLZ);
27948 SDLoc dl(Op);
27949 MVT VT = Op.getSimpleValueType();
27950 MVT EltVT = VT.getVectorElementType();
27951 unsigned NumElems = VT.getVectorNumElements();
27952
27953 assert((EltVT == MVT::i8 || EltVT == MVT::i16) &&
27954 "Unsupported element type");
27955
27956 // Split vector, it's Lo and Hi parts will be handled in next iteration.
27957 if (NumElems > 16 ||
27958 (NumElems == 16 && !Subtarget.canExtendTo512DQ()))
27959 return splitVectorIntUnary(Op, DAG, dl);
27960
27961 MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
27962 assert((NewVT.is256BitVector() || NewVT.is512BitVector()) &&
27963 "Unsupported value type for operation");
27964
27965 // Use native supported vector instruction vplzcntd.
27966 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0));
27967 SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op);
27968 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode);
27969 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT);
27970
27971 return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta);
27972}
27973
27974// Lower CTLZ using a PSHUFB lookup table implementation.
27975static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
27976 const X86Subtarget &Subtarget,
27977 SelectionDAG &DAG) {
27978 MVT VT = Op.getSimpleValueType();
27979 int NumElts = VT.getVectorNumElements();
27980 int NumBytes = NumElts * (VT.getScalarSizeInBits() / 8);
27981 MVT CurrVT = MVT::getVectorVT(MVT::i8, NumBytes);
27982
27983 // Per-nibble leading zero PSHUFB lookup table.
27984 const int LUT[16] = {/* 0 */ 4, /* 1 */ 3, /* 2 */ 2, /* 3 */ 2,
27985 /* 4 */ 1, /* 5 */ 1, /* 6 */ 1, /* 7 */ 1,
27986 /* 8 */ 0, /* 9 */ 0, /* a */ 0, /* b */ 0,
27987 /* c */ 0, /* d */ 0, /* e */ 0, /* f */ 0};
27988
27989 SmallVector<SDValue, 64> LUTVec;
27990 for (int i = 0; i < NumBytes; ++i)
27991 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
27992 SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
27993
27994 // Begin by bitcasting the input to byte vector, then split those bytes
27995 // into lo/hi nibbles and use the PSHUFB LUT to perform CTLZ on each of them.
27996 // If the hi input nibble is zero then we add both results together, otherwise
27997 // we just take the hi result (by masking the lo result to zero before the
27998 // add).
27999 SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0));
28000 SDValue Zero = DAG.getConstant(0, DL, CurrVT);
28001
28002 SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT);
28003 SDValue Lo = Op0;
28004 SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift);
28005 SDValue HiZ;
28006 if (CurrVT.is512BitVector()) {
28007 MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());
28008 HiZ = DAG.getSetCC(DL, MaskVT, Hi, Zero, ISD::SETEQ);
28009 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
28010 } else {
28011 HiZ = DAG.getSetCC(DL, CurrVT, Hi, Zero, ISD::SETEQ);
28012 }
28013
28014 Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo);
28015 Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi);
28016 Lo = DAG.getNode(ISD::AND, DL, CurrVT, Lo, HiZ);
28017 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
28018
28019 // Merge result back from vXi8 back to VT, working on the lo/hi halves
28020 // of the current vector width in the same way we did for the nibbles.
28021 // If the upper half of the input element is zero then add the halves'
28022 // leading zero counts together, otherwise just use the upper half's.
28023 // Double the width of the result until we are at target width.
28024 while (CurrVT != VT) {
28025 int CurrScalarSizeInBits = CurrVT.getScalarSizeInBits();
28026 int CurrNumElts = CurrVT.getVectorNumElements();
28027 MVT NextSVT = MVT::getIntegerVT(CurrScalarSizeInBits * 2);
28028 MVT NextVT = MVT::getVectorVT(NextSVT, CurrNumElts / 2);
28029 SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
28030
28031 // Check if the upper half of the input element is zero.
28032 if (CurrVT.is512BitVector()) {
28033 MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());
28034 HiZ = DAG.getSetCC(DL, MaskVT, DAG.getBitcast(CurrVT, Op0),
28035 DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
28036 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
28037 } else {
28038 HiZ = DAG.getSetCC(DL, CurrVT, DAG.getBitcast(CurrVT, Op0),
28039 DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
28040 }
28041 HiZ = DAG.getBitcast(NextVT, HiZ);
28042
28043 // Move the upper/lower halves to the lower bits as we'll be extending to
28044 // NextVT. Mask the lower result to zero if HiZ is true and add the results
28045 // together.
28046 SDValue ResNext = Res = DAG.getBitcast(NextVT, Res);
28047 SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift);
28048 SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift);
28049 R1 = DAG.getNode(ISD::AND, DL, NextVT, ResNext, R1);
28050 Res = DAG.getNode(ISD::ADD, DL, NextVT, R0, R1);
28051 CurrVT = NextVT;
28052 }
28053
28054 return Res;
28055}
28056
28057static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
28058 const X86Subtarget &Subtarget,
28059 SelectionDAG &DAG) {
28060 MVT VT = Op.getSimpleValueType();
28061
28062 if (Subtarget.hasCDI() &&
28063 // vXi8 vectors need to be promoted to 512-bits for vXi32.
28064 (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8))
28065 return LowerVectorCTLZ_AVX512CDI(Op, DAG, Subtarget);
28066
28067 // Decompose 256-bit ops into smaller 128-bit ops.
28068 if (VT.is256BitVector() && !Subtarget.hasInt256())
28069 return splitVectorIntUnary(Op, DAG, DL);
28070
28071 // Decompose 512-bit ops into smaller 256-bit ops.
28072 if (VT.is512BitVector() && !Subtarget.hasBWI())
28073 return splitVectorIntUnary(Op, DAG, DL);
28074
28075 assert(Subtarget.hasSSSE3() && "Expected SSSE3 support for PSHUFB");
28076 return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG);
28077}
28078
28079static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
28080 SelectionDAG &DAG) {
28081 MVT VT = Op.getSimpleValueType();
28082 MVT OpVT = VT;
28083 unsigned NumBits = VT.getSizeInBits();
28084 SDLoc dl(Op);
28085 unsigned Opc = Op.getOpcode();
28086
28087 if (VT.isVector())
28088 return LowerVectorCTLZ(Op, dl, Subtarget, DAG);
28089
28090 Op = Op.getOperand(0);
28091 if (VT == MVT::i8) {
28092 // Zero extend to i32 since there is not an i8 bsr.
28093 OpVT = MVT::i32;
28094 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
28095 }
28096
28097 // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
28098 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
28099 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
28100
28101 if (Opc == ISD::CTLZ) {
28102 // If src is zero (i.e. bsr sets ZF), returns NumBits.
28103 SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
28104 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
28105 Op.getValue(1)};
28106 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
28107 }
28108
28109 // Finally xor with NumBits-1.
28110 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op,
28111 DAG.getConstant(NumBits - 1, dl, OpVT));
28112
28113 if (VT == MVT::i8)
28114 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
28115 return Op;
28116}
28117
28118static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
28119 SelectionDAG &DAG) {
28120 MVT VT = Op.getSimpleValueType();
28121 unsigned NumBits = VT.getScalarSizeInBits();
28122 SDValue N0 = Op.getOperand(0);
28123 SDLoc dl(Op);
28124
28125 assert(!VT.isVector() && Op.getOpcode() == ISD::CTTZ &&
28126 "Only scalar CTTZ requires custom lowering");
28127
28128 // Issue a bsf (scan bits forward) which also sets EFLAGS.
28129 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
28130 Op = DAG.getNode(X86ISD::BSF, dl, VTs, N0);
28131
28132 // If src is known never zero we can skip the CMOV.
28133 if (DAG.isKnownNeverZero(N0))
28134 return Op;
28135
28136 // If src is zero (i.e. bsf sets ZF), returns NumBits.
28137 SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT),
28138 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
28139 Op.getValue(1)};
28140 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
28141}
28142
28143static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
28144 const X86Subtarget &Subtarget) {
28145 MVT VT = Op.getSimpleValueType();
28146 SDLoc DL(Op);
28147
28148 if (VT == MVT::i16 || VT == MVT::i32)
28149 return lowerAddSubToHorizontalOp(Op, DL, DAG, Subtarget);
28150
28151 if (VT == MVT::v32i16 || VT == MVT::v64i8)
28152 return splitVectorIntBinary(Op, DAG, DL);
28153
28154 assert(Op.getSimpleValueType().is256BitVector() &&
28155 Op.getSimpleValueType().isInteger() &&
28156 "Only handle AVX 256-bit vector integer operation");
28157 return splitVectorIntBinary(Op, DAG, DL);
28158}
28159
28160static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
28161 const X86Subtarget &Subtarget) {
28162 MVT VT = Op.getSimpleValueType();
28163 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
28164 unsigned Opcode = Op.getOpcode();
28165 SDLoc DL(Op);
28166
28167 if (VT == MVT::v32i16 || VT == MVT::v64i8 ||
28168 (VT.is256BitVector() && !Subtarget.hasInt256())) {
28169 assert(Op.getSimpleValueType().isInteger() &&
28170 "Only handle AVX vector integer operation");
28171 return splitVectorIntBinary(Op, DAG, DL);
28172 }
28173
28174 // Avoid the generic expansion with min/max if we don't have pminu*/pmaxu*.
28175 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28176 EVT SetCCResultType =
28177 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
28178
28179 unsigned BitWidth = VT.getScalarSizeInBits();
28180 if (Opcode == ISD::USUBSAT) {
28181 if (!TLI.isOperationLegal(ISD::UMAX, VT) || useVPTERNLOG(Subtarget, VT)) {
28182 // Handle a special-case with a bit-hack instead of cmp+select:
28183 // usubsat X, SMIN --> (X ^ SMIN) & (X s>> BW-1)
28184 // If the target can use VPTERNLOG, DAGToDAG will match this as
28185 // "vpsra + vpternlog" which is better than "vpmax + vpsub" with a
28186 // "broadcast" constant load.
28187 ConstantSDNode *C = isConstOrConstSplat(Y, true);
28188 if (C && C->getAPIntValue().isSignMask()) {
28189 SDValue SignMask = DAG.getConstant(C->getAPIntValue(), DL, VT);
28190 SDValue ShiftAmt = DAG.getConstant(BitWidth - 1, DL, VT);
28191 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, SignMask);
28192 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShiftAmt);
28193 return DAG.getNode(ISD::AND, DL, VT, Xor, Sra);
28194 }
28195 }
28196 if (!TLI.isOperationLegal(ISD::UMAX, VT)) {
28197 // usubsat X, Y --> (X >u Y) ? X - Y : 0
28198 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
28199 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT);
28200 // TODO: Move this to DAGCombiner?
28201 if (SetCCResultType == VT &&
28202 DAG.ComputeNumSignBits(Cmp) == VT.getScalarSizeInBits())
28203 return DAG.getNode(ISD::AND, DL, VT, Cmp, Sub);
28204 return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT));
28205 }
28206 }
28207
28208 if ((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
28209 (!VT.isVector() || VT == MVT::v2i64)) {
28210 APInt MinVal = APInt::getSignedMinValue(BitWidth);
28211 APInt MaxVal = APInt::getSignedMaxValue(BitWidth);
28212 SDValue Zero = DAG.getConstant(0, DL, VT);
28213 SDValue Result =
28214 DAG.getNode(Opcode == ISD::SADDSAT ? ISD::SADDO : ISD::SSUBO, DL,
28215 DAG.getVTList(VT, SetCCResultType), X, Y);
28216 SDValue SumDiff = Result.getValue(0);
28217 SDValue Overflow = Result.getValue(1);
28218 SDValue SatMin = DAG.getConstant(MinVal, DL, VT);
28219 SDValue SatMax = DAG.getConstant(MaxVal, DL, VT);
28220 SDValue SumNeg =
28221 DAG.getSetCC(DL, SetCCResultType, SumDiff, Zero, ISD::SETLT);
28222 Result = DAG.getSelect(DL, VT, SumNeg, SatMax, SatMin);
28223 return DAG.getSelect(DL, VT, Overflow, Result, SumDiff);
28224 }
28225
28226 // Use default expansion.
28227 return SDValue();
28228}
28229
28230static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
28231 SelectionDAG &DAG) {
28232 MVT VT = Op.getSimpleValueType();
28233 SDLoc DL(Op);
28234
28235 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) {
28236 // Since X86 does not have CMOV for 8-bit integer, we don't convert
28237 // 8-bit integer abs to NEG and CMOV.
28238 SDValue N0 = Op.getOperand(0);
28239 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
28240 DAG.getConstant(0, DL, VT), N0);
28241 SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_NS, DL, MVT::i8),
28242 SDValue(Neg.getNode(), 1)};
28243 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
28244 }
28245
28246 // ABS(vXi64 X) --> VPBLENDVPD(X, 0-X, X).
28247 if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) {
28248 SDValue Src = Op.getOperand(0);
28249 SDValue Neg = DAG.getNegative(Src, DL, VT);
28250 return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Neg, Src);
28251 }
28252
28253 if (VT.is256BitVector() && !Subtarget.hasInt256()) {
28254 assert(VT.isInteger() &&
28255 "Only handle AVX 256-bit vector integer operation");
28256 return splitVectorIntUnary(Op, DAG, DL);
28257 }
28258
28259 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
28260 return splitVectorIntUnary(Op, DAG, DL);
28261
28262 // Default to expand.
28263 return SDValue();
28264}
28265
28266static SDValue LowerAVG(SDValue Op, const X86Subtarget &Subtarget,
28267 SelectionDAG &DAG) {
28268 MVT VT = Op.getSimpleValueType();
28269 SDLoc DL(Op);
28270
28271 // For AVX1 cases, split to use legal ops.
28272 if (VT.is256BitVector() && !Subtarget.hasInt256())
28273 return splitVectorIntBinary(Op, DAG, DL);
28274
28275 if (VT == MVT::v32i16 || VT == MVT::v64i8)
28276 return splitVectorIntBinary(Op, DAG, DL);
28277
28278 // Default to expand.
28279 return SDValue();
28280}
28281
28282static SDValue LowerMINMAX(SDValue Op, const X86Subtarget &Subtarget,
28283 SelectionDAG &DAG) {
28284 MVT VT = Op.getSimpleValueType();
28285 SDLoc DL(Op);
28286
28287 // For AVX1 cases, split to use legal ops.
28288 if (VT.is256BitVector() && !Subtarget.hasInt256())
28289 return splitVectorIntBinary(Op, DAG, DL);
28290
28291 if (VT == MVT::v32i16 || VT == MVT::v64i8)
28292 return splitVectorIntBinary(Op, DAG, DL);
28293
28294 // Default to expand.
28295 return SDValue();
28296}
28297
28298static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
28299 SelectionDAG &DAG) {
28300 assert((Op.getOpcode() == ISD::FMAXIMUM || Op.getOpcode() == ISD::FMINIMUM) &&
28301 "Expected FMAXIMUM or FMINIMUM opcode");
28302 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28303 EVT VT = Op.getValueType();
28304 SDValue X = Op.getOperand(0);
28305 SDValue Y = Op.getOperand(1);
28306 SDLoc DL(Op);
28307 uint64_t SizeInBits = VT.getScalarSizeInBits();
28308 APInt PreferredZero = APInt::getZero(SizeInBits);
28309 APInt OppositeZero = PreferredZero;
28310 EVT IVT = VT.changeTypeToInteger();
28311 X86ISD::NodeType MinMaxOp;
28312 if (Op.getOpcode() == ISD::FMAXIMUM) {
28313 MinMaxOp = X86ISD::FMAX;
28314 OppositeZero.setSignBit();
28315 } else {
28316 PreferredZero.setSignBit();
28317 MinMaxOp = X86ISD::FMIN;
28318 }
28319 EVT SetCCType =
28320 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
28321
28322 // The tables below show the expected result of Max in cases of NaN and
28323 // signed zeros.
28324 //
28325 // Y Y
28326 // Num xNaN +0 -0
28327 // --------------- ---------------
28328 // Num | Max | Y | +0 | +0 | +0 |
28329 // X --------------- X ---------------
28330 // xNaN | X | X/Y | -0 | +0 | -0 |
28331 // --------------- ---------------
28332 //
28333 // It is achieved by means of FMAX/FMIN with preliminary checks and operand
28334 // reordering.
28335 //
28336 // We check if any of operands is NaN and return NaN. Then we check if any of
28337 // operands is zero or negative zero (for fmaximum and fminimum respectively)
28338 // to ensure the correct zero is returned.
28339 auto MatchesZero = [](SDValue Op, APInt Zero) {
28340 Op = peekThroughBitcasts(Op);
28341 if (auto *CstOp = dyn_cast<ConstantFPSDNode>(Op))
28342 return CstOp->getValueAPF().bitcastToAPInt() == Zero;
28343 if (auto *CstOp = dyn_cast<ConstantSDNode>(Op))
28344 return CstOp->getAPIntValue() == Zero;
28345 if (Op->getOpcode() == ISD::BUILD_VECTOR ||
28346 Op->getOpcode() == ISD::SPLAT_VECTOR) {
28347 for (const SDValue &OpVal : Op->op_values()) {
28348 if (OpVal.isUndef())
28349 continue;
28350 auto *CstOp = dyn_cast<ConstantFPSDNode>(OpVal);
28351 if (!CstOp)
28352 return false;
28353 if (!CstOp->getValueAPF().isZero())
28354 continue;
28355 if (CstOp->getValueAPF().bitcastToAPInt() != Zero)
28356 return false;
28357 }
28358 return true;
28359 }
28360 return false;
28361 };
28362
28363 bool IsXNeverNaN = DAG.isKnownNeverNaN(X);
28364 bool IsYNeverNaN = DAG.isKnownNeverNaN(Y);
28365 bool IgnoreSignedZero = DAG.getTarget().Options.NoSignedZerosFPMath ||
28366 Op->getFlags().hasNoSignedZeros() ||
28367 DAG.isKnownNeverZeroFloat(X) ||
28368 DAG.isKnownNeverZeroFloat(Y);
28369 SDValue NewX, NewY;
28370 if (IgnoreSignedZero || MatchesZero(Y, PreferredZero) ||
28371 MatchesZero(X, OppositeZero)) {
28372 // Operands are already in right order or order does not matter.
28373 NewX = X;
28374 NewY = Y;
28375 } else if (MatchesZero(X, PreferredZero) || MatchesZero(Y, OppositeZero)) {
28376 NewX = Y;
28377 NewY = X;
28378 } else if (!VT.isVector() && (VT == MVT::f16 || Subtarget.hasDQI()) &&
28379 (Op->getFlags().hasNoNaNs() || IsXNeverNaN || IsYNeverNaN)) {
28380 if (IsXNeverNaN)
28381 std::swap(X, Y);
28382 // VFPCLASSS consumes a vector type. So provide a minimal one corresponded
28383 // xmm register.
28384 MVT VectorType = MVT::getVectorVT(VT.getSimpleVT(), 128 / SizeInBits);
28385 SDValue VX = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VectorType, X);
28386 // Bits of classes:
28387 // Bits Imm8[0] Imm8[1] Imm8[2] Imm8[3] Imm8[4] Imm8[5] Imm8[6] Imm8[7]
28388 // Class QNAN PosZero NegZero PosINF NegINF Denormal Negative SNAN
28389 SDValue Imm = DAG.getTargetConstant(MinMaxOp == X86ISD::FMAX ? 0b11 : 0b101,
28390 DL, MVT::i32);
28391 SDValue IsNanZero = DAG.getNode(X86ISD::VFPCLASSS, DL, MVT::v1i1, VX, Imm);
28392 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1,
28393 DAG.getConstant(0, DL, MVT::v8i1), IsNanZero,
28394 DAG.getIntPtrConstant(0, DL));
28395 SDValue NeedSwap = DAG.getBitcast(MVT::i8, Ins);
28396 NewX = DAG.getSelect(DL, VT, NeedSwap, Y, X);
28397 NewY = DAG.getSelect(DL, VT, NeedSwap, X, Y);
28398 return DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
28399 } else {
28400 SDValue IsXSigned;
28401 if (Subtarget.is64Bit() || VT != MVT::f64) {
28402 SDValue XInt = DAG.getNode(ISD::BITCAST, DL, IVT, X);
28403 SDValue ZeroCst = DAG.getConstant(0, DL, IVT);
28404 IsXSigned = DAG.getSetCC(DL, SetCCType, XInt, ZeroCst, ISD::SETLT);
28405 } else {
28406 assert(VT == MVT::f64);
28407 SDValue Ins = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v2f64,
28408 DAG.getConstantFP(0, DL, MVT::v2f64), X,
28409 DAG.getIntPtrConstant(0, DL));
28410 SDValue VX = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, Ins);
28411 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VX,
28412 DAG.getIntPtrConstant(1, DL));
28413 Hi = DAG.getBitcast(MVT::i32, Hi);
28414 SDValue ZeroCst = DAG.getConstant(0, DL, MVT::i32);
28415 EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(),
28416 *DAG.getContext(), MVT::i32);
28417 IsXSigned = DAG.getSetCC(DL, SetCCType, Hi, ZeroCst, ISD::SETLT);
28418 }
28419 if (MinMaxOp == X86ISD::FMAX) {
28420 NewX = DAG.getSelect(DL, VT, IsXSigned, X, Y);
28421 NewY = DAG.getSelect(DL, VT, IsXSigned, Y, X);
28422 } else {
28423 NewX = DAG.getSelect(DL, VT, IsXSigned, Y, X);
28424 NewY = DAG.getSelect(DL, VT, IsXSigned, X, Y);
28425 }
28426 }
28427
28428 bool IgnoreNaN = DAG.getTarget().Options.NoNaNsFPMath ||
28429 Op->getFlags().hasNoNaNs() || (IsXNeverNaN && IsYNeverNaN);
28430
28431 // If we did no ordering operands for signed zero handling and we need
28432 // to process NaN and we know that the second operand is not NaN then put
28433 // it in first operand and we will not need to post handle NaN after max/min.
28434 if (IgnoreSignedZero && !IgnoreNaN && DAG.isKnownNeverNaN(NewY))
28435 std::swap(NewX, NewY);
28436
28437 SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
28438
28439 if (IgnoreNaN || DAG.isKnownNeverNaN(NewX))
28440 return MinMax;
28441
28442 SDValue IsNaN = DAG.getSetCC(DL, SetCCType, NewX, NewX, ISD::SETUO);
28443 return DAG.getSelect(DL, VT, IsNaN, NewX, MinMax);
28444}
28445
28446static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
28447 SelectionDAG &DAG) {
28448 MVT VT = Op.getSimpleValueType();
28449 SDLoc dl(Op);
28450
28451 // For AVX1 cases, split to use legal ops.
28452 if (VT.is256BitVector() && !Subtarget.hasInt256())
28453 return splitVectorIntBinary(Op, DAG, dl);
28454
28455 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.useBWIRegs())
28456 return splitVectorIntBinary(Op, DAG, dl);
28457
28458 bool IsSigned = Op.getOpcode() == ISD::ABDS;
28459 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28460
28461 if (VT.isScalarInteger()) {
28462 // abds(lhs, rhs) -> select(slt(lhs,rhs),sub(rhs,lhs),sub(lhs,rhs))
28463 // abdu(lhs, rhs) -> select(ult(lhs,rhs),sub(rhs,lhs),sub(lhs,rhs))
28464 if (Subtarget.canUseCMOV() && VT.bitsGE(MVT::i32)) {
28465 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
28466 X86::CondCode CC = IsSigned ? X86::COND_L : X86::COND_B;
28467 SDValue LHS = DAG.getFreeze(Op.getOperand(0));
28468 SDValue RHS = DAG.getFreeze(Op.getOperand(1));
28469 SDValue Diff0 = DAG.getNode(X86ISD::SUB, dl, VTs, LHS, RHS);
28470 SDValue Diff1 = DAG.getNode(X86ISD::SUB, dl, VTs, RHS, LHS);
28471 return DAG.getNode(X86ISD::CMOV, dl, VT, Diff1, Diff0,
28472 DAG.getTargetConstant(CC, dl, MVT::i8),
28473 Diff1.getValue(1));
28474 }
28475
28476 // TODO: Move to TargetLowering expandABD() once we have ABD promotion.
28477 // abds(lhs, rhs) -> trunc(abs(sub(sext(lhs), sext(rhs))))
28478 // abdu(lhs, rhs) -> trunc(abs(sub(zext(lhs), zext(rhs))))
28479 unsigned WideBits = std::max<unsigned>(2 * VT.getScalarSizeInBits(), 32u);
28480 MVT WideVT = MVT::getIntegerVT(WideBits);
28481 if (TLI.isTypeLegal(WideVT)) {
28482 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
28483 SDValue LHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(0));
28484 SDValue RHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(1));
28485 SDValue Diff = DAG.getNode(ISD::SUB, dl, WideVT, LHS, RHS);
28486 SDValue AbsDiff = DAG.getNode(ISD::ABS, dl, WideVT, Diff);
28487 return DAG.getNode(ISD::TRUNCATE, dl, VT, AbsDiff);
28488 }
28489 }
28490
28491 // Default to expand.
28492 return SDValue();
28493}
28494
28495static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
28496 SelectionDAG &DAG) {
28497 SDLoc dl(Op);
28498 MVT VT = Op.getSimpleValueType();
28499
28500 // Decompose 256-bit ops into 128-bit ops.
28501 if (VT.is256BitVector() && !Subtarget.hasInt256())
28502 return splitVectorIntBinary(Op, DAG, dl);
28503
28504 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
28505 return splitVectorIntBinary(Op, DAG, dl);
28506
28507 SDValue A = Op.getOperand(0);
28508 SDValue B = Op.getOperand(1);
28509
28510 // Lower v16i8/v32i8/v64i8 mul as sign-extension to v8i16/v16i16/v32i16
28511 // vector pairs, multiply and truncate.
28512 if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) {
28513 unsigned NumElts = VT.getVectorNumElements();
28514 unsigned NumLanes = VT.getSizeInBits() / 128;
28515 unsigned NumEltsPerLane = NumElts / NumLanes;
28516
28517 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
28518 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
28519 MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
28520 return DAG.getNode(
28521 ISD::TRUNCATE, dl, VT,
28522 DAG.getNode(ISD::MUL, dl, ExVT,
28523 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, A),
28524 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, B)));
28525 }
28526
28527 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
28528
28529 // For vXi8 mul, try PMADDUBSW to avoid the need for extension.
28530 // Don't do this if we only need to unpack one half.
28531 if (Subtarget.hasSSSE3()) {
28532 bool BIsBuildVector = isa<BuildVectorSDNode>(B);
28533 bool IsLoLaneAllZeroOrUndef = BIsBuildVector;
28534 bool IsHiLaneAllZeroOrUndef = BIsBuildVector;
28535 if (BIsBuildVector) {
28536 for (auto [Idx, Val] : enumerate(B->ops())) {
28537 if ((Idx % NumEltsPerLane) >= (NumEltsPerLane / 2))
28538 IsHiLaneAllZeroOrUndef &= isNullConstantOrUndef(Val);
28539 else
28540 IsLoLaneAllZeroOrUndef &= isNullConstantOrUndef(Val);
28541 }
28542 }
28543 if (!(IsLoLaneAllZeroOrUndef || IsHiLaneAllZeroOrUndef)) {
28544 SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(0x00FF, dl, ExVT));
28545 SDValue BLo = DAG.getNode(ISD::AND, dl, VT, Mask, B);
28546 SDValue BHi = DAG.getNode(X86ISD::ANDNP, dl, VT, Mask, B);
28547 SDValue RLo = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BLo);
28548 SDValue RHi = DAG.getNode(X86ISD::VPMADDUBSW, dl, ExVT, A, BHi);
28549 RLo = DAG.getNode(ISD::AND, dl, VT, DAG.getBitcast(VT, RLo), Mask);
28550 RHi = DAG.getNode(X86ISD::VSHLI, dl, ExVT, RHi,
28551 DAG.getTargetConstant(8, dl, MVT::i8));
28552 return DAG.getNode(ISD::OR, dl, VT, RLo, DAG.getBitcast(VT, RHi));
28553 }
28554 }
28555
28556 // Extract the lo/hi parts to any extend to i16.
28557 // We're going to mask off the low byte of each result element of the
28558 // pmullw, so it doesn't matter what's in the high byte of each 16-bit
28559 // element.
28560 SDValue Undef = DAG.getUNDEF(VT);
28561 SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef));
28562 SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef));
28563
28564 SDValue BLo, BHi;
28565 if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
28566 // If the RHS is a constant, manually unpackl/unpackh.
28567 SmallVector<SDValue, 16> LoOps, HiOps;
28568 for (unsigned i = 0; i != NumElts; i += 16) {
28569 for (unsigned j = 0; j != 8; ++j) {
28570 LoOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j), dl,
28571 MVT::i16));
28572 HiOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j + 8), dl,
28573 MVT::i16));
28574 }
28575 }
28576
28577 BLo = DAG.getBuildVector(ExVT, dl, LoOps);
28578 BHi = DAG.getBuildVector(ExVT, dl, HiOps);
28579 } else {
28580 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Undef));
28581 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Undef));
28582 }
28583
28584 // Multiply, mask the lower 8bits of the lo/hi results and pack.
28585 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
28586 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
28587 return getPack(DAG, Subtarget, dl, VT, RLo, RHi);
28588 }
28589
28590 // Lower v4i32 mul as 2x shuffle, 2x pmuludq, 2x shuffle.
28591 if (VT == MVT::v4i32) {
28592 assert(Subtarget.hasSSE2() && !Subtarget.hasSSE41() &&
28593 "Should not custom lower when pmulld is available!");
28594
28595 // Extract the odd parts.
28596 static const int UnpackMask[] = { 1, -1, 3, -1 };
28597 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
28598 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
28599
28600 // Multiply the even parts.
28601 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
28602 DAG.getBitcast(MVT::v2i64, A),
28603 DAG.getBitcast(MVT::v2i64, B));
28604 // Now multiply odd parts.
28605 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
28606 DAG.getBitcast(MVT::v2i64, Aodds),
28607 DAG.getBitcast(MVT::v2i64, Bodds));
28608
28609 Evens = DAG.getBitcast(VT, Evens);
28610 Odds = DAG.getBitcast(VT, Odds);
28611
28612 // Merge the two vectors back together with a shuffle. This expands into 2
28613 // shuffles.
28614 static const int ShufMask[] = { 0, 4, 2, 6 };
28615 return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
28616 }
28617
28618 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
28619 "Only know how to lower V2I64/V4I64/V8I64 multiply");
28620 assert(!Subtarget.hasDQI() && "DQI should use MULLQ");
28621
28622 // Ahi = psrlqi(a, 32);
28623 // Bhi = psrlqi(b, 32);
28624 //
28625 // AloBlo = pmuludq(a, b);
28626 // AloBhi = pmuludq(a, Bhi);
28627 // AhiBlo = pmuludq(Ahi, b);
28628 //
28629 // Hi = psllqi(AloBhi + AhiBlo, 32);
28630 // return AloBlo + Hi;
28631 KnownBits AKnown = DAG.computeKnownBits(A);
28632 KnownBits BKnown = DAG.computeKnownBits(B);
28633
28634 APInt LowerBitsMask = APInt::getLowBitsSet(64, 32);
28635 bool ALoIsZero = LowerBitsMask.isSubsetOf(AKnown.Zero);
28636 bool BLoIsZero = LowerBitsMask.isSubsetOf(BKnown.Zero);
28637
28638 APInt UpperBitsMask = APInt::getHighBitsSet(64, 32);
28639 bool AHiIsZero = UpperBitsMask.isSubsetOf(AKnown.Zero);
28640 bool BHiIsZero = UpperBitsMask.isSubsetOf(BKnown.Zero);
28641
28642 SDValue Zero = DAG.getConstant(0, dl, VT);
28643
28644 // Only multiply lo/hi halves that aren't known to be zero.
28645 SDValue AloBlo = Zero;
28646 if (!ALoIsZero && !BLoIsZero)
28647 AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
28648
28649 SDValue AloBhi = Zero;
28650 if (!ALoIsZero && !BHiIsZero) {
28651 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
28652 AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
28653 }
28654
28655 SDValue AhiBlo = Zero;
28656 if (!AHiIsZero && !BLoIsZero) {
28657 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
28658 AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
28659 }
28660
28661 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo);
28662 Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG);
28663
28664 return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi);
28665}
28666
28667static SDValue LowervXi8MulWithUNPCK(SDValue A, SDValue B, const SDLoc &dl,
28668 MVT VT, bool IsSigned,
28669 const X86Subtarget &Subtarget,
28670 SelectionDAG &DAG,
28671 SDValue *Low = nullptr) {
28672 unsigned NumElts = VT.getVectorNumElements();
28673
28674 // For vXi8 we will unpack the low and high half of each 128 bit lane to widen
28675 // to a vXi16 type. Do the multiplies, shift the results and pack the half
28676 // lane results back together.
28677
28678 // We'll take different approaches for signed and unsigned.
28679 // For unsigned we'll use punpcklbw/punpckhbw to put zero extend the bytes
28680 // and use pmullw to calculate the full 16-bit product.
28681 // For signed we'll use punpcklbw/punpckbw to extend the bytes to words and
28682 // shift them left into the upper byte of each word. This allows us to use
28683 // pmulhw to calculate the full 16-bit product. This trick means we don't
28684 // need to sign extend the bytes to use pmullw.
28685
28686 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
28687 SDValue Zero = DAG.getConstant(0, dl, VT);
28688
28689 SDValue ALo, AHi;
28690 if (IsSigned) {
28691 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, A));
28692 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, A));
28693 } else {
28694 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Zero));
28695 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Zero));
28696 }
28697
28698 SDValue BLo, BHi;
28699 if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
28700 // If the RHS is a constant, manually unpackl/unpackh and extend.
28701 SmallVector<SDValue, 16> LoOps, HiOps;
28702 for (unsigned i = 0; i != NumElts; i += 16) {
28703 for (unsigned j = 0; j != 8; ++j) {
28704 SDValue LoOp = B.getOperand(i + j);
28705 SDValue HiOp = B.getOperand(i + j + 8);
28706
28707 if (IsSigned) {
28708 LoOp = DAG.getAnyExtOrTrunc(LoOp, dl, MVT::i16);
28709 HiOp = DAG.getAnyExtOrTrunc(HiOp, dl, MVT::i16);
28710 LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp,
28711 DAG.getConstant(8, dl, MVT::i16));
28712 HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp,
28713 DAG.getConstant(8, dl, MVT::i16));
28714 } else {
28715 LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16);
28716 HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16);
28717 }
28718
28719 LoOps.push_back(LoOp);
28720 HiOps.push_back(HiOp);
28721 }
28722 }
28723
28724 BLo = DAG.getBuildVector(ExVT, dl, LoOps);
28725 BHi = DAG.getBuildVector(ExVT, dl, HiOps);
28726 } else if (IsSigned) {
28727 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B));
28728 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B));
28729 } else {
28730 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero));
28731 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero));
28732 }
28733
28734 // Multiply, lshr the upper 8bits to the lower 8bits of the lo/hi results and
28735 // pack back to vXi8.
28736 unsigned MulOpc = IsSigned ? ISD::MULHS : ISD::MUL;
28737 SDValue RLo = DAG.getNode(MulOpc, dl, ExVT, ALo, BLo);
28738 SDValue RHi = DAG.getNode(MulOpc, dl, ExVT, AHi, BHi);
28739
28740 if (Low)
28741 *Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi);
28742
28743 return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true);
28744}
28745
28746static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
28747 SelectionDAG &DAG) {
28748 SDLoc dl(Op);
28749 MVT VT = Op.getSimpleValueType();
28750 bool IsSigned = Op->getOpcode() == ISD::MULHS;
28751 unsigned NumElts = VT.getVectorNumElements();
28752 SDValue A = Op.getOperand(0);
28753 SDValue B = Op.getOperand(1);
28754
28755 // Decompose 256-bit ops into 128-bit ops.
28756 if (VT.is256BitVector() && !Subtarget.hasInt256())
28757 return splitVectorIntBinary(Op, DAG, dl);
28758
28759 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())
28760 return splitVectorIntBinary(Op, DAG, dl);
28761
28762 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) {
28763 assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) ||
28764 (VT == MVT::v8i32 && Subtarget.hasInt256()) ||
28765 (VT == MVT::v16i32 && Subtarget.hasAVX512()));
28766
28767 // PMULxD operations multiply each even value (starting at 0) of LHS with
28768 // the related value of RHS and produce a widen result.
28769 // E.g., PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
28770 // => <2 x i64> <ae|cg>
28771 //
28772 // In other word, to have all the results, we need to perform two PMULxD:
28773 // 1. one with the even values.
28774 // 2. one with the odd values.
28775 // To achieve #2, with need to place the odd values at an even position.
28776 //
28777 // Place the odd value at an even position (basically, shift all values 1
28778 // step to the left):
28779 const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1,
28780 9, -1, 11, -1, 13, -1, 15, -1};
28781 // <a|b|c|d> => <b|undef|d|undef>
28782 SDValue Odd0 =
28783 DAG.getVectorShuffle(VT, dl, A, A, ArrayRef(&Mask[0], NumElts));
28784 // <e|f|g|h> => <f|undef|h|undef>
28785 SDValue Odd1 =
28786 DAG.getVectorShuffle(VT, dl, B, B, ArrayRef(&Mask[0], NumElts));
28787
28788 // Emit two multiplies, one for the lower 2 ints and one for the higher 2
28789 // ints.
28790 MVT MulVT = MVT::getVectorVT(MVT::i64, NumElts / 2);
28791 unsigned Opcode =
28792 (IsSigned && Subtarget.hasSSE41()) ? X86ISD::PMULDQ : X86ISD::PMULUDQ;
28793 // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
28794 // => <2 x i64> <ae|cg>
28795 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
28796 DAG.getBitcast(MulVT, A),
28797 DAG.getBitcast(MulVT, B)));
28798 // PMULUDQ <4 x i32> <b|undef|d|undef>, <4 x i32> <f|undef|h|undef>
28799 // => <2 x i64> <bf|dh>
28800 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
28801 DAG.getBitcast(MulVT, Odd0),
28802 DAG.getBitcast(MulVT, Odd1)));
28803
28804 // Shuffle it back into the right order.
28805 SmallVector<int, 16> ShufMask(NumElts);
28806 for (int i = 0; i != (int)NumElts; ++i)
28807 ShufMask[i] = (i / 2) * 2 + ((i % 2) * NumElts) + 1;
28808
28809 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
28810
28811 // If we have a signed multiply but no PMULDQ fix up the result of an
28812 // unsigned multiply.
28813 if (IsSigned && !Subtarget.hasSSE41()) {
28814 SDValue Zero = DAG.getConstant(0, dl, VT);
28815 SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
28816 DAG.getSetCC(dl, VT, Zero, A, ISD::SETGT), B);
28817 SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
28818 DAG.getSetCC(dl, VT, Zero, B, ISD::SETGT), A);
28819
28820 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
28821 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup);
28822 }
28823
28824 return Res;
28825 }
28826
28827 // Only i8 vectors should need custom lowering after this.
28828 assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||
28829 (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
28830 "Unsupported vector type");
28831
28832 // Lower v16i8/v32i8 as extension to v8i16/v16i16 vector pairs, multiply,
28833 // logical shift down the upper half and pack back to i8.
28834
28835 // With SSE41 we can use sign/zero extend, but for pre-SSE41 we unpack
28836 // and then ashr/lshr the upper bits down to the lower bits before multiply.
28837
28838 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
28839 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
28840 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
28841 unsigned ExAVX = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
28842 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
28843 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
28844 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
28845 Mul = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
28846 return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
28847 }
28848
28849 return LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG);
28850}
28851
28852// Custom lowering for SMULO/UMULO.
28853static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget,
28854 SelectionDAG &DAG) {
28855 MVT VT = Op.getSimpleValueType();
28856
28857 // Scalars defer to LowerXALUO.
28858 if (!VT.isVector())
28859 return LowerXALUO(Op, DAG);
28860
28861 SDLoc dl(Op);
28862 bool IsSigned = Op->getOpcode() == ISD::SMULO;
28863 SDValue A = Op.getOperand(0);
28864 SDValue B = Op.getOperand(1);
28865 EVT OvfVT = Op->getValueType(1);
28866
28867 if ((VT == MVT::v32i8 && !Subtarget.hasInt256()) ||
28868 (VT == MVT::v64i8 && !Subtarget.hasBWI())) {
28869 // Extract the LHS Lo/Hi vectors
28870 SDValue LHSLo, LHSHi;
28871 std::tie(LHSLo, LHSHi) = splitVector(A, DAG, dl);
28872
28873 // Extract the RHS Lo/Hi vectors
28874 SDValue RHSLo, RHSHi;
28875 std::tie(RHSLo, RHSHi) = splitVector(B, DAG, dl);
28876
28877 EVT LoOvfVT, HiOvfVT;
28878 std::tie(LoOvfVT, HiOvfVT) = DAG.GetSplitDestVTs(OvfVT);
28879 SDVTList LoVTs = DAG.getVTList(LHSLo.getValueType(), LoOvfVT);
28880 SDVTList HiVTs = DAG.getVTList(LHSHi.getValueType(), HiOvfVT);
28881
28882 // Issue the split operations.
28883 SDValue Lo = DAG.getNode(Op.getOpcode(), dl, LoVTs, LHSLo, RHSLo);
28884 SDValue Hi = DAG.getNode(Op.getOpcode(), dl, HiVTs, LHSHi, RHSHi);
28885
28886 // Join the separate data results and the overflow results.
28887 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
28888 SDValue Ovf = DAG.getNode(ISD::CONCAT_VECTORS, dl, OvfVT, Lo.getValue(1),
28889 Hi.getValue(1));
28890
28891 return DAG.getMergeValues({Res, Ovf}, dl);
28892 }
28893
28894 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28895 EVT SetccVT =
28896 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
28897
28898 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
28899 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
28900 unsigned NumElts = VT.getVectorNumElements();
28901 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
28902 unsigned ExAVX = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
28903 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
28904 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
28905 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
28906
28907 SDValue Low = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
28908
28909 SDValue Ovf;
28910 if (IsSigned) {
28911 SDValue High, LowSign;
28912 if (OvfVT.getVectorElementType() == MVT::i1 &&
28913 (Subtarget.hasBWI() || Subtarget.canExtendTo512DQ())) {
28914 // Rather the truncating try to do the compare on vXi16 or vXi32.
28915 // Shift the high down filling with sign bits.
28916 High = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Mul, 8, DAG);
28917 // Fill all 16 bits with the sign bit from the low.
28918 LowSign =
28919 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExVT, Mul, 8, DAG);
28920 LowSign = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, LowSign,
28921 15, DAG);
28922 SetccVT = OvfVT;
28923 if (!Subtarget.hasBWI()) {
28924 // We can't do a vXi16 compare so sign extend to v16i32.
28925 High = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, High);
28926 LowSign = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, LowSign);
28927 }
28928 } else {
28929 // Otherwise do the compare at vXi8.
28930 High = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
28931 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High);
28932 LowSign =
28933 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT));
28934 }
28935
28936 Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE);
28937 } else {
28938 SDValue High =
28939 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
28940 if (OvfVT.getVectorElementType() == MVT::i1 &&
28941 (Subtarget.hasBWI() || Subtarget.canExtendTo512DQ())) {
28942 // Rather the truncating try to do the compare on vXi16 or vXi32.
28943 SetccVT = OvfVT;
28944 if (!Subtarget.hasBWI()) {
28945 // We can't do a vXi16 compare so sign extend to v16i32.
28946 High = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v16i32, High);
28947 }
28948 } else {
28949 // Otherwise do the compare at vXi8.
28950 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High);
28951 }
28952
28953 Ovf =
28954 DAG.getSetCC(dl, SetccVT, High,
28955 DAG.getConstant(0, dl, High.getValueType()), ISD::SETNE);
28956 }
28957
28958 Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT);
28959
28960 return DAG.getMergeValues({Low, Ovf}, dl);
28961 }
28962
28963 SDValue Low;
28964 SDValue High =
28965 LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG, &Low);
28966
28967 SDValue Ovf;
28968 if (IsSigned) {
28969 // SMULO overflows if the high bits don't match the sign of the low.
28970 SDValue LowSign =
28971 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT));
28972 Ovf = DAG.getSetCC(dl, SetccVT, LowSign, High, ISD::SETNE);
28973 } else {
28974 // UMULO overflows if the high bits are non-zero.
28975 Ovf =
28976 DAG.getSetCC(dl, SetccVT, High, DAG.getConstant(0, dl, VT), ISD::SETNE);
28977 }
28978
28979 Ovf = DAG.getSExtOrTrunc(Ovf, dl, OvfVT);
28980
28981 return DAG.getMergeValues({Low, Ovf}, dl);
28982}
28983
28984SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
28985 assert(Subtarget.isTargetWin64() && "Unexpected target");
28986 EVT VT = Op.getValueType();
28987 assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
28988 "Unexpected return type for lowering");
28989
28990 if (isa<ConstantSDNode>(Op->getOperand(1))) {
28991 SmallVector<SDValue> Result;
28992 if (expandDIVREMByConstant(Op.getNode(), Result, MVT::i64, DAG))
28993 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(Op), VT, Result[0], Result[1]);
28994 }
28995
28996 RTLIB::Libcall LC;
28997 bool isSigned;
28998 switch (Op->getOpcode()) {
28999 // clang-format off
29000 default: llvm_unreachable("Unexpected request for libcall!");
29001 case ISD::SDIV: isSigned = true; LC = RTLIB::SDIV_I128; break;
29002 case ISD::UDIV: isSigned = false; LC = RTLIB::UDIV_I128; break;
29003 case ISD::SREM: isSigned = true; LC = RTLIB::SREM_I128; break;
29004 case ISD::UREM: isSigned = false; LC = RTLIB::UREM_I128; break;
29005 // clang-format on
29006 }
29007
29008 SDLoc dl(Op);
29009 SDValue InChain = DAG.getEntryNode();
29010
29011 TargetLowering::ArgListTy Args;
29012 TargetLowering::ArgListEntry Entry;
29013 for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i) {
29014 EVT ArgVT = Op->getOperand(i).getValueType();
29015 assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
29016 "Unexpected argument type for lowering");
29017 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
29018 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
29019 MachinePointerInfo MPI =
29020 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
29021 Entry.Node = StackPtr;
29022 InChain =
29023 DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16));
29024 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
29025 Entry.Ty = PointerType::get(ArgTy,0);
29026 Entry.IsSExt = false;
29027 Entry.IsZExt = false;
29028 Args.push_back(Entry);
29029 }
29030
29031 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
29032 getPointerTy(DAG.getDataLayout()));
29033
29034 TargetLowering::CallLoweringInfo CLI(DAG);
29035 CLI.setDebugLoc(dl)
29036 .setChain(InChain)
29037 .setLibCallee(
29038 getLibcallCallingConv(LC),
29039 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee,
29040 std::move(Args))
29041 .setInRegister()
29042 .setSExtResult(isSigned)
29043 .setZExtResult(!isSigned);
29044
29045 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
29046 return DAG.getBitcast(VT, CallInfo.first);
29047}
29048
29049SDValue X86TargetLowering::LowerWin64_FP_TO_INT128(SDValue Op,
29050 SelectionDAG &DAG,
29051 SDValue &Chain) const {
29052 assert(Subtarget.isTargetWin64() && "Unexpected target");
29053 EVT VT = Op.getValueType();
29054 bool IsStrict = Op->isStrictFPOpcode();
29055
29056 SDValue Arg = Op.getOperand(IsStrict ? 1 : 0);
29057 EVT ArgVT = Arg.getValueType();
29058
29059 assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
29060 "Unexpected return type for lowering");
29061
29062 RTLIB::Libcall LC;
29063 if (Op->getOpcode() == ISD::FP_TO_SINT ||
29064 Op->getOpcode() == ISD::STRICT_FP_TO_SINT)
29065 LC = RTLIB::getFPTOSINT(ArgVT, VT);
29066 else
29067 LC = RTLIB::getFPTOUINT(ArgVT, VT);
29068 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
29069
29070 SDLoc dl(Op);
29071 MakeLibCallOptions CallOptions;
29072 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
29073
29074 SDValue Result;
29075 // Expect the i128 argument returned as a v2i64 in xmm0, cast back to the
29076 // expected VT (i128).
29077 std::tie(Result, Chain) =
29078 makeLibCall(DAG, LC, MVT::v2i64, Arg, CallOptions, dl, Chain);
29079 Result = DAG.getBitcast(VT, Result);
29080 return Result;
29081}
29082
29083SDValue X86TargetLowering::LowerWin64_INT128_TO_FP(SDValue Op,
29084 SelectionDAG &DAG) const {
29085 assert(Subtarget.isTargetWin64() && "Unexpected target");
29086 EVT VT = Op.getValueType();
29087 bool IsStrict = Op->isStrictFPOpcode();
29088
29089 SDValue Arg = Op.getOperand(IsStrict ? 1 : 0);
29090 EVT ArgVT = Arg.getValueType();
29091
29092 assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
29093 "Unexpected argument type for lowering");
29094
29095 RTLIB::Libcall LC;
29096 if (Op->getOpcode() == ISD::SINT_TO_FP ||
29097 Op->getOpcode() == ISD::STRICT_SINT_TO_FP)
29098 LC = RTLIB::getSINTTOFP(ArgVT, VT);
29099 else
29100 LC = RTLIB::getUINTTOFP(ArgVT, VT);
29101 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
29102
29103 SDLoc dl(Op);
29104 MakeLibCallOptions CallOptions;
29105 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
29106
29107 // Pass the i128 argument as an indirect argument on the stack.
29108 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
29109 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
29110 MachinePointerInfo MPI =
29111 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
29112 Chain = DAG.getStore(Chain, dl, Arg, StackPtr, MPI, Align(16));
29113
29114 SDValue Result;
29115 std::tie(Result, Chain) =
29116 makeLibCall(DAG, LC, VT, StackPtr, CallOptions, dl, Chain);
29117 return IsStrict ? DAG.getMergeValues({Result, Chain}, dl) : Result;
29118}
29119
29120// Generate a GFNI gf2p8affine bitmask for vXi8 bitreverse/shift/rotate.
29121uint64_t getGFNICtrlImm(unsigned Opcode, unsigned Amt = 0) {
29122 assert((Amt < 8) && "Shift/Rotation amount out of range");
29123 switch (Opcode) {
29124 case ISD::BITREVERSE:
29125 return 0x8040201008040201ULL;
29126 case ISD::SHL:
29127 return ((0x0102040810204080ULL >> (Amt)) &
29128 (0x0101010101010101ULL * (0xFF >> (Amt))));
29129 case ISD::SRL:
29130 return ((0x0102040810204080ULL << (Amt)) &
29131 (0x0101010101010101ULL * ((0xFF << (Amt)) & 0xFF)));
29132 case ISD::SRA:
29133 return (getGFNICtrlImm(ISD::SRL, Amt) |
29134 (0x8080808080808080ULL >> (64 - (8 * Amt))));
29135 case ISD::ROTL:
29136 return getGFNICtrlImm(ISD::SRL, 8 - Amt) | getGFNICtrlImm(ISD::SHL, Amt);
29137 case ISD::ROTR:
29138 return getGFNICtrlImm(ISD::SHL, 8 - Amt) | getGFNICtrlImm(ISD::SRL, Amt);
29139 }
29140 llvm_unreachable("Unsupported GFNI opcode");
29141}
29142
29143// Generate a GFNI gf2p8affine bitmask for vXi8 bitreverse/shift/rotate.
29144SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT VT,
29145 unsigned Amt = 0) {
29146 assert(VT.getVectorElementType() == MVT::i8 &&
29147 (VT.getSizeInBits() % 64) == 0 && "Illegal GFNI control type");
29148 uint64_t Imm = getGFNICtrlImm(Opcode, Amt);
29149 SmallVector<SDValue> MaskBits;
29150 for (unsigned I = 0, E = VT.getSizeInBits(); I != E; I += 8) {
29151 uint64_t Bits = (Imm >> (I % 64)) & 255;
29152 MaskBits.push_back(DAG.getConstant(Bits, DL, MVT::i8));
29153 }
29154 return DAG.getBuildVector(VT, DL, MaskBits);
29155}
29156
29157// Return true if the required (according to Opcode) shift-imm form is natively
29158// supported by the Subtarget
29159static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget,
29160 unsigned Opcode) {
29161 assert((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
29162 "Unexpected shift opcode");
29163
29164 if (!VT.isSimple())
29165 return false;
29166
29167 if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))
29168 return false;
29169
29170 if (VT.getScalarSizeInBits() < 16)
29171 return false;
29172
29173 if (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
29174 (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI()))
29175 return true;
29176
29177 bool LShift = (VT.is128BitVector() && Subtarget.hasSSE2()) ||
29178 (VT.is256BitVector() && Subtarget.hasInt256());
29179
29180 bool AShift = LShift && (Subtarget.hasAVX512() ||
29181 (VT != MVT::v2i64 && VT != MVT::v4i64));
29182 return (Opcode == ISD::SRA) ? AShift : LShift;
29183}
29184
29185// The shift amount is a variable, but it is the same for all vector lanes.
29186// These instructions are defined together with shift-immediate.
29187static
29188bool supportedVectorShiftWithBaseAmnt(EVT VT, const X86Subtarget &Subtarget,
29189 unsigned Opcode) {
29190 return supportedVectorShiftWithImm(VT, Subtarget, Opcode);
29191}
29192
29193// Return true if the required (according to Opcode) variable-shift form is
29194// natively supported by the Subtarget
29195static bool supportedVectorVarShift(EVT VT, const X86Subtarget &Subtarget,
29196 unsigned Opcode) {
29197 assert((Opcode == ISD::SHL || Opcode == ISD::SRA || Opcode == ISD::SRL) &&
29198 "Unexpected shift opcode");
29199
29200 if (!VT.isSimple())
29201 return false;
29202
29203 if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))
29204 return false;
29205
29206 if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16)
29207 return false;
29208
29209 // vXi16 supported only on AVX-512, BWI
29210 if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI())
29211 return false;
29212
29213 if (Subtarget.hasAVX512() &&
29214 (Subtarget.useAVX512Regs() || !VT.is512BitVector()))
29215 return true;
29216
29217 bool LShift = VT.is128BitVector() || VT.is256BitVector();
29218 bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64;
29219 return (Opcode == ISD::SRA) ? AShift : LShift;
29220}
29221
29222static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG,
29223 const X86Subtarget &Subtarget) {
29224 MVT VT = Op.getSimpleValueType();
29225 SDLoc dl(Op);
29226 SDValue R = Op.getOperand(0);
29227 SDValue Amt = Op.getOperand(1);
29228 unsigned X86Opc = getTargetVShiftUniformOpcode(Op.getOpcode(), false);
29229 unsigned EltSizeInBits = VT.getScalarSizeInBits();
29230
29231 auto ArithmeticShiftRight64 = [&](uint64_t ShiftAmt) {
29232 assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type");
29233 MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);
29234 SDValue Ex = DAG.getBitcast(ExVT, R);
29235
29236 // ashr(R, 63) === cmp_slt(R, 0)
29237 if (ShiftAmt == 63 && Subtarget.hasSSE42()) {
29238 assert((VT != MVT::v4i64 || Subtarget.hasInt256()) &&
29239 "Unsupported PCMPGT op");
29240 return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R);
29241 }
29242
29243 if (ShiftAmt >= 32) {
29244 // Splat sign to upper i32 dst, and SRA upper i32 src to lower i32.
29245 SDValue Upper =
29246 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex, 31, DAG);
29247 SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
29248 ShiftAmt - 32, DAG);
29249 if (VT == MVT::v2i64)
29250 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {5, 1, 7, 3});
29251 if (VT == MVT::v4i64)
29252 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
29253 {9, 1, 11, 3, 13, 5, 15, 7});
29254 } else {
29255 // SRA upper i32, SRL whole i64 and select lower i32.
29256 SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
29257 ShiftAmt, DAG);
29258 SDValue Lower =
29259 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG);
29260 Lower = DAG.getBitcast(ExVT, Lower);
29261 if (VT == MVT::v2i64)
29262 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {4, 1, 6, 3});
29263 if (VT == MVT::v4i64)
29264 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
29265 {8, 1, 10, 3, 12, 5, 14, 7});
29266 }
29267 return DAG.getBitcast(VT, Ex);
29268 };
29269
29270 // Optimize shl/srl/sra with constant shift amount.
29271 APInt APIntShiftAmt;
29272 if (!X86::isConstantSplat(Amt, APIntShiftAmt))
29273 return SDValue();
29274
29275 // If the shift amount is out of range, return undef.
29276 if (APIntShiftAmt.uge(EltSizeInBits))
29277 return DAG.getUNDEF(VT);
29278
29279 uint64_t ShiftAmt = APIntShiftAmt.getZExtValue();
29280
29281 if (supportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode())) {
29282 // Hardware support for vector shifts is sparse which makes us scalarize the
29283 // vector operations in many cases. Also, on sandybridge ADD is faster than
29284 // shl: (shl V, 1) -> (add (freeze V), (freeze V))
29285 if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1) {
29286 // R may be undef at run-time, but (shl R, 1) must be an even number (LSB
29287 // must be 0). (add undef, undef) however can be any value. To make this
29288 // safe, we must freeze R to ensure that register allocation uses the same
29289 // register for an undefined value. This ensures that the result will
29290 // still be even and preserves the original semantics.
29291 R = DAG.getFreeze(R);
29292 return DAG.getNode(ISD::ADD, dl, VT, R, R);
29293 }
29294
29295 return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG);
29296 }
29297
29298 // i64 SRA needs to be performed as partial shifts.
29299 if (((!Subtarget.hasXOP() && VT == MVT::v2i64) ||
29300 (Subtarget.hasInt256() && VT == MVT::v4i64)) &&
29301 Op.getOpcode() == ISD::SRA)
29302 return ArithmeticShiftRight64(ShiftAmt);
29303
29304 // If we're logical shifting an all-signbits value then we can just perform as
29305 // a mask.
29306 if ((Op.getOpcode() == ISD::SHL || Op.getOpcode() == ISD::SRL) &&
29307 DAG.ComputeNumSignBits(R) == EltSizeInBits) {
29308 SDValue Mask = DAG.getAllOnesConstant(dl, VT);
29309 Mask = DAG.getNode(Op.getOpcode(), dl, VT, Mask, Amt);
29310 return DAG.getNode(ISD::AND, dl, VT, R, Mask);
29311 }
29312
29313 if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) ||
29314 (Subtarget.hasBWI() && VT == MVT::v64i8)) {
29315 unsigned NumElts = VT.getVectorNumElements();
29316 MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
29317
29318 // Simple i8 add case
29319 if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1) {
29320 // R may be undef at run-time, but (shl R, 1) must be an even number (LSB
29321 // must be 0). (add undef, undef) however can be any value. To make this
29322 // safe, we must freeze R to ensure that register allocation uses the same
29323 // register for an undefined value. This ensures that the result will
29324 // still be even and preserves the original semantics.
29325 R = DAG.getFreeze(R);
29326 return DAG.getNode(ISD::ADD, dl, VT, R, R);
29327 }
29328
29329 // ashr(R, 7) === cmp_slt(R, 0)
29330 if (Op.getOpcode() == ISD::SRA && ShiftAmt == 7) {
29331 SDValue Zeros = DAG.getConstant(0, dl, VT);
29332 if (VT.is512BitVector()) {
29333 assert(VT == MVT::v64i8 && "Unexpected element type!");
29334 SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT);
29335 return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP);
29336 }
29337 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
29338 }
29339
29340 // XOP can shift v16i8 directly instead of as shift v8i16 + mask.
29341 if (VT == MVT::v16i8 && Subtarget.hasXOP())
29342 return SDValue();
29343
29344 if (Subtarget.hasGFNI()) {
29345 SDValue Mask = getGFNICtrlMask(Op.getOpcode(), DAG, dl, VT, ShiftAmt);
29346 return DAG.getNode(X86ISD::GF2P8AFFINEQB, dl, VT, R, Mask,
29347 DAG.getTargetConstant(0, dl, MVT::i8));
29348 }
29349
29350 if (Op.getOpcode() == ISD::SHL) {
29351 // Make a large shift.
29352 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT, R,
29353 ShiftAmt, DAG);
29354 SHL = DAG.getBitcast(VT, SHL);
29355 // Zero out the rightmost bits.
29356 APInt Mask = APInt::getHighBitsSet(8, 8 - ShiftAmt);
29357 return DAG.getNode(ISD::AND, dl, VT, SHL, DAG.getConstant(Mask, dl, VT));
29358 }
29359 if (Op.getOpcode() == ISD::SRL) {
29360 // Make a large shift.
29361 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT, R,
29362 ShiftAmt, DAG);
29363 SRL = DAG.getBitcast(VT, SRL);
29364 // Zero out the leftmost bits.
29365 APInt Mask = APInt::getLowBitsSet(8, 8 - ShiftAmt);
29366 return DAG.getNode(ISD::AND, dl, VT, SRL, DAG.getConstant(Mask, dl, VT));
29367 }
29368 if (Op.getOpcode() == ISD::SRA) {
29369 // ashr(R, Amt) === sub(xor(lshr(R, Amt), Mask), Mask)
29370 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
29371
29372 SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT);
29373 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
29374 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
29375 return Res;
29376 }
29377 llvm_unreachable("Unknown shift opcode.");
29378 }
29379
29380 return SDValue();
29381}
29382
29383static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG,
29384 const X86Subtarget &Subtarget) {
29385 MVT VT = Op.getSimpleValueType();
29386 SDLoc dl(Op);
29387 SDValue R = Op.getOperand(0);
29388 SDValue Amt = Op.getOperand(1);
29389 unsigned Opcode = Op.getOpcode();
29390 unsigned X86OpcI = getTargetVShiftUniformOpcode(Opcode, false);
29391
29392 int BaseShAmtIdx = -1;
29393 if (SDValue BaseShAmt = DAG.getSplatSourceVector(Amt, BaseShAmtIdx)) {
29394 if (supportedVectorShiftWithBaseAmnt(VT, Subtarget, Opcode))
29395 return getTargetVShiftNode(X86OpcI, dl, VT, R, BaseShAmt, BaseShAmtIdx,
29396 Subtarget, DAG);
29397
29398 // vXi8 shifts - shift as v8i16 + mask result.
29399 if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) ||
29400 (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) ||
29401 VT == MVT::v64i8) &&
29402 !Subtarget.hasXOP()) {
29403 unsigned NumElts = VT.getVectorNumElements();
29404 MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
29405 if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, Opcode)) {
29406 unsigned LogicalOp = (Opcode == ISD::SHL ? ISD::SHL : ISD::SRL);
29407 unsigned LogicalX86Op = getTargetVShiftUniformOpcode(LogicalOp, false);
29408
29409 // Create the mask using vXi16 shifts. For shift-rights we need to move
29410 // the upper byte down before splatting the vXi8 mask.
29411 SDValue BitMask = DAG.getAllOnesConstant(dl, ExtVT);
29412 BitMask = getTargetVShiftNode(LogicalX86Op, dl, ExtVT, BitMask,
29413 BaseShAmt, BaseShAmtIdx, Subtarget, DAG);
29414 if (Opcode != ISD::SHL)
29415 BitMask = getTargetVShiftByConstNode(LogicalX86Op, dl, ExtVT, BitMask,
29416 8, DAG);
29417 BitMask = DAG.getBitcast(VT, BitMask);
29418 BitMask = DAG.getVectorShuffle(VT, dl, BitMask, BitMask,
29419 SmallVector<int, 64>(NumElts, 0));
29420
29421 SDValue Res = getTargetVShiftNode(LogicalX86Op, dl, ExtVT,
29422 DAG.getBitcast(ExtVT, R), BaseShAmt,
29423 BaseShAmtIdx, Subtarget, DAG);
29424 Res = DAG.getBitcast(VT, Res);
29425 Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask);
29426
29427 if (Opcode == ISD::SRA) {
29428 // ashr(R, Amt) === sub(xor(lshr(R, Amt), SignMask), SignMask)
29429 // SignMask = lshr(SignBit, Amt) - safe to do this with PSRLW.
29430 SDValue SignMask = DAG.getConstant(0x8080, dl, ExtVT);
29431 SignMask =
29432 getTargetVShiftNode(LogicalX86Op, dl, ExtVT, SignMask, BaseShAmt,
29433 BaseShAmtIdx, Subtarget, DAG);
29434 SignMask = DAG.getBitcast(VT, SignMask);
29435 Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask);
29436 Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask);
29437 }
29438 return Res;
29439 }
29440 }
29441 }
29442
29443 return SDValue();
29444}
29445
29446// Convert a shift/rotate left amount to a multiplication scale factor.
29447static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
29448 const X86Subtarget &Subtarget,
29449 SelectionDAG &DAG) {
29450 MVT VT = Amt.getSimpleValueType();
29451 if (!(VT == MVT::v8i16 || VT == MVT::v4i32 ||
29452 (Subtarget.hasInt256() && VT == MVT::v16i16) ||
29453 (Subtarget.hasAVX512() && VT == MVT::v32i16) ||
29454 (!Subtarget.hasAVX512() && VT == MVT::v16i8) ||
29455 (Subtarget.hasInt256() && VT == MVT::v32i8) ||
29456 (Subtarget.hasBWI() && VT == MVT::v64i8)))
29457 return SDValue();
29458
29459 MVT SVT = VT.getVectorElementType();
29460 unsigned SVTBits = SVT.getSizeInBits();
29461 unsigned NumElems = VT.getVectorNumElements();
29462
29463 APInt UndefElts;
29464 SmallVector<APInt> EltBits;
29465 if (getTargetConstantBitsFromNode(Amt, SVTBits, UndefElts, EltBits)) {
29466 APInt One(SVTBits, 1);
29467 SmallVector<SDValue> Elts(NumElems, DAG.getUNDEF(SVT));
29468 for (unsigned I = 0; I != NumElems; ++I) {
29469 if (UndefElts[I] || EltBits[I].uge(SVTBits))
29470 continue;
29471 uint64_t ShAmt = EltBits[I].getZExtValue();
29472 Elts[I] = DAG.getConstant(One.shl(ShAmt), dl, SVT);
29473 }
29474 return DAG.getBuildVector(VT, dl, Elts);
29475 }
29476
29477 // If the target doesn't support variable shifts, use either FP conversion
29478 // or integer multiplication to avoid shifting each element individually.
29479 if (VT == MVT::v4i32) {
29480 Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT));
29481 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt,
29482 DAG.getConstant(0x3f800000U, dl, VT));
29483 Amt = DAG.getBitcast(MVT::v4f32, Amt);
29484 return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt);
29485 }
29486
29487 // AVX2 can more effectively perform this as a zext/trunc to/from v8i32.
29488 if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) {
29489 SDValue Z = DAG.getConstant(0, dl, VT);
29490 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));
29491 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));
29492 Lo = convertShiftLeftToScale(Lo, dl, Subtarget, DAG);
29493 Hi = convertShiftLeftToScale(Hi, dl, Subtarget, DAG);
29494 if (Subtarget.hasSSE41())
29495 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
29496 return getPack(DAG, Subtarget, dl, VT, Lo, Hi);
29497 }
29498
29499 return SDValue();
29500}
29501
29502static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
29503 SelectionDAG &DAG) {
29504 MVT VT = Op.getSimpleValueType();
29505 SDLoc dl(Op);
29506 SDValue R = Op.getOperand(0);
29507 SDValue Amt = Op.getOperand(1);
29508 unsigned EltSizeInBits = VT.getScalarSizeInBits();
29509 bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
29510
29511 unsigned Opc = Op.getOpcode();
29512 unsigned X86OpcV = getTargetVShiftUniformOpcode(Opc, true);
29513 unsigned X86OpcI = getTargetVShiftUniformOpcode(Opc, false);
29514
29515 assert(VT.isVector() && "Custom lowering only for vector shifts!");
29516 assert(Subtarget.hasSSE2() && "Only custom lower when we have SSE2!");
29517
29518 if (SDValue V = LowerShiftByScalarImmediate(Op, DAG, Subtarget))
29519 return V;
29520
29521 if (SDValue V = LowerShiftByScalarVariable(Op, DAG, Subtarget))
29522 return V;
29523
29524 if (supportedVectorVarShift(VT, Subtarget, Opc))
29525 return Op;
29526
29527 // i64 vector arithmetic shift can be emulated with the transform:
29528 // M = lshr(SIGN_MASK, Amt)
29529 // ashr(R, Amt) === sub(xor(lshr(R, Amt), M), M)
29530 if (((VT == MVT::v2i64 && !Subtarget.hasXOP()) ||
29531 (VT == MVT::v4i64 && Subtarget.hasInt256())) &&
29532 Opc == ISD::SRA) {
29533 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);
29534 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
29535 R = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
29536 R = DAG.getNode(ISD::XOR, dl, VT, R, M);
29537 R = DAG.getNode(ISD::SUB, dl, VT, R, M);
29538 return R;
29539 }
29540
29541 // XOP has 128-bit variable logical/arithmetic shifts.
29542 // +ve/-ve Amt = shift left/right.
29543 if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 ||
29544 VT == MVT::v8i16 || VT == MVT::v16i8)) {
29545 if (Opc == ISD::SRL || Opc == ISD::SRA)
29546 Amt = DAG.getNegative(Amt, dl, VT);
29547 if (Opc == ISD::SHL || Opc == ISD::SRL)
29548 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt);
29549 if (Opc == ISD::SRA)
29550 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt);
29551 }
29552
29553 // 2i64 vector logical shifts can efficiently avoid scalarization - do the
29554 // shifts per-lane and then shuffle the partial results back together.
29555 if (VT == MVT::v2i64 && Opc != ISD::SRA) {
29556 // Splat the shift amounts so the scalar shifts above will catch it.
29557 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});
29558 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});
29559 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
29560 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
29561 return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3});
29562 }
29563
29564 // If possible, lower this shift as a sequence of two shifts by
29565 // constant plus a BLENDing shuffle instead of scalarizing it.
29566 // Example:
29567 // (v4i32 (srl A, (build_vector < X, Y, Y, Y>)))
29568 //
29569 // Could be rewritten as:
29570 // (v4i32 (MOVSS (srl A, <Y,Y,Y,Y>), (srl A, <X,X,X,X>)))
29571 //
29572 // The advantage is that the two shifts from the example would be
29573 // lowered as X86ISD::VSRLI nodes in parallel before blending.
29574 if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 ||
29575 (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
29576 SDValue Amt1, Amt2;
29577 unsigned NumElts = VT.getVectorNumElements();
29578 SmallVector<int, 8> ShuffleMask;
29579 for (unsigned i = 0; i != NumElts; ++i) {
29580 SDValue A = Amt->getOperand(i);
29581 if (A.isUndef()) {
29582 ShuffleMask.push_back(SM_SentinelUndef);
29583 continue;
29584 }
29585 if (!Amt1 || Amt1 == A) {
29586 ShuffleMask.push_back(i);
29587 Amt1 = A;
29588 continue;
29589 }
29590 if (!Amt2 || Amt2 == A) {
29591 ShuffleMask.push_back(i + NumElts);
29592 Amt2 = A;
29593 continue;
29594 }
29595 break;
29596 }
29597
29598 // Only perform this blend if we can perform it without loading a mask.
29599 if (ShuffleMask.size() == NumElts && Amt1 && Amt2 &&
29600 (VT != MVT::v16i16 ||
29601 is128BitLaneRepeatedShuffleMask(VT, ShuffleMask)) &&
29602 (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL ||
29603 canWidenShuffleElements(ShuffleMask))) {
29604 auto *Cst1 = dyn_cast<ConstantSDNode>(Amt1);
29605 auto *Cst2 = dyn_cast<ConstantSDNode>(Amt2);
29606 if (Cst1 && Cst2 && Cst1->getAPIntValue().ult(EltSizeInBits) &&
29607 Cst2->getAPIntValue().ult(EltSizeInBits)) {
29608 SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
29609 Cst1->getZExtValue(), DAG);
29610 SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
29611 Cst2->getZExtValue(), DAG);
29612 return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask);
29613 }
29614 }
29615 }
29616
29617 // If possible, lower this packed shift into a vector multiply instead of
29618 // expanding it into a sequence of scalar shifts.
29619 // For v32i8 cases, it might be quicker to split/extend to vXi16 shifts.
29620 if (Opc == ISD::SHL && !(VT == MVT::v32i8 && (Subtarget.hasXOP() ||
29621 Subtarget.canExtendTo512BW())))
29622 if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG))
29623 return DAG.getNode(ISD::MUL, dl, VT, R, Scale);
29624
29625 // Constant ISD::SRL can be performed efficiently on vXi16 vectors as we
29626 // can replace with ISD::MULHU, creating scale factor from (NumEltBits - Amt).
29627 if (Opc == ISD::SRL && ConstantAmt &&
29628 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
29629 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
29630 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
29631 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
29632 SDValue Zero = DAG.getConstant(0, dl, VT);
29633 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);
29634 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);
29635 return DAG.getSelect(dl, VT, ZAmt, R, Res);
29636 }
29637 }
29638
29639 // Constant ISD::SRA can be performed efficiently on vXi16 vectors as we
29640 // can replace with ISD::MULHS, creating scale factor from (NumEltBits - Amt).
29641 // TODO: Special case handling for shift by 0/1, really we can afford either
29642 // of these cases in pre-SSE41/XOP/AVX512 but not both.
29643 if (Opc == ISD::SRA && ConstantAmt &&
29644 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) &&
29645 ((Subtarget.hasSSE41() && !Subtarget.hasXOP() &&
29646 !Subtarget.hasAVX512()) ||
29647 DAG.isKnownNeverZero(Amt))) {
29648 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
29649 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
29650 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
29651 SDValue Amt0 =
29652 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ);
29653 SDValue Amt1 =
29654 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ);
29655 SDValue Sra1 =
29656 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG);
29657 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);
29658 Res = DAG.getSelect(dl, VT, Amt0, R, Res);
29659 return DAG.getSelect(dl, VT, Amt1, Sra1, Res);
29660 }
29661 }
29662
29663 // v4i32 Non Uniform Shifts.
29664 // If the shift amount is constant we can shift each lane using the SSE2
29665 // immediate shifts, else we need to zero-extend each lane to the lower i64
29666 // and shift using the SSE2 variable shifts.
29667 // The separate results can then be blended together.
29668 if (VT == MVT::v4i32) {
29669 SDValue Amt0, Amt1, Amt2, Amt3;
29670 if (ConstantAmt) {
29671 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0});
29672 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1});
29673 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2});
29674 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3});
29675 } else {
29676 // The SSE2 shifts use the lower i64 as the same shift amount for
29677 // all lanes and the upper i64 is ignored. On AVX we're better off
29678 // just zero-extending, but for SSE just duplicating the top 16-bits is
29679 // cheaper and has the same effect for out of range values.
29680 if (Subtarget.hasAVX()) {
29681 SDValue Z = DAG.getConstant(0, dl, VT);
29682 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1});
29683 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1});
29684 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1});
29685 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1});
29686 } else {
29687 SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt);
29688 SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
29689 {4, 5, 6, 7, -1, -1, -1, -1});
29690 SDValue Msk02 = getV4X86ShuffleImm8ForMask({0, 1, 1, 1}, dl, DAG);
29691 SDValue Msk13 = getV4X86ShuffleImm8ForMask({2, 3, 3, 3}, dl, DAG);
29692 Amt0 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk02);
29693 Amt1 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk13);
29694 Amt2 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk02);
29695 Amt3 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk13);
29696 }
29697 }
29698
29699 unsigned ShOpc = ConstantAmt ? Opc : X86OpcV;
29700 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
29701 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
29702 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
29703 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
29704
29705 // Merge the shifted lane results optimally with/without PBLENDW.
29706 // TODO - ideally shuffle combining would handle this.
29707 if (Subtarget.hasSSE41()) {
29708 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
29709 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
29710 return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7});
29711 }
29712 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});
29713 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});
29714 return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7});
29715 }
29716
29717 // It's worth extending once and using the vXi16/vXi32 shifts for smaller
29718 // types, but without AVX512 the extra overheads to get from vXi8 to vXi32
29719 // make the existing SSE solution better.
29720 // NOTE: We honor prefered vector width before promoting to 512-bits.
29721 if ((Subtarget.hasInt256() && VT == MVT::v8i16) ||
29722 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) ||
29723 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) ||
29724 (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) ||
29725 (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) {
29726 assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) &&
29727 "Unexpected vector type");
29728 MVT EvtSVT = Subtarget.hasBWI() ? MVT::i16 : MVT::i32;
29729 MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements());
29730 unsigned ExtOpc = Opc == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
29731 R = DAG.getNode(ExtOpc, dl, ExtVT, R);
29732 Amt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Amt);
29733 return DAG.getNode(ISD::TRUNCATE, dl, VT,
29734 DAG.getNode(Opc, dl, ExtVT, R, Amt));
29735 }
29736
29737 // Constant ISD::SRA/SRL can be performed efficiently on vXi8 vectors as we
29738 // extend to vXi16 to perform a MUL scale effectively as a MUL_LOHI.
29739 if (ConstantAmt && (Opc == ISD::SRA || Opc == ISD::SRL) &&
29740 (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||
29741 (VT == MVT::v64i8 && Subtarget.hasBWI())) &&
29742 !Subtarget.hasXOP()) {
29743 int NumElts = VT.getVectorNumElements();
29744 SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);
29745
29746 // Extend constant shift amount to vXi16 (it doesn't matter if the type
29747 // isn't legal).
29748 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
29749 Amt = DAG.getZExtOrTrunc(Amt, dl, ExVT);
29750 Amt = DAG.getNode(ISD::SUB, dl, ExVT, DAG.getConstant(8, dl, ExVT), Amt);
29751 Amt = DAG.getNode(ISD::SHL, dl, ExVT, DAG.getConstant(1, dl, ExVT), Amt);
29752 assert(ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()) &&
29753 "Constant build vector expected");
29754
29755 if (VT == MVT::v16i8 && Subtarget.hasInt256()) {
29756 bool IsSigned = Opc == ISD::SRA;
29757 R = DAG.getExtOrTrunc(IsSigned, R, dl, ExVT);
29758 R = DAG.getNode(ISD::MUL, dl, ExVT, R, Amt);
29759 R = DAG.getNode(X86ISD::VSRLI, dl, ExVT, R, Cst8);
29760 return DAG.getZExtOrTrunc(R, dl, VT);
29761 }
29762
29763 SmallVector<SDValue, 16> LoAmt, HiAmt;
29764 for (int i = 0; i != NumElts; i += 16) {
29765 for (int j = 0; j != 8; ++j) {
29766 LoAmt.push_back(Amt.getOperand(i + j));
29767 HiAmt.push_back(Amt.getOperand(i + j + 8));
29768 }
29769 }
29770
29771 MVT VT16 = MVT::getVectorVT(MVT::i16, NumElts / 2);
29772 SDValue LoA = DAG.getBuildVector(VT16, dl, LoAmt);
29773 SDValue HiA = DAG.getBuildVector(VT16, dl, HiAmt);
29774
29775 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));
29776 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));
29777 LoR = DAG.getNode(X86OpcI, dl, VT16, LoR, Cst8);
29778 HiR = DAG.getNode(X86OpcI, dl, VT16, HiR, Cst8);
29779 LoR = DAG.getNode(ISD::MUL, dl, VT16, LoR, LoA);
29780 HiR = DAG.getNode(ISD::MUL, dl, VT16, HiR, HiA);
29781 LoR = DAG.getNode(X86ISD::VSRLI, dl, VT16, LoR, Cst8);
29782 HiR = DAG.getNode(X86ISD::VSRLI, dl, VT16, HiR, Cst8);
29783 return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR);
29784 }
29785
29786 if (VT == MVT::v16i8 ||
29787 (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) ||
29788 (VT == MVT::v64i8 && Subtarget.hasBWI())) {
29789 MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);
29790
29791 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
29792 if (VT.is512BitVector()) {
29793 // On AVX512BW targets we make use of the fact that VSELECT lowers
29794 // to a masked blend which selects bytes based just on the sign bit
29795 // extracted to a mask.
29796 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
29797 V0 = DAG.getBitcast(VT, V0);
29798 V1 = DAG.getBitcast(VT, V1);
29799 Sel = DAG.getBitcast(VT, Sel);
29800 Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel,
29801 ISD::SETGT);
29802 return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
29803 } else if (Subtarget.hasSSE41()) {
29804 // On SSE41 targets we can use PBLENDVB which selects bytes based just
29805 // on the sign bit.
29806 V0 = DAG.getBitcast(VT, V0);
29807 V1 = DAG.getBitcast(VT, V1);
29808 Sel = DAG.getBitcast(VT, Sel);
29809 return DAG.getBitcast(SelVT,
29810 DAG.getNode(X86ISD::BLENDV, dl, VT, Sel, V0, V1));
29811 }
29812 // On pre-SSE41 targets we test for the sign bit by comparing to
29813 // zero - a negative value will set all bits of the lanes to true
29814 // and VSELECT uses that in its OR(AND(V0,C),AND(V1,~C)) lowering.
29815 SDValue Z = DAG.getConstant(0, dl, SelVT);
29816 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
29817 return DAG.getSelect(dl, SelVT, C, V0, V1);
29818 };
29819
29820 // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
29821 // We can safely do this using i16 shifts as we're only interested in
29822 // the 3 lower bits of each byte.
29823 Amt = DAG.getBitcast(ExtVT, Amt);
29824 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExtVT, Amt, 5, DAG);
29825 Amt = DAG.getBitcast(VT, Amt);
29826
29827 if (Opc == ISD::SHL || Opc == ISD::SRL) {
29828 // r = VSELECT(r, shift(r, 4), a);
29829 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
29830 R = SignBitSelect(VT, Amt, M, R);
29831
29832 // a += a
29833 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29834
29835 // r = VSELECT(r, shift(r, 2), a);
29836 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT));
29837 R = SignBitSelect(VT, Amt, M, R);
29838
29839 // a += a
29840 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29841
29842 // return VSELECT(r, shift(r, 1), a);
29843 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT));
29844 R = SignBitSelect(VT, Amt, M, R);
29845 return R;
29846 }
29847
29848 if (Opc == ISD::SRA) {
29849 // For SRA we need to unpack each byte to the higher byte of a i16 vector
29850 // so we can correctly sign extend. We don't care what happens to the
29851 // lower byte.
29852 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
29853 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
29854 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
29855 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
29856 ALo = DAG.getBitcast(ExtVT, ALo);
29857 AHi = DAG.getBitcast(ExtVT, AHi);
29858 RLo = DAG.getBitcast(ExtVT, RLo);
29859 RHi = DAG.getBitcast(ExtVT, RHi);
29860
29861 // r = VSELECT(r, shift(r, 4), a);
29862 SDValue MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 4, DAG);
29863 SDValue MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 4, DAG);
29864 RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
29865 RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
29866
29867 // a += a
29868 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
29869 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
29870
29871 // r = VSELECT(r, shift(r, 2), a);
29872 MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 2, DAG);
29873 MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 2, DAG);
29874 RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
29875 RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
29876
29877 // a += a
29878 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
29879 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
29880
29881 // r = VSELECT(r, shift(r, 1), a);
29882 MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 1, DAG);
29883 MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 1, DAG);
29884 RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
29885 RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
29886
29887 // Logical shift the result back to the lower byte, leaving a zero upper
29888 // byte meaning that we can safely pack with PACKUSWB.
29889 RLo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RLo, 8, DAG);
29890 RHi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RHi, 8, DAG);
29891 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
29892 }
29893 }
29894
29895 if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) {
29896 MVT ExtVT = MVT::v8i32;
29897 SDValue Z = DAG.getConstant(0, dl, VT);
29898 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);
29899 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);
29900 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);
29901 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);
29902 ALo = DAG.getBitcast(ExtVT, ALo);
29903 AHi = DAG.getBitcast(ExtVT, AHi);
29904 RLo = DAG.getBitcast(ExtVT, RLo);
29905 RHi = DAG.getBitcast(ExtVT, RHi);
29906 SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo);
29907 SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi);
29908 Lo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Lo, 16, DAG);
29909 Hi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Hi, 16, DAG);
29910 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
29911 }
29912
29913 if (VT == MVT::v8i16) {
29914 // If we have a constant shift amount, the non-SSE41 path is best as
29915 // avoiding bitcasts make it easier to constant fold and reduce to PBLENDW.
29916 bool UseSSE41 = Subtarget.hasSSE41() &&
29917 !ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
29918
29919 auto SignBitSelect = [&](SDValue Sel, SDValue V0, SDValue V1) {
29920 // On SSE41 targets we can use PBLENDVB which selects bytes based just on
29921 // the sign bit.
29922 if (UseSSE41) {
29923 MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2);
29924 V0 = DAG.getBitcast(ExtVT, V0);
29925 V1 = DAG.getBitcast(ExtVT, V1);
29926 Sel = DAG.getBitcast(ExtVT, Sel);
29927 return DAG.getBitcast(
29928 VT, DAG.getNode(X86ISD::BLENDV, dl, ExtVT, Sel, V0, V1));
29929 }
29930 // On pre-SSE41 targets we splat the sign bit - a negative value will
29931 // set all bits of the lanes to true and VSELECT uses that in
29932 // its OR(AND(V0,C),AND(V1,~C)) lowering.
29933 SDValue C =
29934 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG);
29935 return DAG.getSelect(dl, VT, C, V0, V1);
29936 };
29937
29938 // Turn 'a' into a mask suitable for VSELECT: a = a << 12;
29939 if (UseSSE41) {
29940 // On SSE41 targets we need to replicate the shift mask in both
29941 // bytes for PBLENDVB.
29942 Amt = DAG.getNode(
29943 ISD::OR, dl, VT,
29944 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG),
29945 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG));
29946 } else {
29947 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG);
29948 }
29949
29950 // r = VSELECT(r, shift(r, 8), a);
29951 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);
29952 R = SignBitSelect(Amt, M, R);
29953
29954 // a += a
29955 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29956
29957 // r = VSELECT(r, shift(r, 4), a);
29958 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG);
29959 R = SignBitSelect(Amt, M, R);
29960
29961 // a += a
29962 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29963
29964 // r = VSELECT(r, shift(r, 2), a);
29965 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG);
29966 R = SignBitSelect(Amt, M, R);
29967
29968 // a += a
29969 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
29970
29971 // return VSELECT(r, shift(r, 1), a);
29972 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG);
29973 R = SignBitSelect(Amt, M, R);
29974 return R;
29975 }
29976
29977 // Decompose 256-bit shifts into 128-bit shifts.
29978 if (VT.is256BitVector())
29979 return splitVectorIntBinary(Op, DAG, dl);
29980
29981 if (VT == MVT::v32i16 || VT == MVT::v64i8)
29982 return splitVectorIntBinary(Op, DAG, dl);
29983
29984 return SDValue();
29985}
29986
29987static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
29988 SelectionDAG &DAG) {
29989 MVT VT = Op.getSimpleValueType();
29990 assert((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) &&
29991 "Unexpected funnel shift opcode!");
29992
29993 SDLoc DL(Op);
29994 SDValue Op0 = Op.getOperand(0);
29995 SDValue Op1 = Op.getOperand(1);
29996 SDValue Amt = Op.getOperand(2);
29997 unsigned EltSizeInBits = VT.getScalarSizeInBits();
29998 bool IsFSHR = Op.getOpcode() == ISD::FSHR;
29999
30000 if (VT.isVector()) {
30001 APInt APIntShiftAmt;
30002 bool IsCstSplat = X86::isConstantSplat(Amt, APIntShiftAmt);
30003 unsigned NumElts = VT.getVectorNumElements();
30004
30005 if (Subtarget.hasVBMI2() && EltSizeInBits > 8) {
30006 if (IsFSHR)
30007 std::swap(Op0, Op1);
30008
30009 if (IsCstSplat) {
30010 uint64_t ShiftAmt = APIntShiftAmt.urem(EltSizeInBits);
30011 SDValue Imm = DAG.getTargetConstant(ShiftAmt, DL, MVT::i8);
30012 return getAVX512Node(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT,
30013 {Op0, Op1, Imm}, DAG, Subtarget);
30014 }
30015 return getAVX512Node(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT,
30016 {Op0, Op1, Amt}, DAG, Subtarget);
30017 }
30018 assert((VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8 ||
30019 VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16 ||
30020 VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) &&
30021 "Unexpected funnel shift type!");
30022
30023 // fshl(x,y,z) -> unpack(y,x) << (z & (bw-1))) >> bw.
30024 // fshr(x,y,z) -> unpack(y,x) >> (z & (bw-1))).
30025 if (IsCstSplat) {
30026 // TODO: Can't use generic expansion as UNDEF amt elements can be
30027 // converted to other values when folded to shift amounts, losing the
30028 // splat.
30029 uint64_t ShiftAmt = APIntShiftAmt.urem(EltSizeInBits);
30030 uint64_t ShXAmt = IsFSHR ? (EltSizeInBits - ShiftAmt) : ShiftAmt;
30031 uint64_t ShYAmt = IsFSHR ? ShiftAmt : (EltSizeInBits - ShiftAmt);
30032 assert((ShXAmt + ShYAmt) == EltSizeInBits && "Illegal funnel shift");
30033 MVT WideVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
30034
30035 if (EltSizeInBits == 8 &&
30036 (Subtarget.hasXOP() ||
30037 (useVPTERNLOG(Subtarget, VT) &&
30038 supportedVectorShiftWithImm(WideVT, Subtarget, ISD::SHL)))) {
30039 // For vXi8 cases on Subtargets that can perform VPCMOV/VPTERNLOG
30040 // bit-select - lower using vXi16 shifts and then perform the bitmask at
30041 // the original vector width to handle cases where we split.
30042 APInt MaskX = APInt::getHighBitsSet(8, 8 - ShXAmt);
30043 APInt MaskY = APInt::getLowBitsSet(8, 8 - ShYAmt);
30044 SDValue ShX =
30045 DAG.getNode(ISD::SHL, DL, WideVT, DAG.getBitcast(WideVT, Op0),
30046 DAG.getShiftAmountConstant(ShXAmt, WideVT, DL));
30047 SDValue ShY =
30048 DAG.getNode(ISD::SRL, DL, WideVT, DAG.getBitcast(WideVT, Op1),
30049 DAG.getShiftAmountConstant(ShYAmt, WideVT, DL));
30050 ShX = DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, ShX),
30051 DAG.getConstant(MaskX, DL, VT));
30052 ShY = DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, ShY),
30053 DAG.getConstant(MaskY, DL, VT));
30054 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
30055 }
30056
30057 SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, Op0,
30058 DAG.getShiftAmountConstant(ShXAmt, VT, DL));
30059 SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Op1,
30060 DAG.getShiftAmountConstant(ShYAmt, VT, DL));
30061 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
30062 }
30063
30064 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
30065 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30066 bool IsCst = ISD::isBuildVectorOfConstantSDNodes(AmtMod.getNode());
30067
30068 // Constant vXi16 funnel shifts can be efficiently handled by default.
30069 if (IsCst && EltSizeInBits == 16)
30070 return SDValue();
30071
30072 unsigned ShiftOpc = IsFSHR ? ISD::SRL : ISD::SHL;
30073 MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);
30074 MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2);
30075
30076 // Split 256-bit integers on XOP/pre-AVX2 targets.
30077 // Split 512-bit integers on non 512-bit BWI targets.
30078 if ((VT.is256BitVector() && ((Subtarget.hasXOP() && EltSizeInBits < 16) ||
30079 !Subtarget.hasAVX2())) ||
30080 (VT.is512BitVector() && !Subtarget.useBWIRegs() &&
30081 EltSizeInBits < 32)) {
30082 // Pre-mask the amount modulo using the wider vector.
30083 Op = DAG.getNode(Op.getOpcode(), DL, VT, Op0, Op1, AmtMod);
30084 return splitVectorOp(Op, DAG, DL);
30085 }
30086
30087 // Attempt to fold scalar shift as unpack(y,x) << zext(splat(z))
30088 if (supportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, ShiftOpc)) {
30089 int ScalarAmtIdx = -1;
30090 if (SDValue ScalarAmt = DAG.getSplatSourceVector(AmtMod, ScalarAmtIdx)) {
30091 // Uniform vXi16 funnel shifts can be efficiently handled by default.
30092 if (EltSizeInBits == 16)
30093 return SDValue();
30094
30095 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
30096 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
30097 Lo = getTargetVShiftNode(ShiftOpc, DL, ExtVT, Lo, ScalarAmt,
30098 ScalarAmtIdx, Subtarget, DAG);
30099 Hi = getTargetVShiftNode(ShiftOpc, DL, ExtVT, Hi, ScalarAmt,
30100 ScalarAmtIdx, Subtarget, DAG);
30101 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR);
30102 }
30103 }
30104
30105 MVT WideSVT = MVT::getIntegerVT(
30106 std::min<unsigned>(EltSizeInBits * 2, Subtarget.hasBWI() ? 16 : 32));
30107 MVT WideVT = MVT::getVectorVT(WideSVT, NumElts);
30108
30109 // If per-element shifts are legal, fallback to generic expansion.
30110 if (supportedVectorVarShift(VT, Subtarget, ShiftOpc) || Subtarget.hasXOP())
30111 return SDValue();
30112
30113 // Attempt to fold as:
30114 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw.
30115 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))).
30116 if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) &&
30117 supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) {
30118 Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Op0);
30119 Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op1);
30120 AmtMod = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod);
30121 Op0 = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, Op0,
30122 EltSizeInBits, DAG);
30123 SDValue Res = DAG.getNode(ISD::OR, DL, WideVT, Op0, Op1);
30124 Res = DAG.getNode(ShiftOpc, DL, WideVT, Res, AmtMod);
30125 if (!IsFSHR)
30126 Res = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, Res,
30127 EltSizeInBits, DAG);
30128 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
30129 }
30130
30131 // Attempt to fold per-element (ExtVT) shift as unpack(y,x) << zext(z)
30132 if (((IsCst || !Subtarget.hasAVX512()) && !IsFSHR && EltSizeInBits <= 16) ||
30133 supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc)) {
30134 SDValue Z = DAG.getConstant(0, DL, VT);
30135 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0));
30136 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0));
30137 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
30138 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
30139 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
30140 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
30141 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR);
30142 }
30143
30144 // Fallback to generic expansion.
30145 return SDValue();
30146 }
30147 assert(
30148 (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
30149 "Unexpected funnel shift type!");
30150
30151 // Expand slow SHLD/SHRD cases if we are not optimizing for size.
30152 bool OptForSize = DAG.shouldOptForSize();
30153 bool ExpandFunnel = !OptForSize && Subtarget.isSHLDSlow();
30154
30155 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z & (bw-1))) >> bw.
30156 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z & (bw-1))).
30157 if ((VT == MVT::i8 || (ExpandFunnel && VT == MVT::i16)) &&
30158 !isa<ConstantSDNode>(Amt)) {
30159 SDValue Mask = DAG.getConstant(EltSizeInBits - 1, DL, Amt.getValueType());
30160 SDValue HiShift = DAG.getConstant(EltSizeInBits, DL, Amt.getValueType());
30161 Op0 = DAG.getAnyExtOrTrunc(Op0, DL, MVT::i32);
30162 Op1 = DAG.getZExtOrTrunc(Op1, DL, MVT::i32);
30163 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt, Mask);
30164 SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);
30165 Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1);
30166 if (IsFSHR) {
30167 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt);
30168 } else {
30169 Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt);
30170 Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift);
30171 }
30172 return DAG.getZExtOrTrunc(Res, DL, VT);
30173 }
30174
30175 if (VT == MVT::i8 || ExpandFunnel)
30176 return SDValue();
30177
30178 // i16 needs to modulo the shift amount, but i32/i64 have implicit modulo.
30179 if (VT == MVT::i16) {
30180 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt,
30181 DAG.getConstant(15, DL, Amt.getValueType()));
30182 unsigned FSHOp = (IsFSHR ? X86ISD::FSHR : X86ISD::FSHL);
30183 return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt);
30184 }
30185
30186 return Op;
30187}
30188
30189static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
30190 SelectionDAG &DAG) {
30191 MVT VT = Op.getSimpleValueType();
30192 assert(VT.isVector() && "Custom lowering only for vector rotates!");
30193
30194 SDLoc DL(Op);
30195 SDValue R = Op.getOperand(0);
30196 SDValue Amt = Op.getOperand(1);
30197 unsigned Opcode = Op.getOpcode();
30198 unsigned EltSizeInBits = VT.getScalarSizeInBits();
30199 int NumElts = VT.getVectorNumElements();
30200 bool IsROTL = Opcode == ISD::ROTL;
30201
30202 // Check for constant splat rotation amount.
30203 APInt CstSplatValue;
30204 bool IsCstSplat = X86::isConstantSplat(Amt, CstSplatValue);
30205
30206 // Check for splat rotate by zero.
30207 if (IsCstSplat && CstSplatValue.urem(EltSizeInBits) == 0)
30208 return R;
30209
30210 // AVX512 implicitly uses modulo rotation amounts.
30211 if ((Subtarget.hasVLX() ||
30212 (Subtarget.hasAVX512() && Subtarget.hasEVEX512())) &&
30213 32 <= EltSizeInBits) {
30214 // Attempt to rotate by immediate.
30215 if (IsCstSplat) {
30216 unsigned RotOpc = IsROTL ? X86ISD::VROTLI : X86ISD::VROTRI;
30217 uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
30218 return DAG.getNode(RotOpc, DL, VT, R,
30219 DAG.getTargetConstant(RotAmt, DL, MVT::i8));
30220 }
30221
30222 // Else, fall-back on VPROLV/VPRORV.
30223 return Op;
30224 }
30225
30226 // AVX512 VBMI2 vXi16 - lower to funnel shifts.
30227 if (Subtarget.hasVBMI2() && 16 == EltSizeInBits) {
30228 unsigned FunnelOpc = IsROTL ? ISD::FSHL : ISD::FSHR;
30229 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
30230 }
30231
30232 SDValue Z = DAG.getConstant(0, DL, VT);
30233
30234 if (!IsROTL) {
30235 // If the ISD::ROTR amount is constant, we're always better converting to
30236 // ISD::ROTL.
30237 if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt}))
30238 return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt);
30239
30240 // XOP targets always prefers ISD::ROTL.
30241 if (Subtarget.hasXOP())
30242 return DAG.getNode(ISD::ROTL, DL, VT, R,
30243 DAG.getNode(ISD::SUB, DL, VT, Z, Amt));
30244 }
30245
30246 // Attempt to use GFNI gf2p8affine to rotate vXi8 by an uniform constant.
30247 if (IsCstSplat && Subtarget.hasGFNI() && VT.getScalarType() == MVT::i8 &&
30248 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
30249 uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
30250 SDValue Mask = getGFNICtrlMask(Opcode, DAG, DL, VT, RotAmt);
30251 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, R, Mask,
30252 DAG.getTargetConstant(0, DL, MVT::i8));
30253 }
30254
30255 // Split 256-bit integers on XOP/pre-AVX2 targets.
30256 if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2()))
30257 return splitVectorIntBinary(Op, DAG, DL);
30258
30259 // XOP has 128-bit vector variable + immediate rotates.
30260 // +ve/-ve Amt = rotate left/right - just need to handle ISD::ROTL.
30261 // XOP implicitly uses modulo rotation amounts.
30262 if (Subtarget.hasXOP()) {
30263 assert(IsROTL && "Only ROTL expected");
30264 assert(VT.is128BitVector() && "Only rotate 128-bit vectors!");
30265
30266 // Attempt to rotate by immediate.
30267 if (IsCstSplat) {
30268 uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
30269 return DAG.getNode(X86ISD::VROTLI, DL, VT, R,
30270 DAG.getTargetConstant(RotAmt, DL, MVT::i8));
30271 }
30272
30273 // Use general rotate by variable (per-element).
30274 return Op;
30275 }
30276
30277 // Rotate by an uniform constant - expand back to shifts.
30278 // TODO: Can't use generic expansion as UNDEF amt elements can be converted
30279 // to other values when folded to shift amounts, losing the splat.
30280 if (IsCstSplat) {
30281 uint64_t RotAmt = CstSplatValue.urem(EltSizeInBits);
30282 uint64_t ShlAmt = IsROTL ? RotAmt : (EltSizeInBits - RotAmt);
30283 uint64_t SrlAmt = IsROTL ? (EltSizeInBits - RotAmt) : RotAmt;
30284 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, R,
30285 DAG.getShiftAmountConstant(ShlAmt, VT, DL));
30286 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, R,
30287 DAG.getShiftAmountConstant(SrlAmt, VT, DL));
30288 return DAG.getNode(ISD::OR, DL, VT, Shl, Srl);
30289 }
30290
30291 // Split 512-bit integers on non 512-bit BWI targets.
30292 if (VT.is512BitVector() && !Subtarget.useBWIRegs())
30293 return splitVectorIntBinary(Op, DAG, DL);
30294
30295 assert(
30296 (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
30297 ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) &&
30298 Subtarget.hasAVX2()) ||
30299 ((VT == MVT::v32i16 || VT == MVT::v64i8) && Subtarget.useBWIRegs())) &&
30300 "Only vXi32/vXi16/vXi8 vector rotates supported");
30301
30302 MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);
30303 MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2);
30304
30305 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT);
30306 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30307
30308 // Attempt to fold as unpack(x,x) << zext(splat(y)):
30309 // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw.
30310 // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))).
30311 if (EltSizeInBits == 8 || EltSizeInBits == 16 || EltSizeInBits == 32) {
30312 int BaseRotAmtIdx = -1;
30313 if (SDValue BaseRotAmt = DAG.getSplatSourceVector(AmtMod, BaseRotAmtIdx)) {
30314 if (EltSizeInBits == 16 && Subtarget.hasSSE41()) {
30315 unsigned FunnelOpc = IsROTL ? ISD::FSHL : ISD::FSHR;
30316 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
30317 }
30318 unsigned ShiftX86Opc = IsROTL ? X86ISD::VSHLI : X86ISD::VSRLI;
30319 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
30320 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
30321 Lo = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Lo, BaseRotAmt,
30322 BaseRotAmtIdx, Subtarget, DAG);
30323 Hi = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Hi, BaseRotAmt,
30324 BaseRotAmtIdx, Subtarget, DAG);
30325 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
30326 }
30327 }
30328
30329 bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
30330 unsigned ShiftOpc = IsROTL ? ISD::SHL : ISD::SRL;
30331
30332 // Attempt to fold as unpack(x,x) << zext(y):
30333 // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw.
30334 // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))).
30335 // Const vXi16/vXi32 are excluded in favor of MUL-based lowering.
30336 if (!(ConstantAmt && EltSizeInBits != 8) &&
30337 !supportedVectorVarShift(VT, Subtarget, ShiftOpc) &&
30338 (ConstantAmt || supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc))) {
30339 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R));
30340 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R));
30341 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z));
30342 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z));
30343 SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo);
30344 SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi);
30345 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL);
30346 }
30347
30348 // v16i8/v32i8/v64i8: Split rotation into rot4/rot2/rot1 stages and select by
30349 // the amount bit.
30350 // TODO: We're doing nothing here that we couldn't do for funnel shifts.
30351 if (EltSizeInBits == 8) {
30352 MVT WideVT =
30353 MVT::getVectorVT(Subtarget.hasBWI() ? MVT::i16 : MVT::i32, NumElts);
30354
30355 // Attempt to fold as:
30356 // rotl(x,y) -> (((aext(x) << bw) | zext(x)) << (y & (bw-1))) >> bw.
30357 // rotr(x,y) -> (((aext(x) << bw) | zext(x)) >> (y & (bw-1))).
30358 if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) &&
30359 supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) {
30360 // If we're rotating by constant, just use default promotion.
30361 if (ConstantAmt)
30362 return SDValue();
30363 // See if we can perform this by widening to vXi16 or vXi32.
30364 R = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, R);
30365 R = DAG.getNode(
30366 ISD::OR, DL, WideVT, R,
30367 getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, R, 8, DAG));
30368 Amt = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod);
30369 R = DAG.getNode(ShiftOpc, DL, WideVT, R, Amt);
30370 if (IsROTL)
30371 R = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, R, 8, DAG);
30372 return DAG.getNode(ISD::TRUNCATE, DL, VT, R);
30373 }
30374
30375 // We don't need ModuloAmt here as we just peek at individual bits.
30376 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
30377 if (Subtarget.hasSSE41()) {
30378 // On SSE41 targets we can use PBLENDVB which selects bytes based just
30379 // on the sign bit.
30380 V0 = DAG.getBitcast(VT, V0);
30381 V1 = DAG.getBitcast(VT, V1);
30382 Sel = DAG.getBitcast(VT, Sel);
30383 return DAG.getBitcast(SelVT,
30384 DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1));
30385 }
30386 // On pre-SSE41 targets we test for the sign bit by comparing to
30387 // zero - a negative value will set all bits of the lanes to true
30388 // and VSELECT uses that in its OR(AND(V0,C),AND(V1,~C)) lowering.
30389 SDValue Z = DAG.getConstant(0, DL, SelVT);
30390 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
30391 return DAG.getSelect(DL, SelVT, C, V0, V1);
30392 };
30393
30394 // ISD::ROTR is currently only profitable on AVX512 targets with VPTERNLOG.
30395 if (!IsROTL && !useVPTERNLOG(Subtarget, VT)) {
30396 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
30397 IsROTL = true;
30398 }
30399
30400 unsigned ShiftLHS = IsROTL ? ISD::SHL : ISD::SRL;
30401 unsigned ShiftRHS = IsROTL ? ISD::SRL : ISD::SHL;
30402
30403 // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
30404 // We can safely do this using i16 shifts as we're only interested in
30405 // the 3 lower bits of each byte.
30406 Amt = DAG.getBitcast(ExtVT, Amt);
30407 Amt = DAG.getNode(ISD::SHL, DL, ExtVT, Amt, DAG.getConstant(5, DL, ExtVT));
30408 Amt = DAG.getBitcast(VT, Amt);
30409
30410 // r = VSELECT(r, rot(r, 4), a);
30411 SDValue M;
30412 M = DAG.getNode(
30413 ISD::OR, DL, VT,
30414 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(4, DL, VT)),
30415 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(4, DL, VT)));
30416 R = SignBitSelect(VT, Amt, M, R);
30417
30418 // a += a
30419 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
30420
30421 // r = VSELECT(r, rot(r, 2), a);
30422 M = DAG.getNode(
30423 ISD::OR, DL, VT,
30424 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(2, DL, VT)),
30425 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(6, DL, VT)));
30426 R = SignBitSelect(VT, Amt, M, R);
30427
30428 // a += a
30429 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
30430
30431 // return VSELECT(r, rot(r, 1), a);
30432 M = DAG.getNode(
30433 ISD::OR, DL, VT,
30434 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(1, DL, VT)),
30435 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(7, DL, VT)));
30436 return SignBitSelect(VT, Amt, M, R);
30437 }
30438
30439 bool IsSplatAmt = DAG.isSplatValue(Amt);
30440 bool LegalVarShifts = supportedVectorVarShift(VT, Subtarget, ISD::SHL) &&
30441 supportedVectorVarShift(VT, Subtarget, ISD::SRL);
30442
30443 // Fallback for splats + all supported variable shifts.
30444 // Fallback for non-constants AVX2 vXi16 as well.
30445 if (IsSplatAmt || LegalVarShifts || (Subtarget.hasAVX2() && !ConstantAmt)) {
30446 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30447 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
30448 AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt);
30449 SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt);
30450 SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR);
30451 return DAG.getNode(ISD::OR, DL, VT, SHL, SRL);
30452 }
30453
30454 // Everything below assumes ISD::ROTL.
30455 if (!IsROTL) {
30456 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt);
30457 IsROTL = true;
30458 }
30459
30460 // ISD::ROT* uses modulo rotate amounts.
30461 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask);
30462
30463 assert(IsROTL && "Only ROTL supported");
30464
30465 // As with shifts, attempt to convert the rotation amount to a multiplication
30466 // factor, fallback to general expansion.
30467 SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
30468 if (!Scale)
30469 return SDValue();
30470
30471 // v8i16/v16i16: perform unsigned multiply hi/lo and OR the results.
30472 if (EltSizeInBits == 16) {
30473 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);
30474 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);
30475 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
30476 }
30477
30478 // v4i32: make use of the PMULUDQ instruction to multiply 2 lanes of v4i32
30479 // to v2i64 results at a time. The upper 32-bits contain the wrapped bits
30480 // that can then be OR'd with the lower 32-bits.
30481 assert(VT == MVT::v4i32 && "Only v4i32 vector rotate expected");
30482 static const int OddMask[] = {1, -1, 3, -1};
30483 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);
30484 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);
30485
30486 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
30487 DAG.getBitcast(MVT::v2i64, R),
30488 DAG.getBitcast(MVT::v2i64, Scale));
30489 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
30490 DAG.getBitcast(MVT::v2i64, R13),
30491 DAG.getBitcast(MVT::v2i64, Scale13));
30492 Res02 = DAG.getBitcast(VT, Res02);
30493 Res13 = DAG.getBitcast(VT, Res13);
30494
30495 return DAG.getNode(ISD::OR, DL, VT,
30496 DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}),
30497 DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7}));
30498}
30499
30500/// Returns true if the operand type is exactly twice the native width, and
30501/// the corresponding cmpxchg8b or cmpxchg16b instruction is available.
30502/// Used to know whether to use cmpxchg8/16b when expanding atomic operations
30503/// (otherwise we leave them alone to become __sync_fetch_and_... calls).
30504bool X86TargetLowering::needsCmpXchgNb(Type *MemType) const {
30505 unsigned OpWidth = MemType->getPrimitiveSizeInBits();
30506
30507 if (OpWidth == 64)
30508 return Subtarget.canUseCMPXCHG8B() && !Subtarget.is64Bit();
30509 if (OpWidth == 128)
30510 return Subtarget.canUseCMPXCHG16B();
30511
30512 return false;
30513}
30514
30515TargetLoweringBase::AtomicExpansionKind
30516X86TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
30517 Type *MemType = SI->getValueOperand()->getType();
30518
30519 if (!SI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) &&
30520 !Subtarget.useSoftFloat()) {
30521 if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
30522 (Subtarget.hasSSE1() || Subtarget.hasX87()))
30523 return AtomicExpansionKind::None;
30524
30525 if (MemType->getPrimitiveSizeInBits() == 128 && Subtarget.is64Bit() &&
30526 Subtarget.hasAVX())
30527 return AtomicExpansionKind::None;
30528 }
30529
30530 return needsCmpXchgNb(MemType) ? AtomicExpansionKind::Expand
30531 : AtomicExpansionKind::None;
30532}
30533
30534// Note: this turns large loads into lock cmpxchg8b/16b.
30535TargetLowering::AtomicExpansionKind
30536X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
30537 Type *MemType = LI->getType();
30538
30539 if (!LI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) &&
30540 !Subtarget.useSoftFloat()) {
30541 // If this a 64 bit atomic load on a 32-bit target and SSE2 is enabled, we
30542 // can use movq to do the load. If we have X87 we can load into an 80-bit
30543 // X87 register and store it to a stack temporary.
30544 if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
30545 (Subtarget.hasSSE1() || Subtarget.hasX87()))
30546 return AtomicExpansionKind::None;
30547
30548 // If this is a 128-bit load with AVX, 128-bit SSE loads/stores are atomic.
30549 if (MemType->getPrimitiveSizeInBits() == 128 && Subtarget.is64Bit() &&
30550 Subtarget.hasAVX())
30551 return AtomicExpansionKind::None;
30552 }
30553
30554 return needsCmpXchgNb(MemType) ? AtomicExpansionKind::CmpXChg
30555 : AtomicExpansionKind::None;
30556}
30557
30558enum BitTestKind : unsigned {
30559 UndefBit,
30560 ConstantBit,
30561 NotConstantBit,
30562 ShiftBit,
30563 NotShiftBit
30564};
30565
30566static std::pair<Value *, BitTestKind> FindSingleBitChange(Value *V) {
30567 using namespace llvm::PatternMatch;
30568 BitTestKind BTK = UndefBit;
30569 if (auto *C = dyn_cast<ConstantInt>(V)) {
30570 // Check if V is a power of 2 or NOT power of 2.
30571 if (isPowerOf2_64(C->getZExtValue()))
30572 BTK = ConstantBit;
30573 else if (isPowerOf2_64((~C->getValue()).getZExtValue()))
30574 BTK = NotConstantBit;
30575 return {V, BTK};
30576 }
30577
30578 // Check if V is some power of 2 pattern known to be non-zero
30579 if (auto *I = dyn_cast<Instruction>(V)) {
30580 bool Not = false;
30581 // Check if we have a NOT
30582 Value *PeekI;
30583 if (match(I, m_Not(m_Value(PeekI))) ||
30584 match(I, m_Sub(m_AllOnes(), m_Value(PeekI)))) {
30585 Not = true;
30586 I = dyn_cast<Instruction>(PeekI);
30587
30588 // If I is constant, it will fold and we can evaluate later. If its an
30589 // argument or something of that nature, we can't analyze.
30590 if (I == nullptr)
30591 return {nullptr, UndefBit};
30592 }
30593 // We can only use 1 << X without more sophisticated analysis. C << X where
30594 // C is a power of 2 but not 1 can result in zero which cannot be translated
30595 // to bittest. Likewise any C >> X (either arith or logical) can be zero.
30596 if (I->getOpcode() == Instruction::Shl) {
30597 // Todo(1): The cmpxchg case is pretty costly so matching `BLSI(X)`, `X &
30598 // -X` and some other provable power of 2 patterns that we can use CTZ on
30599 // may be profitable.
30600 // Todo(2): It may be possible in some cases to prove that Shl(C, X) is
30601 // non-zero even where C != 1. Likewise LShr(C, X) and AShr(C, X) may also
30602 // be provably a non-zero power of 2.
30603 // Todo(3): ROTL and ROTR patterns on a power of 2 C should also be
30604 // transformable to bittest.
30605 auto *ShiftVal = dyn_cast<ConstantInt>(I->getOperand(0));
30606 if (!ShiftVal)
30607 return {nullptr, UndefBit};
30608 if (ShiftVal->equalsInt(1))
30609 BTK = Not ? NotShiftBit : ShiftBit;
30610
30611 if (BTK == UndefBit)
30612 return {nullptr, UndefBit};
30613
30614 Value *BitV = I->getOperand(1);
30615
30616 // Read past a shiftmask instruction to find count
30617 Value *AndOp;
30618 uint64_t ShiftMask = I->getType()->getPrimitiveSizeInBits() - 1;
30619 if (match(BitV, m_c_And(m_Value(AndOp), m_SpecificInt(ShiftMask))))
30620 BitV = AndOp;
30621
30622 return {BitV, BTK};
30623 }
30624 }
30625 return {nullptr, UndefBit};
30626}
30627
30628TargetLowering::AtomicExpansionKind
30629X86TargetLowering::shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const {
30630 using namespace llvm::PatternMatch;
30631 // If the atomicrmw's result isn't actually used, we can just add a "lock"
30632 // prefix to a normal instruction for these operations.
30633 if (AI->use_empty())
30634 return AtomicExpansionKind::None;
30635
30636 if (AI->getOperation() == AtomicRMWInst::Xor) {
30637 // A ^ SignBit -> A + SignBit. This allows us to use `xadd` which is
30638 // preferable to both `cmpxchg` and `btc`.
30639 if (match(AI->getOperand(1), m_SignMask()))
30640 return AtomicExpansionKind::None;
30641 }
30642
30643 // If the atomicrmw's result is used by a single bit AND, we may use
30644 // bts/btr/btc instruction for these operations.
30645 // Note: InstCombinePass can cause a de-optimization here. It replaces the
30646 // SETCC(And(AtomicRMW(P, power_of_2), power_of_2)) with LShr and Xor
30647 // (depending on CC). This pattern can only use bts/btr/btc but we don't
30648 // detect it.
30649 Instruction *I = AI->user_back();
30650 auto BitChange = FindSingleBitChange(AI->getValOperand());
30651 if (BitChange.second == UndefBit || !AI->hasOneUse() ||
30652 I->getOpcode() != Instruction::And ||
30653 AI->getType()->getPrimitiveSizeInBits() == 8 ||
30654 AI->getParent() != I->getParent())
30655 return AtomicExpansionKind::CmpXChg;
30656
30657 unsigned OtherIdx = I->getOperand(0) == AI ? 1 : 0;
30658
30659 // This is a redundant AND, it should get cleaned up elsewhere.
30660 if (AI == I->getOperand(OtherIdx))
30661 return AtomicExpansionKind::CmpXChg;
30662
30663 // The following instruction must be a AND single bit.
30664 if (BitChange.second == ConstantBit || BitChange.second == NotConstantBit) {
30665 auto *C1 = cast<ConstantInt>(AI->getValOperand());
30666 auto *C2 = dyn_cast<ConstantInt>(I->getOperand(OtherIdx));
30667 if (!C2 || !isPowerOf2_64(C2->getZExtValue())) {
30668 return AtomicExpansionKind::CmpXChg;
30669 }
30670 if (AI->getOperation() == AtomicRMWInst::And) {
30671 return ~C1->getValue() == C2->getValue()
30672 ? AtomicExpansionKind::BitTestIntrinsic
30673 : AtomicExpansionKind::CmpXChg;
30674 }
30675 return C1 == C2 ? AtomicExpansionKind::BitTestIntrinsic
30676 : AtomicExpansionKind::CmpXChg;
30677 }
30678
30679 assert(BitChange.second == ShiftBit || BitChange.second == NotShiftBit);
30680
30681 auto BitTested = FindSingleBitChange(I->getOperand(OtherIdx));
30682 if (BitTested.second != ShiftBit && BitTested.second != NotShiftBit)
30683 return AtomicExpansionKind::CmpXChg;
30684
30685 assert(BitChange.first != nullptr && BitTested.first != nullptr);
30686
30687 // If shift amounts are not the same we can't use BitTestIntrinsic.
30688 if (BitChange.first != BitTested.first)
30689 return AtomicExpansionKind::CmpXChg;
30690
30691 // If atomic AND need to be masking all be one bit and testing the one bit
30692 // unset in the mask.
30693 if (AI->getOperation() == AtomicRMWInst::And)
30694 return (BitChange.second == NotShiftBit && BitTested.second == ShiftBit)
30695 ? AtomicExpansionKind::BitTestIntrinsic
30696 : AtomicExpansionKind::CmpXChg;
30697
30698 // If atomic XOR/OR need to be setting and testing the same bit.
30699 return (BitChange.second == ShiftBit && BitTested.second == ShiftBit)
30700 ? AtomicExpansionKind::BitTestIntrinsic
30701 : AtomicExpansionKind::CmpXChg;
30702}
30703
30704void X86TargetLowering::emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const {
30705 IRBuilder<> Builder(AI);
30706 Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
30707 Intrinsic::ID IID_C = Intrinsic::not_intrinsic;
30708 Intrinsic::ID IID_I = Intrinsic::not_intrinsic;
30709 switch (AI->getOperation()) {
30710 default:
30711 llvm_unreachable("Unknown atomic operation");
30712 case AtomicRMWInst::Or:
30713 IID_C = Intrinsic::x86_atomic_bts;
30714 IID_I = Intrinsic::x86_atomic_bts_rm;
30715 break;
30716 case AtomicRMWInst::Xor:
30717 IID_C = Intrinsic::x86_atomic_btc;
30718 IID_I = Intrinsic::x86_atomic_btc_rm;
30719 break;
30720 case AtomicRMWInst::And:
30721 IID_C = Intrinsic::x86_atomic_btr;
30722 IID_I = Intrinsic::x86_atomic_btr_rm;
30723 break;
30724 }
30725 Instruction *I = AI->user_back();
30726 LLVMContext &Ctx = AI->getContext();
30727 Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(),
30728 PointerType::getUnqual(Ctx));
30729 Function *BitTest = nullptr;
30730 Value *Result = nullptr;
30731 auto BitTested = FindSingleBitChange(AI->getValOperand());
30732 assert(BitTested.first != nullptr);
30733
30734 if (BitTested.second == ConstantBit || BitTested.second == NotConstantBit) {
30735 auto *C = cast<ConstantInt>(I->getOperand(I->getOperand(0) == AI ? 1 : 0));
30736
30737 BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_C, AI->getType());
30738
30739 unsigned Imm = llvm::countr_zero(C->getZExtValue());
30740 Result = Builder.CreateCall(BitTest, {Addr, Builder.getInt8(Imm)});
30741 } else {
30742 BitTest = Intrinsic::getDeclaration(AI->getModule(), IID_I, AI->getType());
30743
30744 assert(BitTested.second == ShiftBit || BitTested.second == NotShiftBit);
30745
30746 Value *SI = BitTested.first;
30747 assert(SI != nullptr);
30748
30749 // BT{S|R|C} on memory operand don't modulo bit position so we need to
30750 // mask it.
30751 unsigned ShiftBits = SI->getType()->getPrimitiveSizeInBits();
30752 Value *BitPos =
30753 Builder.CreateAnd(SI, Builder.getIntN(ShiftBits, ShiftBits - 1));
30754 // Todo(1): In many cases it may be provable that SI is less than
30755 // ShiftBits in which case this mask is unnecessary
30756 // Todo(2): In the fairly idiomatic case of P[X / sizeof_bits(X)] OP 1
30757 // << (X % sizeof_bits(X)) we can drop the shift mask and AGEN in
30758 // favor of just a raw BT{S|R|C}.
30759
30760 Result = Builder.CreateCall(BitTest, {Addr, BitPos});
30761 Result = Builder.CreateZExtOrTrunc(Result, AI->getType());
30762
30763 // If the result is only used for zero/non-zero status then we don't need to
30764 // shift value back. Otherwise do so.
30765 for (auto It = I->user_begin(); It != I->user_end(); ++It) {
30766 if (auto *ICmp = dyn_cast<ICmpInst>(*It)) {
30767 if (ICmp->isEquality()) {
30768 auto *C0 = dyn_cast<ConstantInt>(ICmp->getOperand(0));
30769 auto *C1 = dyn_cast<ConstantInt>(ICmp->getOperand(1));
30770 if (C0 || C1) {
30771 assert(C0 == nullptr || C1 == nullptr);
30772 if ((C0 ? C0 : C1)->isZero())
30773 continue;
30774 }
30775 }
30776 }
30777 Result = Builder.CreateShl(Result, BitPos);
30778 break;
30779 }
30780 }
30781
30782 I->replaceAllUsesWith(Result);
30783 I->eraseFromParent();
30784 AI->eraseFromParent();
30785}
30786
30787static bool shouldExpandCmpArithRMWInIR(AtomicRMWInst *AI) {
30788 using namespace llvm::PatternMatch;
30789 if (!AI->hasOneUse())
30790 return false;
30791
30792 Value *Op = AI->getOperand(1);
30793 ICmpInst::Predicate Pred;
30794 Instruction *I = AI->user_back();
30795 AtomicRMWInst::BinOp Opc = AI->getOperation();
30796 if (Opc == AtomicRMWInst::Add) {
30797 if (match(I, m_c_ICmp(Pred, m_Sub(m_ZeroInt(), m_Specific(Op)), m_Value())))
30798 return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE;
30799 if (match(I, m_OneUse(m_c_Add(m_Specific(Op), m_Value())))) {
30800 if (match(I->user_back(),
30801 m_SpecificICmp(CmpInst::ICMP_SLT, m_Value(), m_ZeroInt())))
30802 return true;
30803 if (match(I->user_back(),
30804 m_SpecificICmp(CmpInst::ICMP_SGT, m_Value(), m_AllOnes())))
30805 return true;
30806 }
30807 return false;
30808 }
30809 if (Opc == AtomicRMWInst::Sub) {
30810 if (match(I, m_c_ICmp(Pred, m_Specific(Op), m_Value())))
30811 return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE;
30812 if (match(I, m_OneUse(m_Sub(m_Value(), m_Specific(Op))))) {
30813 if (match(I->user_back(),
30814 m_SpecificICmp(CmpInst::ICMP_SLT, m_Value(), m_ZeroInt())))
30815 return true;
30816 if (match(I->user_back(),
30817 m_SpecificICmp(CmpInst::ICMP_SGT, m_Value(), m_AllOnes())))
30818 return true;
30819 }
30820 return false;
30821 }
30822 if ((Opc == AtomicRMWInst::Or &&
30823 match(I, m_OneUse(m_c_Or(m_Specific(Op), m_Value())))) ||
30824 (Opc == AtomicRMWInst::And &&
30825 match(I, m_OneUse(m_c_And(m_Specific(Op), m_Value()))))) {
30826 if (match(I->user_back(), m_ICmp(Pred, m_Value(), m_ZeroInt())))
30827 return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE ||
30828 Pred == CmpInst::ICMP_SLT;
30829 if (match(I->user_back(),
30830 m_SpecificICmp(CmpInst::ICMP_SGT, m_Value(), m_AllOnes())))
30831 return true;
30832 return false;
30833 }
30834 if (Opc == AtomicRMWInst::Xor) {
30835 if (match(I, m_c_ICmp(Pred, m_Specific(Op), m_Value())))
30836 return Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE;
30837 if (match(I, m_OneUse(m_c_Xor(m_Specific(Op), m_Value())))) {
30838 if (match(I->user_back(),
30839 m_SpecificICmp(CmpInst::ICMP_SLT, m_Value(), m_ZeroInt())))
30840 return true;
30841 if (match(I->user_back(),
30842 m_SpecificICmp(CmpInst::ICMP_SGT, m_Value(), m_AllOnes())))
30843 return true;
30844 }
30845 return false;
30846 }
30847
30848 return false;
30849}
30850
30851void X86TargetLowering::emitCmpArithAtomicRMWIntrinsic(
30852 AtomicRMWInst *AI) const {
30853 IRBuilder<> Builder(AI);
30854 Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
30855 Instruction *TempI = nullptr;
30856 LLVMContext &Ctx = AI->getContext();
30857 ICmpInst *ICI = dyn_cast<ICmpInst>(AI->user_back());
30858 if (!ICI) {
30859 TempI = AI->user_back();
30860 assert(TempI->hasOneUse() && "Must have one use");
30861 ICI = cast<ICmpInst>(TempI->user_back());
30862 }
30863 X86::CondCode CC = X86::COND_INVALID;
30864 ICmpInst::Predicate Pred = ICI->getPredicate();
30865 switch (Pred) {
30866 default:
30867 llvm_unreachable("Not supported Pred");
30868 case CmpInst::ICMP_EQ:
30869 CC = X86::COND_E;
30870 break;
30871 case CmpInst::ICMP_NE:
30872 CC = X86::COND_NE;
30873 break;
30874 case CmpInst::ICMP_SLT:
30875 CC = X86::COND_S;
30876 break;
30877 case CmpInst::ICMP_SGT:
30878 CC = X86::COND_NS;
30879 break;
30880 }
30881 Intrinsic::ID IID = Intrinsic::not_intrinsic;
30882 switch (AI->getOperation()) {
30883 default:
30884 llvm_unreachable("Unknown atomic operation");
30885 case AtomicRMWInst::Add:
30886 IID = Intrinsic::x86_atomic_add_cc;
30887 break;
30888 case AtomicRMWInst::Sub:
30889 IID = Intrinsic::x86_atomic_sub_cc;
30890 break;
30891 case AtomicRMWInst::Or:
30892 IID = Intrinsic::x86_atomic_or_cc;
30893 break;
30894 case AtomicRMWInst::And:
30895 IID = Intrinsic::x86_atomic_and_cc;
30896 break;
30897 case AtomicRMWInst::Xor:
30898 IID = Intrinsic::x86_atomic_xor_cc;
30899 break;
30900 }
30901 Function *CmpArith =
30902 Intrinsic::getDeclaration(AI->getModule(), IID, AI->getType());
30903 Value *Addr = Builder.CreatePointerCast(AI->getPointerOperand(),
30904 PointerType::getUnqual(Ctx));
30905 Value *Call = Builder.CreateCall(
30906 CmpArith, {Addr, AI->getValOperand(), Builder.getInt32((unsigned)CC)});
30907 Value *Result = Builder.CreateTrunc(Call, Type::getInt1Ty(Ctx));
30908 ICI->replaceAllUsesWith(Result);
30909 ICI->eraseFromParent();
30910 if (TempI)
30911 TempI->eraseFromParent();
30912 AI->eraseFromParent();
30913}
30914
30915TargetLowering::AtomicExpansionKind
30916X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
30917 unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
30918 Type *MemType = AI->getType();
30919
30920 // If the operand is too big, we must see if cmpxchg8/16b is available
30921 // and default to library calls otherwise.
30922 if (MemType->getPrimitiveSizeInBits() > NativeWidth) {
30923 return needsCmpXchgNb(MemType) ? AtomicExpansionKind::CmpXChg
30924 : AtomicExpansionKind::None;
30925 }
30926
30927 AtomicRMWInst::BinOp Op = AI->getOperation();
30928 switch (Op) {
30929 case AtomicRMWInst::Xchg:
30930 return AtomicExpansionKind::None;
30931 case AtomicRMWInst::Add:
30932 case AtomicRMWInst::Sub:
30933 if (shouldExpandCmpArithRMWInIR(AI))
30934 return AtomicExpansionKind::CmpArithIntrinsic;
30935 // It's better to use xadd, xsub or xchg for these in other cases.
30936 return AtomicExpansionKind::None;
30937 case AtomicRMWInst::Or:
30938 case AtomicRMWInst::And:
30939 case AtomicRMWInst::Xor:
30940 if (shouldExpandCmpArithRMWInIR(AI))
30941 return AtomicExpansionKind::CmpArithIntrinsic;
30942 return shouldExpandLogicAtomicRMWInIR(AI);
30943 case AtomicRMWInst::Nand:
30944 case AtomicRMWInst::Max:
30945 case AtomicRMWInst::Min:
30946 case AtomicRMWInst::UMax:
30947 case AtomicRMWInst::UMin:
30948 case AtomicRMWInst::FAdd:
30949 case AtomicRMWInst::FSub:
30950 case AtomicRMWInst::FMax:
30951 case AtomicRMWInst::FMin:
30952 case AtomicRMWInst::UIncWrap:
30953 case AtomicRMWInst::UDecWrap:
30954 default:
30955 // These always require a non-trivial set of data operations on x86. We must
30956 // use a cmpxchg loop.
30957 return AtomicExpansionKind::CmpXChg;
30958 }
30959}
30960
30961LoadInst *
30962X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
30963 unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
30964 Type *MemType = AI->getType();
30965 // Accesses larger than the native width are turned into cmpxchg/libcalls, so
30966 // there is no benefit in turning such RMWs into loads, and it is actually
30967 // harmful as it introduces a mfence.
30968 if (MemType->getPrimitiveSizeInBits() > NativeWidth)
30969 return nullptr;
30970
30971 // If this is a canonical idempotent atomicrmw w/no uses, we have a better
30972 // lowering available in lowerAtomicArith.
30973 // TODO: push more cases through this path.
30974 if (auto *C = dyn_cast<ConstantInt>(AI->getValOperand()))
30975 if (AI->getOperation() == AtomicRMWInst::Or && C->isZero() &&
30976 AI->use_empty())
30977 return nullptr;
30978
30979 IRBuilder<> Builder(AI);
30980 Builder.CollectMetadataToCopy(AI, {LLVMContext::MD_pcsections});
30981 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
30982 auto SSID = AI->getSyncScopeID();
30983 // We must restrict the ordering to avoid generating loads with Release or
30984 // ReleaseAcquire orderings.
30985 auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
30986
30987 // Before the load we need a fence. Here is an example lifted from
30988 // http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence
30989 // is required:
30990 // Thread 0:
30991 // x.store(1, relaxed);
30992 // r1 = y.fetch_add(0, release);
30993 // Thread 1:
30994 // y.fetch_add(42, acquire);
30995 // r2 = x.load(relaxed);
30996 // r1 = r2 = 0 is impossible, but becomes possible if the idempotent rmw is
30997 // lowered to just a load without a fence. A mfence flushes the store buffer,
30998 // making the optimization clearly correct.
30999 // FIXME: it is required if isReleaseOrStronger(Order) but it is not clear
31000 // otherwise, we might be able to be more aggressive on relaxed idempotent
31001 // rmw. In practice, they do not look useful, so we don't try to be
31002 // especially clever.
31003 if (SSID == SyncScope::SingleThread)
31004 // FIXME: we could just insert an ISD::MEMBARRIER here, except we are at
31005 // the IR level, so we must wrap it in an intrinsic.
31006 return nullptr;
31007
31008 if (!Subtarget.hasMFence())
31009 // FIXME: it might make sense to use a locked operation here but on a
31010 // different cache-line to prevent cache-line bouncing. In practice it
31011 // is probably a small win, and x86 processors without mfence are rare
31012 // enough that we do not bother.
31013 return nullptr;
31014
31015 Function *MFence =
31016 llvm::Intrinsic::getDeclaration(M, Intrinsic::x86_sse2_mfence);
31017 Builder.CreateCall(MFence, {});
31018
31019 // Finally we can emit the atomic load.
31020 LoadInst *Loaded = Builder.CreateAlignedLoad(
31021 AI->getType(), AI->getPointerOperand(), AI->getAlign());
31022 Loaded->setAtomic(Order, SSID);
31023 AI->replaceAllUsesWith(Loaded);
31024 AI->eraseFromParent();
31025 return Loaded;
31026}
31027
31028/// Emit a locked operation on a stack location which does not change any
31029/// memory location, but does involve a lock prefix. Location is chosen to be
31030/// a) very likely accessed only by a single thread to minimize cache traffic,
31031/// and b) definitely dereferenceable. Returns the new Chain result.
31032static SDValue emitLockedStackOp(SelectionDAG &DAG,
31033 const X86Subtarget &Subtarget, SDValue Chain,
31034 const SDLoc &DL) {
31035 // Implementation notes:
31036 // 1) LOCK prefix creates a full read/write reordering barrier for memory
31037 // operations issued by the current processor. As such, the location
31038 // referenced is not relevant for the ordering properties of the instruction.
31039 // See: Intel® 64 and IA-32 ArchitecturesSoftware Developer’s Manual,
31040 // 8.2.3.9 Loads and Stores Are Not Reordered with Locked Instructions
31041 // 2) Using an immediate operand appears to be the best encoding choice
31042 // here since it doesn't require an extra register.
31043 // 3) OR appears to be very slightly faster than ADD. (Though, the difference
31044 // is small enough it might just be measurement noise.)
31045 // 4) When choosing offsets, there are several contributing factors:
31046 // a) If there's no redzone, we default to TOS. (We could allocate a cache
31047 // line aligned stack object to improve this case.)
31048 // b) To minimize our chances of introducing a false dependence, we prefer
31049 // to offset the stack usage from TOS slightly.
31050 // c) To minimize concerns about cross thread stack usage - in particular,
31051 // the idiomatic MyThreadPool.run([&StackVars]() {...}) pattern which
31052 // captures state in the TOS frame and accesses it from many threads -
31053 // we want to use an offset such that the offset is in a distinct cache
31054 // line from the TOS frame.
31055 //
31056 // For a general discussion of the tradeoffs and benchmark results, see:
31057 // https://shipilev.net/blog/2014/on-the-fence-with-dependencies/
31058
31059 auto &MF = DAG.getMachineFunction();
31060 auto &TFL = *Subtarget.getFrameLowering();
31061 const unsigned SPOffset = TFL.has128ByteRedZone(MF) ? -64 : 0;
31062
31063 if (Subtarget.is64Bit()) {
31064 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
31065 SDValue Ops[] = {
31066 DAG.getRegister(X86::RSP, MVT::i64), // Base
31067 DAG.getTargetConstant(1, DL, MVT::i8), // Scale
31068 DAG.getRegister(0, MVT::i64), // Index
31069 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
31070 DAG.getRegister(0, MVT::i16), // Segment.
31071 Zero,
31072 Chain};
31073 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
31074 MVT::Other, Ops);
31075 return SDValue(Res, 1);
31076 }
31077
31078 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
31079 SDValue Ops[] = {
31080 DAG.getRegister(X86::ESP, MVT::i32), // Base
31081 DAG.getTargetConstant(1, DL, MVT::i8), // Scale
31082 DAG.getRegister(0, MVT::i32), // Index
31083 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
31084 DAG.getRegister(0, MVT::i16), // Segment.
31085 Zero,
31086 Chain
31087 };
31088 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
31089 MVT::Other, Ops);
31090 return SDValue(Res, 1);
31091}
31092
31093static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
31094 SelectionDAG &DAG) {
31095 SDLoc dl(Op);
31096 AtomicOrdering FenceOrdering =
31097 static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
31098 SyncScope::ID FenceSSID =
31099 static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
31100
31101 // The only fence that needs an instruction is a sequentially-consistent
31102 // cross-thread fence.
31103 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
31104 FenceSSID == SyncScope::System) {
31105 if (Subtarget.hasMFence())
31106 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
31107
31108 SDValue Chain = Op.getOperand(0);
31109 return emitLockedStackOp(DAG, Subtarget, Chain, dl);
31110 }
31111
31112 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
31113 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
31114}
31115
31116static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
31117 SelectionDAG &DAG) {
31118 MVT T = Op.getSimpleValueType();
31119 SDLoc DL(Op);
31120 unsigned Reg = 0;
31121 unsigned size = 0;
31122 switch(T.SimpleTy) {
31123 default: llvm_unreachable("Invalid value type!");
31124 case MVT::i8: Reg = X86::AL; size = 1; break;
31125 case MVT::i16: Reg = X86::AX; size = 2; break;
31126 case MVT::i32: Reg = X86::EAX; size = 4; break;
31127 case MVT::i64:
31128 assert(Subtarget.is64Bit() && "Node not type legal!");
31129 Reg = X86::RAX; size = 8;
31130 break;
31131 }
31132 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
31133 Op.getOperand(2), SDValue());
31134 SDValue Ops[] = { cpIn.getValue(0),
31135 Op.getOperand(1),
31136 Op.getOperand(3),
31137 DAG.getTargetConstant(size, DL, MVT::i8),
31138 cpIn.getValue(1) };
31139 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
31140 MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
31141 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
31142 Ops, T, MMO);
31143
31144 SDValue cpOut =
31145 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
31146 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
31147 MVT::i32, cpOut.getValue(2));
31148 SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
31149
31150 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
31151 cpOut, Success, EFLAGS.getValue(1));
31152}
31153
31154// Create MOVMSKB, taking into account whether we need to split for AVX1.
31155static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
31156 const X86Subtarget &Subtarget) {
31157 MVT InVT = V.getSimpleValueType();
31158
31159 if (InVT == MVT::v64i8) {
31160 SDValue Lo, Hi;
31161 std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
31162 Lo = getPMOVMSKB(DL, Lo, DAG, Subtarget);
31163 Hi = getPMOVMSKB(DL, Hi, DAG, Subtarget);
31164 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo);
31165 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi);
31166 Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi,
31167 DAG.getConstant(32, DL, MVT::i8));
31168 return DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi);
31169 }
31170 if (InVT == MVT::v32i8 && !Subtarget.hasInt256()) {
31171 SDValue Lo, Hi;
31172 std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
31173 Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo);
31174 Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi);
31175 Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi,
31176 DAG.getConstant(16, DL, MVT::i8));
31177 return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi);
31178 }
31179
31180 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
31181}
31182
31183static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
31184 SelectionDAG &DAG) {
31185 SDValue Src = Op.getOperand(0);
31186 MVT SrcVT = Src.getSimpleValueType();
31187 MVT DstVT = Op.getSimpleValueType();
31188
31189 // Legalize (v64i1 (bitcast i64 (X))) by splitting the i64, bitcasting each
31190 // half to v32i1 and concatenating the result.
31191 if (SrcVT == MVT::i64 && DstVT == MVT::v64i1) {
31192 assert(!Subtarget.is64Bit() && "Expected 32-bit mode");
31193 assert(Subtarget.hasBWI() && "Expected BWI target");
31194 SDLoc dl(Op);
31195 SDValue Lo, Hi;
31196 std::tie(Lo, Hi) = DAG.SplitScalar(Src, dl, MVT::i32, MVT::i32);
31197 Lo = DAG.getBitcast(MVT::v32i1, Lo);
31198 Hi = DAG.getBitcast(MVT::v32i1, Hi);
31199 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
31200 }
31201
31202 // Use MOVMSK for vector to scalar conversion to prevent scalarization.
31203 if ((SrcVT == MVT::v16i1 || SrcVT == MVT::v32i1) && DstVT.isScalarInteger()) {
31204 assert(!Subtarget.hasAVX512() && "Should use K-registers with AVX512");
31205 MVT SExtVT = SrcVT == MVT::v16i1 ? MVT::v16i8 : MVT::v32i8;
31206 SDLoc DL(Op);
31207 SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT);
31208 V = getPMOVMSKB(DL, V, DAG, Subtarget);
31209 return DAG.getZExtOrTrunc(V, DL, DstVT);
31210 }
31211
31212 assert((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 ||
31213 SrcVT == MVT::i64) && "Unexpected VT!");
31214
31215 assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
31216 if (!(DstVT == MVT::f64 && SrcVT == MVT::i64) &&
31217 !(DstVT == MVT::x86mmx && SrcVT.isVector()))
31218 // This conversion needs to be expanded.
31219 return SDValue();
31220
31221 SDLoc dl(Op);
31222 if (SrcVT.isVector()) {
31223 // Widen the vector in input in the case of MVT::v2i32.
31224 // Example: from MVT::v2i32 to MVT::v4i32.
31225 MVT NewVT = MVT::getVectorVT(SrcVT.getVectorElementType(),
31226 SrcVT.getVectorNumElements() * 2);
31227 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src,
31228 DAG.getUNDEF(SrcVT));
31229 } else {
31230 assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() &&
31231 "Unexpected source type in LowerBITCAST");
31232 Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
31233 }
31234
31235 MVT V2X64VT = DstVT == MVT::f64 ? MVT::v2f64 : MVT::v2i64;
31236 Src = DAG.getNode(ISD::BITCAST, dl, V2X64VT, Src);
31237
31238 if (DstVT == MVT::x86mmx)
31239 return DAG.getNode(X86ISD::MOVDQ2Q, dl, DstVT, Src);
31240
31241 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, DstVT, Src,
31242 DAG.getIntPtrConstant(0, dl));
31243}
31244
31245/// Compute the horizontal sum of bytes in V for the elements of VT.
31246///
31247/// Requires V to be a byte vector and VT to be an integer vector type with
31248/// wider elements than V's type. The width of the elements of VT determines
31249/// how many bytes of V are summed horizontally to produce each element of the
31250/// result.
31251static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,
31252 const X86Subtarget &Subtarget,
31253 SelectionDAG &DAG) {
31254 SDLoc DL(V);
31255 MVT ByteVecVT = V.getSimpleValueType();
31256 MVT EltVT = VT.getVectorElementType();
31257 assert(ByteVecVT.getVectorElementType() == MVT::i8 &&
31258 "Expected value to have byte element type.");
31259 assert(EltVT != MVT::i8 &&
31260 "Horizontal byte sum only makes sense for wider elements!");
31261 unsigned VecSize = VT.getSizeInBits();
31262 assert(ByteVecVT.getSizeInBits() == VecSize && "Cannot change vector size!");
31263
31264 // PSADBW instruction horizontally add all bytes and leave the result in i64
31265 // chunks, thus directly computes the pop count for v2i64 and v4i64.
31266 if (EltVT == MVT::i64) {
31267 SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT);
31268 MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);
31269 V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros);
31270 return DAG.getBitcast(VT, V);
31271 }
31272
31273 if (EltVT == MVT::i32) {
31274 // We unpack the low half and high half into i32s interleaved with zeros so
31275 // that we can use PSADBW to horizontally sum them. The most useful part of
31276 // this is that it lines up the results of two PSADBW instructions to be
31277 // two v2i64 vectors which concatenated are the 4 population counts. We can
31278 // then use PACKUSWB to shrink and concatenate them into a v4i32 again.
31279 SDValue Zeros = DAG.getConstant(0, DL, VT);
31280 SDValue V32 = DAG.getBitcast(VT, V);
31281 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);
31282 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);
31283
31284 // Do the horizontal sums into two v2i64s.
31285 Zeros = DAG.getConstant(0, DL, ByteVecVT);
31286 MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);
31287 Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
31288 DAG.getBitcast(ByteVecVT, Low), Zeros);
31289 High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
31290 DAG.getBitcast(ByteVecVT, High), Zeros);
31291
31292 // Merge them together.
31293 MVT ShortVecVT = MVT::getVectorVT(MVT::i16, VecSize / 16);
31294 V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT,
31295 DAG.getBitcast(ShortVecVT, Low),
31296 DAG.getBitcast(ShortVecVT, High));
31297
31298 return DAG.getBitcast(VT, V);
31299 }
31300
31301 // The only element type left is i16.
31302 assert(EltVT == MVT::i16 && "Unknown how to handle type");
31303
31304 // To obtain pop count for each i16 element starting from the pop count for
31305 // i8 elements, shift the i16s left by 8, sum as i8s, and then shift as i16s
31306 // right by 8. It is important to shift as i16s as i8 vector shift isn't
31307 // directly supported.
31308 SDValue ShifterV = DAG.getConstant(8, DL, VT);
31309 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
31310 V = DAG.getNode(ISD::ADD, DL, ByteVecVT, DAG.getBitcast(ByteVecVT, Shl),
31311 DAG.getBitcast(ByteVecVT, V));
31312 return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
31313}
31314
31315static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
31316 const X86Subtarget &Subtarget,
31317 SelectionDAG &DAG) {
31318 MVT VT = Op.getSimpleValueType();
31319 MVT EltVT = VT.getVectorElementType();
31320 int NumElts = VT.getVectorNumElements();
31321 (void)EltVT;
31322 assert(EltVT == MVT::i8 && "Only vXi8 vector CTPOP lowering supported.");
31323
31324 // Implement a lookup table in register by using an algorithm based on:
31325 // http://wm.ite.pl/articles/sse-popcount.html
31326 //
31327 // The general idea is that every lower byte nibble in the input vector is an
31328 // index into a in-register pre-computed pop count table. We then split up the
31329 // input vector in two new ones: (1) a vector with only the shifted-right
31330 // higher nibbles for each byte and (2) a vector with the lower nibbles (and
31331 // masked out higher ones) for each byte. PSHUFB is used separately with both
31332 // to index the in-register table. Next, both are added and the result is a
31333 // i8 vector where each element contains the pop count for input byte.
31334 const int LUT[16] = {/* 0 */ 0, /* 1 */ 1, /* 2 */ 1, /* 3 */ 2,
31335 /* 4 */ 1, /* 5 */ 2, /* 6 */ 2, /* 7 */ 3,
31336 /* 8 */ 1, /* 9 */ 2, /* a */ 2, /* b */ 3,
31337 /* c */ 2, /* d */ 3, /* e */ 3, /* f */ 4};
31338
31339 SmallVector<SDValue, 64> LUTVec;
31340 for (int i = 0; i < NumElts; ++i)
31341 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
31342 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);
31343 SDValue M0F = DAG.getConstant(0x0F, DL, VT);
31344
31345 // High nibbles
31346 SDValue FourV = DAG.getConstant(4, DL, VT);
31347 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);
31348
31349 // Low nibbles
31350 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);
31351
31352 // The input vector is used as the shuffle mask that index elements into the
31353 // LUT. After counting low and high nibbles, add the vector to obtain the
31354 // final pop count per i8 element.
31355 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);
31356 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);
31357 return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt);
31358}
31359
31360// Please ensure that any codegen change from LowerVectorCTPOP is reflected in
31361// updated cost models in X86TTIImpl::getIntrinsicInstrCost.
31362static SDValue LowerVectorCTPOP(SDValue Op, const SDLoc &DL,
31363 const X86Subtarget &Subtarget,
31364 SelectionDAG &DAG) {
31365 MVT VT = Op.getSimpleValueType();
31366 assert((VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector()) &&
31367 "Unknown CTPOP type to handle");
31368 SDValue Op0 = Op.getOperand(0);
31369
31370 // TRUNC(CTPOP(ZEXT(X))) to make use of vXi32/vXi64 VPOPCNT instructions.
31371 if (Subtarget.hasVPOPCNTDQ()) {
31372 unsigned NumElems = VT.getVectorNumElements();
31373 assert((VT.getVectorElementType() == MVT::i8 ||
31374 VT.getVectorElementType() == MVT::i16) && "Unexpected type");
31375 if (NumElems < 16 || (NumElems == 16 && Subtarget.canExtendTo512DQ())) {
31376 MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
31377 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0);
31378 Op = DAG.getNode(ISD::CTPOP, DL, NewVT, Op);
31379 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
31380 }
31381 }
31382
31383 // Decompose 256-bit ops into smaller 128-bit ops.
31384 if (VT.is256BitVector() && !Subtarget.hasInt256())
31385 return splitVectorIntUnary(Op, DAG, DL);
31386
31387 // Decompose 512-bit ops into smaller 256-bit ops.
31388 if (VT.is512BitVector() && !Subtarget.hasBWI())
31389 return splitVectorIntUnary(Op, DAG, DL);
31390
31391 // For element types greater than i8, do vXi8 pop counts and a bytesum.
31392 if (VT.getScalarType() != MVT::i8) {
31393 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
31394 SDValue ByteOp = DAG.getBitcast(ByteVT, Op0);
31395 SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp);
31396 return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG);
31397 }
31398
31399 // We can't use the fast LUT approach, so fall back on LegalizeDAG.
31400 if (!Subtarget.hasSSSE3())
31401 return SDValue();
31402
31403 return LowerVectorCTPOPInRegLUT(Op0, DL, Subtarget, DAG);
31404}
31405
31406static SDValue LowerCTPOP(SDValue N, const X86Subtarget &Subtarget,
31407 SelectionDAG &DAG) {
31408 MVT VT = N.getSimpleValueType();
31409 SDValue Op = N.getOperand(0);
31410 SDLoc DL(N);
31411
31412 if (VT.isScalarInteger()) {
31413 // Compute the lower/upper bounds of the active bits of the value,
31414 // allowing us to shift the active bits down if necessary to fit into the
31415 // special cases below.
31416 KnownBits Known = DAG.computeKnownBits(Op);
31417 unsigned LZ = Known.countMinLeadingZeros();
31418 unsigned TZ = Known.countMinTrailingZeros();
31419 assert((LZ + TZ) < Known.getBitWidth() && "Illegal shifted mask");
31420 unsigned ActiveBits = Known.getBitWidth() - LZ;
31421 unsigned ShiftedActiveBits = Known.getBitWidth() - (LZ + TZ);
31422
31423 // i2 CTPOP - "ctpop(x) --> sub(x, (x >> 1))".
31424 if (ShiftedActiveBits <= 2) {
31425 if (ActiveBits > 2)
31426 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
31427 DAG.getShiftAmountConstant(TZ, VT, DL));
31428 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32);
31429 Op = DAG.getNode(ISD::SUB, DL, MVT::i32, Op,
31430 DAG.getNode(ISD::SRL, DL, MVT::i32, Op,
31431 DAG.getShiftAmountConstant(1, VT, DL)));
31432 return DAG.getZExtOrTrunc(Op, DL, VT);
31433 }
31434
31435 // i3 CTPOP - perform LUT into i32 integer.
31436 if (ShiftedActiveBits <= 3) {
31437 if (ActiveBits > 3)
31438 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
31439 DAG.getShiftAmountConstant(TZ, VT, DL));
31440 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32);
31441 Op = DAG.getNode(ISD::SHL, DL, MVT::i32, Op,
31442 DAG.getShiftAmountConstant(1, VT, DL));
31443 Op = DAG.getNode(ISD::SRL, DL, MVT::i32,
31444 DAG.getConstant(0b1110100110010100U, DL, MVT::i32), Op);
31445 Op = DAG.getNode(ISD::AND, DL, MVT::i32, Op,
31446 DAG.getConstant(0x3, DL, MVT::i32));
31447 return DAG.getZExtOrTrunc(Op, DL, VT);
31448 }
31449
31450 // i4 CTPOP - perform LUT into i64 integer.
31451 if (ShiftedActiveBits <= 4 &&
31452 DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64)) {
31453 SDValue LUT = DAG.getConstant(0x4332322132212110ULL, DL, MVT::i64);
31454 if (ActiveBits > 4)
31455 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
31456 DAG.getShiftAmountConstant(TZ, VT, DL));
31457 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32);
31458 Op = DAG.getNode(ISD::MUL, DL, MVT::i32, Op,
31459 DAG.getConstant(4, DL, MVT::i32));
31460 Op = DAG.getNode(ISD::SRL, DL, MVT::i64, LUT,
31461 DAG.getShiftAmountOperand(MVT::i64, Op));
31462 Op = DAG.getNode(ISD::AND, DL, MVT::i64, Op,
31463 DAG.getConstant(0x7, DL, MVT::i64));
31464 return DAG.getZExtOrTrunc(Op, DL, VT);
31465 }
31466
31467 // i8 CTPOP - with efficient i32 MUL, then attempt multiply-mask-multiply.
31468 if (ShiftedActiveBits <= 8) {
31469 SDValue Mask11 = DAG.getConstant(0x11111111U, DL, MVT::i32);
31470 if (ActiveBits > 8)
31471 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
31472 DAG.getShiftAmountConstant(TZ, VT, DL));
31473 Op = DAG.getZExtOrTrunc(Op, DL, MVT::i32);
31474 Op = DAG.getNode(ISD::MUL, DL, MVT::i32, Op,
31475 DAG.getConstant(0x08040201U, DL, MVT::i32));
31476 Op = DAG.getNode(ISD::SRL, DL, MVT::i32, Op,
31477 DAG.getShiftAmountConstant(3, MVT::i32, DL));
31478 Op = DAG.getNode(ISD::AND, DL, MVT::i32, Op, Mask11);
31479 Op = DAG.getNode(ISD::MUL, DL, MVT::i32, Op, Mask11);
31480 Op = DAG.getNode(ISD::SRL, DL, MVT::i32, Op,
31481 DAG.getShiftAmountConstant(28, MVT::i32, DL));
31482 return DAG.getZExtOrTrunc(Op, DL, VT);
31483 }
31484
31485 return SDValue(); // fallback to generic expansion.
31486 }
31487
31488 assert(VT.isVector() &&
31489 "We only do custom lowering for vector population count.");
31490 return LowerVectorCTPOP(N, DL, Subtarget, DAG);
31491}
31492
31493static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
31494 MVT VT = Op.getSimpleValueType();
31495 SDValue In = Op.getOperand(0);
31496 SDLoc DL(Op);
31497
31498 // For scalars, its still beneficial to transfer to/from the SIMD unit to
31499 // perform the BITREVERSE.
31500 if (!VT.isVector()) {
31501 MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits());
31502 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
31503 Res = DAG.getNode(ISD::BITREVERSE, DL, VecVT, Res);
31504 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res,
31505 DAG.getIntPtrConstant(0, DL));
31506 }
31507
31508 int NumElts = VT.getVectorNumElements();
31509 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
31510
31511 // Decompose 256-bit ops into smaller 128-bit ops.
31512 if (VT.is256BitVector())
31513 return splitVectorIntUnary(Op, DAG, DL);
31514
31515 assert(VT.is128BitVector() &&
31516 "Only 128-bit vector bitreverse lowering supported.");
31517
31518 // VPPERM reverses the bits of a byte with the permute Op (2 << 5), and we
31519 // perform the BSWAP in the shuffle.
31520 // Its best to shuffle using the second operand as this will implicitly allow
31521 // memory folding for multiple vectors.
31522 SmallVector<SDValue, 16> MaskElts;
31523 for (int i = 0; i != NumElts; ++i) {
31524 for (int j = ScalarSizeInBytes - 1; j >= 0; --j) {
31525 int SourceByte = 16 + (i * ScalarSizeInBytes) + j;
31526 int PermuteByte = SourceByte | (2 << 5);
31527 MaskElts.push_back(DAG.getConstant(PermuteByte, DL, MVT::i8));
31528 }
31529 }
31530
31531 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);
31532 SDValue Res = DAG.getBitcast(MVT::v16i8, In);
31533 Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8),
31534 Res, Mask);
31535 return DAG.getBitcast(VT, Res);
31536}
31537
31538static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
31539 SelectionDAG &DAG) {
31540 MVT VT = Op.getSimpleValueType();
31541
31542 if (Subtarget.hasXOP() && !VT.is512BitVector())
31543 return LowerBITREVERSE_XOP(Op, DAG);
31544
31545 assert(Subtarget.hasSSSE3() && "SSSE3 required for BITREVERSE");
31546
31547 SDValue In = Op.getOperand(0);
31548 SDLoc DL(Op);
31549
31550 // Split 512-bit ops without BWI so that we can still use the PSHUFB lowering.
31551 if (VT.is512BitVector() && !Subtarget.hasBWI())
31552 return splitVectorIntUnary(Op, DAG, DL);
31553
31554 // Decompose 256-bit ops into smaller 128-bit ops on pre-AVX2.
31555 if (VT.is256BitVector() && !Subtarget.hasInt256())
31556 return splitVectorIntUnary(Op, DAG, DL);
31557
31558 // Lower i8/i16/i32/i64 as vXi8 BITREVERSE + BSWAP
31559 if (!VT.isVector()) {
31560 assert(
31561 (VT == MVT::i32 || VT == MVT::i64 || VT == MVT::i16 || VT == MVT::i8) &&
31562 "Only tested for i8/i16/i32/i64");
31563 MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits());
31564 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
31565 Res = DAG.getNode(ISD::BITREVERSE, DL, MVT::v16i8,
31566 DAG.getBitcast(MVT::v16i8, Res));
31567 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
31568 DAG.getBitcast(VecVT, Res), DAG.getIntPtrConstant(0, DL));
31569 return (VT == MVT::i8) ? Res : DAG.getNode(ISD::BSWAP, DL, VT, Res);
31570 }
31571
31572 assert(VT.isVector() && VT.getSizeInBits() >= 128);
31573
31574 // Lower vXi16/vXi32/vXi64 as BSWAP + vXi8 BITREVERSE.
31575 if (VT.getScalarType() != MVT::i8) {
31576 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
31577 SDValue Res = DAG.getNode(ISD::BSWAP, DL, VT, In);
31578 Res = DAG.getBitcast(ByteVT, Res);
31579 Res = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Res);
31580 return DAG.getBitcast(VT, Res);
31581 }
31582 assert(VT.isVector() && VT.getScalarType() == MVT::i8 &&
31583 "Only byte vector BITREVERSE supported");
31584
31585 unsigned NumElts = VT.getVectorNumElements();
31586
31587 // If we have GFNI, we can use GF2P8AFFINEQB to reverse the bits.
31588 if (Subtarget.hasGFNI()) {
31589 SDValue Matrix = getGFNICtrlMask(ISD::BITREVERSE, DAG, DL, VT);
31590 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix,
31591 DAG.getTargetConstant(0, DL, MVT::i8));
31592 }
31593
31594 // Perform BITREVERSE using PSHUFB lookups. Each byte is split into
31595 // two nibbles and a PSHUFB lookup to find the bitreverse of each
31596 // 0-15 value (moved to the other nibble).
31597 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);
31598 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);
31599 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));
31600
31601 const int LoLUT[16] = {
31602 /* 0 */ 0x00, /* 1 */ 0x80, /* 2 */ 0x40, /* 3 */ 0xC0,
31603 /* 4 */ 0x20, /* 5 */ 0xA0, /* 6 */ 0x60, /* 7 */ 0xE0,
31604 /* 8 */ 0x10, /* 9 */ 0x90, /* a */ 0x50, /* b */ 0xD0,
31605 /* c */ 0x30, /* d */ 0xB0, /* e */ 0x70, /* f */ 0xF0};
31606 const int HiLUT[16] = {
31607 /* 0 */ 0x00, /* 1 */ 0x08, /* 2 */ 0x04, /* 3 */ 0x0C,
31608 /* 4 */ 0x02, /* 5 */ 0x0A, /* 6 */ 0x06, /* 7 */ 0x0E,
31609 /* 8 */ 0x01, /* 9 */ 0x09, /* a */ 0x05, /* b */ 0x0D,
31610 /* c */ 0x03, /* d */ 0x0B, /* e */ 0x07, /* f */ 0x0F};
31611
31612 SmallVector<SDValue, 16> LoMaskElts, HiMaskElts;
31613 for (unsigned i = 0; i < NumElts; ++i) {
31614 LoMaskElts.push_back(DAG.getConstant(LoLUT[i % 16], DL, MVT::i8));
31615 HiMaskElts.push_back(DAG.getConstant(HiLUT[i % 16], DL, MVT::i8));
31616 }
31617
31618 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);
31619 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);
31620 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo);
31621 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi);
31622 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
31623}
31624
31625static SDValue LowerPARITY(SDValue Op, const X86Subtarget &Subtarget,
31626 SelectionDAG &DAG) {
31627 SDLoc DL(Op);
31628 SDValue X = Op.getOperand(0);
31629 MVT VT = Op.getSimpleValueType();
31630
31631 // Special case. If the input fits in 8-bits we can use a single 8-bit TEST.
31632 if (VT == MVT::i8 ||
31633 DAG.MaskedValueIsZero(X, APInt::getBitsSetFrom(VT.getSizeInBits(), 8))) {
31634 X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
31635 SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X,
31636 DAG.getConstant(0, DL, MVT::i8));
31637 // Copy the inverse of the parity flag into a register with setcc.
31638 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
31639 // Extend to the original type.
31640 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp);
31641 }
31642
31643 // If we have POPCNT, use the default expansion.
31644 if (Subtarget.hasPOPCNT())
31645 return SDValue();
31646
31647 if (VT == MVT::i64) {
31648 // Xor the high and low 16-bits together using a 32-bit operation.
31649 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
31650 DAG.getNode(ISD::SRL, DL, MVT::i64, X,
31651 DAG.getConstant(32, DL, MVT::i8)));
31652 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);
31653 X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi);
31654 }
31655
31656 if (VT != MVT::i16) {
31657 // Xor the high and low 16-bits together using a 32-bit operation.
31658 SDValue Hi16 = DAG.getNode(ISD::SRL, DL, MVT::i32, X,
31659 DAG.getConstant(16, DL, MVT::i8));
31660 X = DAG.getNode(ISD::XOR, DL, MVT::i32, X, Hi16);
31661 } else {
31662 // If the input is 16-bits, we need to extend to use an i32 shift below.
31663 X = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, X);
31664 }
31665
31666 // Finally xor the low 2 bytes together and use a 8-bit flag setting xor.
31667 // This should allow an h-reg to be used to save a shift.
31668 SDValue Hi = DAG.getNode(
31669 ISD::TRUNCATE, DL, MVT::i8,
31670 DAG.getNode(ISD::SRL, DL, MVT::i32, X, DAG.getConstant(8, DL, MVT::i8)));
31671 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
31672 SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32);
31673 SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1);
31674
31675 // Copy the inverse of the parity flag into a register with setcc.
31676 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
31677 // Extend to the original type.
31678 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp);
31679}
31680
31681static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
31682 const X86Subtarget &Subtarget) {
31683 unsigned NewOpc = 0;
31684 switch (N->getOpcode()) {
31685 case ISD::ATOMIC_LOAD_ADD:
31686 NewOpc = X86ISD::LADD;
31687 break;
31688 case ISD::ATOMIC_LOAD_SUB:
31689 NewOpc = X86ISD::LSUB;
31690 break;
31691 case ISD::ATOMIC_LOAD_OR:
31692 NewOpc = X86ISD::LOR;
31693 break;
31694 case ISD::ATOMIC_LOAD_XOR:
31695 NewOpc = X86ISD::LXOR;
31696 break;
31697 case ISD::ATOMIC_LOAD_AND:
31698 NewOpc = X86ISD::LAND;
31699 break;
31700 default:
31701 llvm_unreachable("Unknown ATOMIC_LOAD_ opcode");
31702 }
31703
31704 MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand();
31705
31706 return DAG.getMemIntrinsicNode(
31707 NewOpc, SDLoc(N), DAG.getVTList(MVT::i32, MVT::Other),
31708 {N->getOperand(0), N->getOperand(1), N->getOperand(2)},
31709 /*MemVT=*/N->getSimpleValueType(0), MMO);
31710}
31711
31712/// Lower atomic_load_ops into LOCK-prefixed operations.
31713static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
31714 const X86Subtarget &Subtarget) {
31715 AtomicSDNode *AN = cast<AtomicSDNode>(N.getNode());
31716 SDValue Chain = N->getOperand(0);
31717 SDValue LHS = N->getOperand(1);
31718 SDValue RHS = N->getOperand(2);
31719 unsigned Opc = N->getOpcode();
31720 MVT VT = N->getSimpleValueType(0);
31721 SDLoc DL(N);
31722
31723 // We can lower atomic_load_add into LXADD. However, any other atomicrmw op
31724 // can only be lowered when the result is unused. They should have already
31725 // been transformed into a cmpxchg loop in AtomicExpand.
31726 if (N->hasAnyUseOfValue(0)) {
31727 // Handle (atomic_load_sub p, v) as (atomic_load_add p, -v), to be able to
31728 // select LXADD if LOCK_SUB can't be selected.
31729 // Handle (atomic_load_xor p, SignBit) as (atomic_load_add p, SignBit) so we
31730 // can use LXADD as opposed to cmpxchg.
31731 if (Opc == ISD::ATOMIC_LOAD_SUB ||
31732 (Opc == ISD::ATOMIC_LOAD_XOR && isMinSignedConstant(RHS)))
31733 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS,
31734 DAG.getNegative(RHS, DL, VT), AN->getMemOperand());
31735
31736 assert(Opc == ISD::ATOMIC_LOAD_ADD &&
31737 "Used AtomicRMW ops other than Add should have been expanded!");
31738 return N;
31739 }
31740
31741 // Specialized lowering for the canonical form of an idemptotent atomicrmw.
31742 // The core idea here is that since the memory location isn't actually
31743 // changing, all we need is a lowering for the *ordering* impacts of the
31744 // atomicrmw. As such, we can chose a different operation and memory
31745 // location to minimize impact on other code.
31746 // The above holds unless the node is marked volatile in which
31747 // case it needs to be preserved according to the langref.
31748 if (Opc == ISD::ATOMIC_LOAD_OR && isNullConstant(RHS) && !AN->isVolatile()) {
31749 // On X86, the only ordering which actually requires an instruction is
31750 // seq_cst which isn't SingleThread, everything just needs to be preserved
31751 // during codegen and then dropped. Note that we expect (but don't assume),
31752 // that orderings other than seq_cst and acq_rel have been canonicalized to
31753 // a store or load.
31754 if (AN->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent &&
31755 AN->getSyncScopeID() == SyncScope::System) {
31756 // Prefer a locked operation against a stack location to minimize cache
31757 // traffic. This assumes that stack locations are very likely to be
31758 // accessed only by the owning thread.
31759 SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
31760 assert(!N->hasAnyUseOfValue(0));
31761 // NOTE: The getUNDEF is needed to give something for the unused result 0.
31762 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
31763 DAG.getUNDEF(VT), NewChain);
31764 }
31765 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
31766 SDValue NewChain = DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Chain);
31767 assert(!N->hasAnyUseOfValue(0));
31768 // NOTE: The getUNDEF is needed to give something for the unused result 0.
31769 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
31770 DAG.getUNDEF(VT), NewChain);
31771 }
31772
31773 SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
31774 // RAUW the chain, but don't worry about the result, as it's unused.
31775 assert(!N->hasAnyUseOfValue(0));
31776 // NOTE: The getUNDEF is needed to give something for the unused result 0.
31777 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
31778 DAG.getUNDEF(VT), LockOp.getValue(1));
31779}
31780
31781static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
31782 const X86Subtarget &Subtarget) {
31783 auto *Node = cast<AtomicSDNode>(Op.getNode());
31784 SDLoc dl(Node);
31785 EVT VT = Node->getMemoryVT();
31786
31787 bool IsSeqCst =
31788 Node->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent;
31789 bool IsTypeLegal = DAG.getTargetLoweringInfo().isTypeLegal(VT);
31790
31791 // If this store is not sequentially consistent and the type is legal
31792 // we can just keep it.
31793 if (!IsSeqCst && IsTypeLegal)
31794 return Op;
31795
31796 if (!IsTypeLegal && !Subtarget.useSoftFloat() &&
31797 !DAG.getMachineFunction().getFunction().hasFnAttribute(
31798 Attribute::NoImplicitFloat)) {
31799 SDValue Chain;
31800 // For illegal i128 atomic_store, when AVX is enabled, we can simply emit a
31801 // vector store.
31802 if (VT == MVT::i128 && Subtarget.is64Bit() && Subtarget.hasAVX()) {
31803 SDValue VecVal = DAG.getBitcast(MVT::v2i64, Node->getVal());
31804 Chain = DAG.getStore(Node->getChain(), dl, VecVal, Node->getBasePtr(),
31805 Node->getMemOperand());
31806 }
31807
31808 // For illegal i64 atomic_stores, we can try to use MOVQ or MOVLPS if SSE
31809 // is enabled.
31810 if (VT == MVT::i64) {
31811 if (Subtarget.hasSSE1()) {
31812 SDValue SclToVec =
31813 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Node->getVal());
31814 MVT StVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;
31815 SclToVec = DAG.getBitcast(StVT, SclToVec);
31816 SDVTList Tys = DAG.getVTList(MVT::Other);
31817 SDValue Ops[] = {Node->getChain(), SclToVec, Node->getBasePtr()};
31818 Chain = DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops,
31819 MVT::i64, Node->getMemOperand());
31820 } else if (Subtarget.hasX87()) {
31821 // First load this into an 80-bit X87 register using a stack temporary.
31822 // This will put the whole integer into the significand.
31823 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
31824 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
31825 MachinePointerInfo MPI =
31826 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
31827 Chain = DAG.getStore(Node->getChain(), dl, Node->getVal(), StackPtr,
31828 MPI, MaybeAlign(), MachineMemOperand::MOStore);
31829 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
31830 SDValue LdOps[] = {Chain, StackPtr};
31831 SDValue Value = DAG.getMemIntrinsicNode(
31832 X86ISD::FILD, dl, Tys, LdOps, MVT::i64, MPI,
31833 /*Align*/ std::nullopt, MachineMemOperand::MOLoad);
31834 Chain = Value.getValue(1);
31835
31836 // Now use an FIST to do the atomic store.
31837 SDValue StoreOps[] = {Chain, Value, Node->getBasePtr()};
31838 Chain =
31839 DAG.getMemIntrinsicNode(X86ISD::FIST, dl, DAG.getVTList(MVT::Other),
31840 StoreOps, MVT::i64, Node->getMemOperand());
31841 }
31842 }
31843
31844 if (Chain) {
31845 // If this is a sequentially consistent store, also emit an appropriate
31846 // barrier.
31847 if (IsSeqCst)
31848 Chain = emitLockedStackOp(DAG, Subtarget, Chain, dl);
31849
31850 return Chain;
31851 }
31852 }
31853
31854 // Convert seq_cst store -> xchg
31855 // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
31856 // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
31857 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, Node->getMemoryVT(),
31858 Node->getOperand(0), Node->getOperand(2),
31859 Node->getOperand(1), Node->getMemOperand());
31860 return Swap.getValue(1);
31861}
31862
31863static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) {
31864 SDNode *N = Op.getNode();
31865 MVT VT = N->getSimpleValueType(0);
31866 unsigned Opc = Op.getOpcode();
31867
31868 // Let legalize expand this if it isn't a legal type yet.
31869 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
31870 return SDValue();
31871
31872 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
31873 SDLoc DL(N);
31874
31875 // Set the carry flag.
31876 SDValue Carry = Op.getOperand(2);
31877 EVT CarryVT = Carry.getValueType();
31878 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
31879 Carry, DAG.getAllOnesConstant(DL, CarryVT));
31880
31881 bool IsAdd = Opc == ISD::UADDO_CARRY || Opc == ISD::SADDO_CARRY;
31882 SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs,
31883 Op.getOperand(0), Op.getOperand(1),
31884 Carry.getValue(1));
31885
31886 bool IsSigned = Opc == ISD::SADDO_CARRY || Opc == ISD::SSUBO_CARRY;
31887 SDValue SetCC = getSETCC(IsSigned ? X86::COND_O : X86::COND_B,
31888 Sum.getValue(1), DL, DAG);
31889 if (N->getValueType(1) == MVT::i1)
31890 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
31891
31892 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
31893}
31894
31895static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
31896 SelectionDAG &DAG) {
31897 assert(Subtarget.isTargetDarwin() && Subtarget.is64Bit());
31898
31899 // For MacOSX, we want to call an alternative entry point: __sincos_stret,
31900 // which returns the values as { float, float } (in XMM0) or
31901 // { double, double } (which is returned in XMM0, XMM1).
31902 SDLoc dl(Op);
31903 SDValue Arg = Op.getOperand(0);
31904 EVT ArgVT = Arg.getValueType();
31905 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
31906
31907 TargetLowering::ArgListTy Args;
31908 TargetLowering::ArgListEntry Entry;
31909
31910 Entry.Node = Arg;
31911 Entry.Ty = ArgTy;
31912 Entry.IsSExt = false;
31913 Entry.IsZExt = false;
31914 Args.push_back(Entry);
31915
31916 bool isF64 = ArgVT == MVT::f64;
31917 // Only optimize x86_64 for now. i386 is a bit messy. For f32,
31918 // the small struct {f32, f32} is returned in (eax, edx). For f64,
31919 // the results are returned via SRet in memory.
31920 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
31921 RTLIB::Libcall LC = isF64 ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
31922 const char *LibcallName = TLI.getLibcallName(LC);
31923 SDValue Callee =
31924 DAG.getExternalSymbol(LibcallName, TLI.getPointerTy(DAG.getDataLayout()));
31925
31926 Type *RetTy = isF64 ? (Type *)StructType::get(ArgTy, ArgTy)
31927 : (Type *)FixedVectorType::get(ArgTy, 4);
31928
31929 TargetLowering::CallLoweringInfo CLI(DAG);
31930 CLI.setDebugLoc(dl)
31931 .setChain(DAG.getEntryNode())
31932 .setLibCallee(CallingConv::C, RetTy, Callee, std::move(Args));
31933
31934 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
31935
31936 if (isF64)
31937 // Returned in xmm0 and xmm1.
31938 return CallResult.first;
31939
31940 // Returned in bits 0:31 and 32:64 xmm0.
31941 SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
31942 CallResult.first, DAG.getIntPtrConstant(0, dl));
31943 SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
31944 CallResult.first, DAG.getIntPtrConstant(1, dl));
31945 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
31946 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);
31947}
31948
31949/// Widen a vector input to a vector of NVT. The
31950/// input vector must have the same element type as NVT.
31951static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,
31952 bool FillWithZeroes = false) {
31953 // Check if InOp already has the right width.
31954 MVT InVT = InOp.getSimpleValueType();
31955 if (InVT == NVT)
31956 return InOp;
31957
31958 if (InOp.isUndef())
31959 return DAG.getUNDEF(NVT);
31960
31961 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
31962 "input and widen element type must match");
31963
31964 unsigned InNumElts = InVT.getVectorNumElements();
31965 unsigned WidenNumElts = NVT.getVectorNumElements();
31966 assert(WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0 &&
31967 "Unexpected request for vector widening");
31968
31969 SDLoc dl(InOp);
31970 if (InOp.getOpcode() == ISD::CONCAT_VECTORS &&
31971 InOp.getNumOperands() == 2) {
31972 SDValue N1 = InOp.getOperand(1);
31973 if ((ISD::isBuildVectorAllZeros(N1.getNode()) && FillWithZeroes) ||
31974 N1.isUndef()) {
31975 InOp = InOp.getOperand(0);
31976 InVT = InOp.getSimpleValueType();
31977 InNumElts = InVT.getVectorNumElements();
31978 }
31979 }
31980 if (ISD::isBuildVectorOfConstantSDNodes(InOp.getNode()) ||
31981 ISD::isBuildVectorOfConstantFPSDNodes(InOp.getNode())) {
31982 SmallVector<SDValue, 16> Ops;
31983 for (unsigned i = 0; i < InNumElts; ++i)
31984 Ops.push_back(InOp.getOperand(i));
31985
31986 EVT EltVT = InOp.getOperand(0).getValueType();
31987
31988 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
31989 DAG.getUNDEF(EltVT);
31990 for (unsigned i = 0; i < WidenNumElts - InNumElts; ++i)
31991 Ops.push_back(FillVal);
31992 return DAG.getBuildVector(NVT, dl, Ops);
31993 }
31994 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, NVT) :
31995 DAG.getUNDEF(NVT);
31996 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NVT, FillVal,
31997 InOp, DAG.getIntPtrConstant(0, dl));
31998}
31999
32000static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
32001 SelectionDAG &DAG) {
32002 assert(Subtarget.hasAVX512() &&
32003 "MGATHER/MSCATTER are supported on AVX-512 arch only");
32004
32005 MaskedScatterSDNode *N = cast<MaskedScatterSDNode>(Op.getNode());
32006 SDValue Src = N->getValue();
32007 MVT VT = Src.getSimpleValueType();
32008 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op");
32009 SDLoc dl(Op);
32010
32011 SDValue Scale = N->getScale();
32012 SDValue Index = N->getIndex();
32013 SDValue Mask = N->getMask();
32014 SDValue Chain = N->getChain();
32015 SDValue BasePtr = N->getBasePtr();
32016
32017 if (VT == MVT::v2f32 || VT == MVT::v2i32) {
32018 assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
32019 // If the index is v2i64 and we have VLX we can use xmm for data and index.
32020 if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) {
32021 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
32022 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
32023 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT));
32024 SDVTList VTs = DAG.getVTList(MVT::Other);
32025 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
32026 return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
32027 N->getMemoryVT(), N->getMemOperand());
32028 }
32029 return SDValue();
32030 }
32031
32032 MVT IndexVT = Index.getSimpleValueType();
32033
32034 // If the index is v2i32, we're being called by type legalization and we
32035 // should just let the default handling take care of it.
32036 if (IndexVT == MVT::v2i32)
32037 return SDValue();
32038
32039 // If we don't have VLX and neither the passthru or index is 512-bits, we
32040 // need to widen until one is.
32041 if (!Subtarget.hasVLX() && !VT.is512BitVector() &&
32042 !Index.getSimpleValueType().is512BitVector()) {
32043 // Determine how much we need to widen by to get a 512-bit type.
32044 unsigned Factor = std::min(512/VT.getSizeInBits(),
32045 512/IndexVT.getSizeInBits());
32046 unsigned NumElts = VT.getVectorNumElements() * Factor;
32047
32048 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
32049 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);
32050 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
32051
32052 Src = ExtendToType(Src, VT, DAG);
32053 Index = ExtendToType(Index, IndexVT, DAG);
32054 Mask = ExtendToType(Mask, MaskVT, DAG, true);
32055 }
32056
32057 SDVTList VTs = DAG.getVTList(MVT::Other);
32058 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
32059 return DAG.getMemIntrinsicNode(X86ISD::MSCATTER, dl, VTs, Ops,
32060 N->getMemoryVT(), N->getMemOperand());
32061}
32062
32063static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
32064 SelectionDAG &DAG) {
32065
32066 MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode());
32067 MVT VT = Op.getSimpleValueType();
32068 MVT ScalarVT = VT.getScalarType();
32069 SDValue Mask = N->getMask();
32070 MVT MaskVT = Mask.getSimpleValueType();
32071 SDValue PassThru = N->getPassThru();
32072 SDLoc dl(Op);
32073
32074 // Handle AVX masked loads which don't support passthru other than 0.
32075 if (MaskVT.getVectorElementType() != MVT::i1) {
32076 // We also allow undef in the isel pattern.
32077 if (PassThru.isUndef() || ISD::isBuildVectorAllZeros(PassThru.getNode()))
32078 return Op;
32079
32080 SDValue NewLoad = DAG.getMaskedLoad(
32081 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
32082 getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(),
32083 N->getMemOperand(), N->getAddressingMode(), N->getExtensionType(),
32084 N->isExpandingLoad());
32085 // Emit a blend.
32086 SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
32087 return DAG.getMergeValues({ Select, NewLoad.getValue(1) }, dl);
32088 }
32089
32090 assert((!N->isExpandingLoad() || Subtarget.hasAVX512()) &&
32091 "Expanding masked load is supported on AVX-512 target only!");
32092
32093 assert((!N->isExpandingLoad() || ScalarVT.getSizeInBits() >= 32) &&
32094 "Expanding masked load is supported for 32 and 64-bit types only!");
32095
32096 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
32097 "Cannot lower masked load op.");
32098
32099 assert((ScalarVT.getSizeInBits() >= 32 ||
32100 (Subtarget.hasBWI() &&
32101 (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&
32102 "Unsupported masked load op.");
32103
32104 // This operation is legal for targets with VLX, but without
32105 // VLX the vector should be widened to 512 bit
32106 unsigned NumEltsInWideVec = 512 / VT.getScalarSizeInBits();
32107 MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
32108 PassThru = ExtendToType(PassThru, WideDataVT, DAG);
32109
32110 // Mask element has to be i1.
32111 assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
32112 "Unexpected mask type");
32113
32114 MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);
32115
32116 Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
32117 SDValue NewLoad = DAG.getMaskedLoad(
32118 WideDataVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
32119 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
32120 N->getExtensionType(), N->isExpandingLoad());
32121
32122 SDValue Extract =
32123 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, NewLoad.getValue(0),
32124 DAG.getIntPtrConstant(0, dl));
32125 SDValue RetOps[] = {Extract, NewLoad.getValue(1)};
32126 return DAG.getMergeValues(RetOps, dl);
32127}
32128
32129static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
32130 SelectionDAG &DAG) {
32131 MaskedStoreSDNode *N = cast<MaskedStoreSDNode>(Op.getNode());
32132 SDValue DataToStore = N->getValue();
32133 MVT VT = DataToStore.getSimpleValueType();
32134 MVT ScalarVT = VT.getScalarType();
32135 SDValue Mask = N->getMask();
32136 SDLoc dl(Op);
32137
32138 assert((!N->isCompressingStore() || Subtarget.hasAVX512()) &&
32139 "Expanding masked load is supported on AVX-512 target only!");
32140
32141 assert((!N->isCompressingStore() || ScalarVT.getSizeInBits() >= 32) &&
32142 "Expanding masked load is supported for 32 and 64-bit types only!");
32143
32144 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
32145 "Cannot lower masked store op.");
32146
32147 assert((ScalarVT.getSizeInBits() >= 32 ||
32148 (Subtarget.hasBWI() &&
32149 (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&
32150 "Unsupported masked store op.");
32151
32152 // This operation is legal for targets with VLX, but without
32153 // VLX the vector should be widened to 512 bit
32154 unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits();
32155 MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
32156
32157 // Mask element has to be i1.
32158 assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
32159 "Unexpected mask type");
32160
32161 MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);
32162
32163 DataToStore = ExtendToType(DataToStore, WideDataVT, DAG);
32164 Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
32165 return DAG.getMaskedStore(N->getChain(), dl, DataToStore, N->getBasePtr(),
32166 N->getOffset(), Mask, N->getMemoryVT(),
32167 N->getMemOperand(), N->getAddressingMode(),
32168 N->isTruncatingStore(), N->isCompressingStore());
32169}
32170
32171static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
32172 SelectionDAG &DAG) {
32173 assert(Subtarget.hasAVX2() &&
32174 "MGATHER/MSCATTER are supported on AVX-512/AVX-2 arch only");
32175
32176 MaskedGatherSDNode *N = cast<MaskedGatherSDNode>(Op.getNode());
32177 SDLoc dl(Op);
32178 MVT VT = Op.getSimpleValueType();
32179 SDValue Index = N->getIndex();
32180 SDValue Mask = N->getMask();
32181 SDValue PassThru = N->getPassThru();
32182 MVT IndexVT = Index.getSimpleValueType();
32183
32184 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported gather op");
32185
32186 // If the index is v2i32, we're being called by type legalization.
32187 if (IndexVT == MVT::v2i32)
32188 return SDValue();
32189
32190 // If we don't have VLX and neither the passthru or index is 512-bits, we
32191 // need to widen until one is.
32192 MVT OrigVT = VT;
32193 if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
32194 !IndexVT.is512BitVector()) {
32195 // Determine how much we need to widen by to get a 512-bit type.
32196 unsigned Factor = std::min(512/VT.getSizeInBits(),
32197 512/IndexVT.getSizeInBits());
32198
32199 unsigned NumElts = VT.getVectorNumElements() * Factor;
32200
32201 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
32202 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);
32203 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
32204
32205 PassThru = ExtendToType(PassThru, VT, DAG);
32206 Index = ExtendToType(Index, IndexVT, DAG);
32207 Mask = ExtendToType(Mask, MaskVT, DAG, true);
32208 }
32209
32210 // Break dependency on the data register.
32211 if (PassThru.isUndef())
32212 PassThru = getZeroVector(VT, Subtarget, DAG, dl);
32213
32214 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
32215 N->getScale() };
32216 SDValue NewGather = DAG.getMemIntrinsicNode(
32217 X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(),
32218 N->getMemOperand());
32219 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT,
32220 NewGather, DAG.getIntPtrConstant(0, dl));
32221 return DAG.getMergeValues({Extract, NewGather.getValue(1)}, dl);
32222}
32223
32224static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
32225 SDLoc dl(Op);
32226 SDValue Src = Op.getOperand(0);
32227 MVT DstVT = Op.getSimpleValueType();
32228
32229 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Op.getNode());
32230 unsigned SrcAS = N->getSrcAddressSpace();
32231
32232 assert(SrcAS != N->getDestAddressSpace() &&
32233 "addrspacecast must be between different address spaces");
32234
32235 if (SrcAS == X86AS::PTR32_UPTR && DstVT == MVT::i64) {
32236 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Src);
32237 } else if (DstVT == MVT::i64) {
32238 Op = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Src);
32239 } else if (DstVT == MVT::i32) {
32240 Op = DAG.getNode(ISD::TRUNCATE, dl, DstVT, Src);
32241 } else {
32242 report_fatal_error("Bad address space in addrspacecast");
32243 }
32244 return Op;
32245}
32246
32247SDValue X86TargetLowering::LowerGC_TRANSITION(SDValue Op,
32248 SelectionDAG &DAG) const {
32249 // TODO: Eventually, the lowering of these nodes should be informed by or
32250 // deferred to the GC strategy for the function in which they appear. For
32251 // now, however, they must be lowered to something. Since they are logically
32252 // no-ops in the case of a null GC strategy (or a GC strategy which does not
32253 // require special handling for these nodes), lower them as literal NOOPs for
32254 // the time being.
32255 SmallVector<SDValue, 2> Ops;
32256 Ops.push_back(Op.getOperand(0));
32257 if (Op->getGluedNode())
32258 Ops.push_back(Op->getOperand(Op->getNumOperands() - 1));
32259
32260 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
32261 return SDValue(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
32262}
32263
32264// Custom split CVTPS2PH with wide types.
32265static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG) {
32266 SDLoc dl(Op);
32267 EVT VT = Op.getValueType();
32268 SDValue Lo, Hi;
32269 std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
32270 EVT LoVT, HiVT;
32271 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
32272 SDValue RC = Op.getOperand(1);
32273 Lo = DAG.getNode(X86ISD::CVTPS2PH, dl, LoVT, Lo, RC);
32274 Hi = DAG.getNode(X86ISD::CVTPS2PH, dl, HiVT, Hi, RC);
32275 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32276}
32277
32278static SDValue LowerPREFETCH(SDValue Op, const X86Subtarget &Subtarget,
32279 SelectionDAG &DAG) {
32280 unsigned IsData = Op.getConstantOperandVal(4);
32281
32282 // We don't support non-data prefetch without PREFETCHI.
32283 // Just preserve the chain.
32284 if (!IsData && !Subtarget.hasPREFETCHI())
32285 return Op.getOperand(0);
32286
32287 return Op;
32288}
32289
32290static StringRef getInstrStrFromOpNo(const SmallVectorImpl<StringRef> &AsmStrs,
32291 unsigned OpNo) {
32292 const APInt Operand(32, OpNo);
32293 std::string OpNoStr = llvm::toString(Operand, 10, false);
32294 std::string Str(" $");
32295
32296 std::string OpNoStr1(Str + OpNoStr); // e.g. " $1" (OpNo=1)
32297 std::string OpNoStr2(Str + "{" + OpNoStr + ":"); // With modifier, e.g. ${1:P}
32298
32299 auto I = StringRef::npos;
32300 for (auto &AsmStr : AsmStrs) {
32301 // Match the OpNo string. We should match exactly to exclude match
32302 // sub-string, e.g. "$12" contain "$1"
32303 if (AsmStr.ends_with(OpNoStr1))
32304 I = AsmStr.size() - OpNoStr1.size();
32305
32306 // Get the index of operand in AsmStr.
32307 if (I == StringRef::npos)
32308 I = AsmStr.find(OpNoStr1 + ",");
32309 if (I == StringRef::npos)
32310 I = AsmStr.find(OpNoStr2);
32311
32312 if (I == StringRef::npos)
32313 continue;
32314
32315 assert(I > 0 && "Unexpected inline asm string!");
32316 // Remove the operand string and label (if exsit).
32317 // For example:
32318 // ".L__MSASMLABEL_.${:uid}__l:call dword ptr ${0:P}"
32319 // ==>
32320 // ".L__MSASMLABEL_.${:uid}__l:call dword ptr "
32321 // ==>
32322 // "call dword ptr "
32323 auto TmpStr = AsmStr.substr(0, I);
32324 I = TmpStr.rfind(':');
32325 if (I != StringRef::npos)
32326 TmpStr = TmpStr.substr(I + 1);
32327 return TmpStr.take_while(llvm::isAlpha);
32328 }
32329
32330 return StringRef();
32331}
32332
32333bool X86TargetLowering::isInlineAsmTargetBranch(
32334 const SmallVectorImpl<StringRef> &AsmStrs, unsigned OpNo) const {
32335 // In a __asm block, __asm inst foo where inst is CALL or JMP should be
32336 // changed from indirect TargetLowering::C_Memory to direct
32337 // TargetLowering::C_Address.
32338 // We don't need to special case LOOP* and Jcc, which cannot target a memory
32339 // location.
32340 StringRef Inst = getInstrStrFromOpNo(AsmStrs, OpNo);
32341 return Inst.equals_insensitive("call") || Inst.equals_insensitive("jmp");
32342}
32343
32344static SDValue getFlagsOfCmpZeroFori1(SelectionDAG &DAG, const SDLoc &DL,
32345 SDValue Mask) {
32346 EVT Ty = MVT::i8;
32347 auto V = DAG.getBitcast(MVT::i1, Mask);
32348 auto VE = DAG.getZExtOrTrunc(V, DL, Ty);
32349 auto Zero = DAG.getConstant(0, DL, Ty);
32350 SDVTList X86SubVTs = DAG.getVTList(Ty, MVT::i32);
32351 auto CmpZero = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, VE);
32352 return SDValue(CmpZero.getNode(), 1);
32353}
32354
32355SDValue X86TargetLowering::visitMaskedLoad(
32356 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO,
32357 SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const {
32358 // @llvm.masked.load.v1*(ptr, alignment, mask, passthru)
32359 // ->
32360 // _, flags = SUB 0, mask
32361 // res, chain = CLOAD inchain, ptr, (bit_cast_to_scalar passthru), cond, flags
32362 // bit_cast_to_vector<res>
32363 EVT VTy = PassThru.getValueType();
32364 EVT Ty = VTy.getVectorElementType();
32365 SDVTList Tys = DAG.getVTList(Ty, MVT::Other);
32366 auto ScalarPassThru = PassThru.isUndef() ? DAG.getConstant(0, DL, Ty)
32367 : DAG.getBitcast(Ty, PassThru);
32368 auto Flags = getFlagsOfCmpZeroFori1(DAG, DL, Mask);
32369 auto COND_NE = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
32370 SDValue Ops[] = {Chain, Ptr, ScalarPassThru, COND_NE, Flags};
32371 NewLoad = DAG.getMemIntrinsicNode(X86ISD::CLOAD, DL, Tys, Ops, Ty, MMO);
32372 return DAG.getBitcast(VTy, NewLoad);
32373}
32374
32375SDValue X86TargetLowering::visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL,
32376 SDValue Chain,
32377 MachineMemOperand *MMO, SDValue Ptr,
32378 SDValue Val, SDValue Mask) const {
32379 // llvm.masked.store.v1*(Src0, Ptr, alignment, Mask)
32380 // ->
32381 // _, flags = SUB 0, mask
32382 // chain = CSTORE inchain, (bit_cast_to_scalar val), ptr, cond, flags
32383 EVT Ty = Val.getValueType().getVectorElementType();
32384 SDVTList Tys = DAG.getVTList(MVT::Other);
32385 auto ScalarVal = DAG.getBitcast(Ty, Val);
32386 auto Flags = getFlagsOfCmpZeroFori1(DAG, DL, Mask);
32387 auto COND_NE = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
32388 SDValue Ops[] = {Chain, ScalarVal, Ptr, COND_NE, Flags};
32389 return DAG.getMemIntrinsicNode(X86ISD::CSTORE, DL, Tys, Ops, Ty, MMO);
32390}
32391
32392/// Provide custom lowering hooks for some operations.
32393SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
32394 switch (Op.getOpcode()) {
32395 // clang-format off
32396 default: llvm_unreachable("Should not custom lower this!");
32397 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG);
32398 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
32399 return LowerCMP_SWAP(Op, Subtarget, DAG);
32400 case ISD::CTPOP: return LowerCTPOP(Op, Subtarget, DAG);
32401 case ISD::ATOMIC_LOAD_ADD:
32402 case ISD::ATOMIC_LOAD_SUB:
32403 case ISD::ATOMIC_LOAD_OR:
32404 case ISD::ATOMIC_LOAD_XOR:
32405 case ISD::ATOMIC_LOAD_AND: return lowerAtomicArith(Op, DAG, Subtarget);
32406 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op, DAG, Subtarget);
32407 case ISD::BITREVERSE: return LowerBITREVERSE(Op, Subtarget, DAG);
32408 case ISD::PARITY: return LowerPARITY(Op, Subtarget, DAG);
32409 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
32410 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, Subtarget, DAG);
32411 case ISD::VECTOR_SHUFFLE: return lowerVECTOR_SHUFFLE(Op, Subtarget, DAG);
32412 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
32413 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
32414 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
32415 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
32416 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
32417 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, Subtarget,DAG);
32418 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
32419 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
32420 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
32421 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
32422 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
32423 case ISD::SHL_PARTS:
32424 case ISD::SRA_PARTS:
32425 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
32426 case ISD::FSHL:
32427 case ISD::FSHR: return LowerFunnelShift(Op, Subtarget, DAG);
32428 case ISD::STRICT_SINT_TO_FP:
32429 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
32430 case ISD::STRICT_UINT_TO_FP:
32431 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
32432 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
32433 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, Subtarget, DAG);
32434 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, Subtarget, DAG);
32435 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, Subtarget, DAG);
32436 case ISD::ZERO_EXTEND_VECTOR_INREG:
32437 case ISD::SIGN_EXTEND_VECTOR_INREG:
32438 return LowerEXTEND_VECTOR_INREG(Op, Subtarget, DAG);
32439 case ISD::FP_TO_SINT:
32440 case ISD::STRICT_FP_TO_SINT:
32441 case ISD::FP_TO_UINT:
32442 case ISD::STRICT_FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
32443 case ISD::FP_TO_SINT_SAT:
32444 case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG);
32445 case ISD::FP_EXTEND:
32446 case ISD::STRICT_FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
32447 case ISD::FP_ROUND:
32448 case ISD::STRICT_FP_ROUND: return LowerFP_ROUND(Op, DAG);
32449 case ISD::FP16_TO_FP:
32450 case ISD::STRICT_FP16_TO_FP: return LowerFP16_TO_FP(Op, DAG);
32451 case ISD::FP_TO_FP16:
32452 case ISD::STRICT_FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
32453 case ISD::FP_TO_BF16: return LowerFP_TO_BF16(Op, DAG);
32454 case ISD::LOAD: return LowerLoad(Op, Subtarget, DAG);
32455 case ISD::STORE: return LowerStore(Op, Subtarget, DAG);
32456 case ISD::FADD:
32457 case ISD::FSUB: return lowerFaddFsub(Op, DAG);
32458 case ISD::FROUND: return LowerFROUND(Op, DAG);
32459 case ISD::FABS:
32460 case ISD::FNEG: return LowerFABSorFNEG(Op, DAG);
32461 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
32462 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG);
32463 case ISD::LRINT:
32464 case ISD::LLRINT: return LowerLRINT_LLRINT(Op, DAG);
32465 case ISD::SETCC:
32466 case ISD::STRICT_FSETCC:
32467 case ISD::STRICT_FSETCCS: return LowerSETCC(Op, DAG);
32468 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
32469 case ISD::SELECT: return LowerSELECT(Op, DAG);
32470 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
32471 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
32472 case ISD::VASTART: return LowerVASTART(Op, DAG);
32473 case ISD::VAARG: return LowerVAARG(Op, DAG);
32474 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG);
32475 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
32476 case ISD::INTRINSIC_VOID:
32477 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG);
32478 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
32479 case ISD::ADDROFRETURNADDR: return LowerADDROFRETURNADDR(Op, DAG);
32480 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
32481 case ISD::FRAME_TO_ARGS_OFFSET:
32482 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
32483 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
32484 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
32485 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
32486 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
32487 case ISD::EH_SJLJ_SETUP_DISPATCH:
32488 return lowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
32489 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
32490 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
32491 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
32492 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG);
32493 case ISD::GET_FPENV_MEM: return LowerGET_FPENV_MEM(Op, DAG);
32494 case ISD::SET_FPENV_MEM: return LowerSET_FPENV_MEM(Op, DAG);
32495 case ISD::RESET_FPENV: return LowerRESET_FPENV(Op, DAG);
32496 case ISD::CTLZ:
32497 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ(Op, Subtarget, DAG);
32498 case ISD::CTTZ:
32499 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op, Subtarget, DAG);
32500 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG);
32501 case ISD::MULHS:
32502 case ISD::MULHU: return LowerMULH(Op, Subtarget, DAG);
32503 case ISD::ROTL:
32504 case ISD::ROTR: return LowerRotate(Op, Subtarget, DAG);
32505 case ISD::SRA:
32506 case ISD::SRL:
32507 case ISD::SHL: return LowerShift(Op, Subtarget, DAG);
32508 case ISD::SADDO:
32509 case ISD::UADDO:
32510 case ISD::SSUBO:
32511 case ISD::USUBO: return LowerXALUO(Op, DAG);
32512 case ISD::SMULO:
32513 case ISD::UMULO: return LowerMULO(Op, Subtarget, DAG);
32514 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
32515 case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG);
32516 case ISD::SADDO_CARRY:
32517 case ISD::SSUBO_CARRY:
32518 case ISD::UADDO_CARRY:
32519 case ISD::USUBO_CARRY: return LowerADDSUBO_CARRY(Op, DAG);
32520 case ISD::ADD:
32521 case ISD::SUB: return lowerAddSub(Op, DAG, Subtarget);
32522 case ISD::UADDSAT:
32523 case ISD::SADDSAT:
32524 case ISD::USUBSAT:
32525 case ISD::SSUBSAT: return LowerADDSAT_SUBSAT(Op, DAG, Subtarget);
32526 case ISD::SMAX:
32527 case ISD::SMIN:
32528 case ISD::UMAX:
32529 case ISD::UMIN: return LowerMINMAX(Op, Subtarget, DAG);
32530 case ISD::FMINIMUM:
32531 case ISD::FMAXIMUM:
32532 return LowerFMINIMUM_FMAXIMUM(Op, Subtarget, DAG);
32533 case ISD::ABS: return LowerABS(Op, Subtarget, DAG);
32534 case ISD::ABDS:
32535 case ISD::ABDU: return LowerABD(Op, Subtarget, DAG);
32536 case ISD::AVGCEILU: return LowerAVG(Op, Subtarget, DAG);
32537 case ISD::FSINCOS: return LowerFSINCOS(Op, Subtarget, DAG);
32538 case ISD::MLOAD: return LowerMLOAD(Op, Subtarget, DAG);
32539 case ISD::MSTORE: return LowerMSTORE(Op, Subtarget, DAG);
32540 case ISD::MGATHER: return LowerMGATHER(Op, Subtarget, DAG);
32541 case ISD::MSCATTER: return LowerMSCATTER(Op, Subtarget, DAG);
32542 case ISD::GC_TRANSITION_START:
32543 case ISD::GC_TRANSITION_END: return LowerGC_TRANSITION(Op, DAG);
32544 case ISD::ADDRSPACECAST: return LowerADDRSPACECAST(Op, DAG);
32545 case X86ISD::CVTPS2PH: return LowerCVTPS2PH(Op, DAG);
32546 case ISD::PREFETCH: return LowerPREFETCH(Op, Subtarget, DAG);
32547 // clang-format on
32548 }
32549}
32550
32551/// Replace a node with an illegal result type with a new node built out of
32552/// custom code.
32553void X86TargetLowering::ReplaceNodeResults(SDNode *N,
32554 SmallVectorImpl<SDValue>&Results,
32555 SelectionDAG &DAG) const {
32556 SDLoc dl(N);
32557 switch (N->getOpcode()) {
32558 default:
32559#ifndef NDEBUG
32560 dbgs() << "ReplaceNodeResults: ";
32561 N->dump(&DAG);
32562#endif
32563 llvm_unreachable("Do not know how to custom type legalize this operation!");
32564 case X86ISD::CVTPH2PS: {
32565 EVT VT = N->getValueType(0);
32566 SDValue Lo, Hi;
32567 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
32568 EVT LoVT, HiVT;
32569 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
32570 Lo = DAG.getNode(X86ISD::CVTPH2PS, dl, LoVT, Lo);
32571 Hi = DAG.getNode(X86ISD::CVTPH2PS, dl, HiVT, Hi);
32572 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32573 Results.push_back(Res);
32574 return;
32575 }
32576 case X86ISD::STRICT_CVTPH2PS: {
32577 EVT VT = N->getValueType(0);
32578 SDValue Lo, Hi;
32579 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 1);
32580 EVT LoVT, HiVT;
32581 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
32582 Lo = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {LoVT, MVT::Other},
32583 {N->getOperand(0), Lo});
32584 Hi = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {HiVT, MVT::Other},
32585 {N->getOperand(0), Hi});
32586 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
32587 Lo.getValue(1), Hi.getValue(1));
32588 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32589 Results.push_back(Res);
32590 Results.push_back(Chain);
32591 return;
32592 }
32593 case X86ISD::CVTPS2PH:
32594 Results.push_back(LowerCVTPS2PH(SDValue(N, 0), DAG));
32595 return;
32596 case ISD::CTPOP: {
32597 assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
32598 // If we have at most 32 active bits, then perform as i32 CTPOP.
32599 // TODO: Perform this in generic legalizer?
32600 KnownBits Known = DAG.computeKnownBits(N->getOperand(0));
32601 unsigned LZ = Known.countMinLeadingZeros();
32602 unsigned TZ = Known.countMinTrailingZeros();
32603 if ((LZ + TZ) >= 32) {
32604 SDValue Op = DAG.getNode(ISD::SRL, dl, MVT::i64, N->getOperand(0),
32605 DAG.getShiftAmountConstant(TZ, MVT::i64, dl));
32606 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Op);
32607 Op = DAG.getNode(ISD::CTPOP, dl, MVT::i32, Op);
32608 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Op);
32609 Results.push_back(Op);
32610 return;
32611 }
32612 // Use a v2i64 if possible.
32613 bool NoImplicitFloatOps =
32614 DAG.getMachineFunction().getFunction().hasFnAttribute(
32615 Attribute::NoImplicitFloat);
32616 if (isTypeLegal(MVT::v2i64) && !NoImplicitFloatOps) {
32617 SDValue Wide =
32618 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, N->getOperand(0));
32619 Wide = DAG.getNode(ISD::CTPOP, dl, MVT::v2i64, Wide);
32620 // Bit count should fit in 32-bits, extract it as that and then zero
32621 // extend to i64. Otherwise we end up extracting bits 63:32 separately.
32622 Wide = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Wide);
32623 Wide = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Wide,
32624 DAG.getIntPtrConstant(0, dl));
32625 Wide = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Wide);
32626 Results.push_back(Wide);
32627 }
32628 return;
32629 }
32630 case ISD::MUL: {
32631 EVT VT = N->getValueType(0);
32632 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32633 VT.getVectorElementType() == MVT::i8 && "Unexpected VT!");
32634 // Pre-promote these to vXi16 to avoid op legalization thinking all 16
32635 // elements are needed.
32636 MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
32637 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
32638 SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
32639 SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
32640 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
32641 unsigned NumConcats = 16 / VT.getVectorNumElements();
32642 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
32643 ConcatOps[0] = Res;
32644 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps);
32645 Results.push_back(Res);
32646 return;
32647 }
32648 case ISD::SMULO:
32649 case ISD::UMULO: {
32650 EVT VT = N->getValueType(0);
32651 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32652 VT == MVT::v2i32 && "Unexpected VT!");
32653 bool IsSigned = N->getOpcode() == ISD::SMULO;
32654 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
32655 SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0));
32656 SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1));
32657 SDValue Res = DAG.getNode(ISD::MUL, dl, MVT::v2i64, Op0, Op1);
32658 // Extract the high 32 bits from each result using PSHUFD.
32659 // TODO: Could use SRL+TRUNCATE but that doesn't become a PSHUFD.
32660 SDValue Hi = DAG.getBitcast(MVT::v4i32, Res);
32661 Hi = DAG.getVectorShuffle(MVT::v4i32, dl, Hi, Hi, {1, 3, -1, -1});
32662 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Hi,
32663 DAG.getIntPtrConstant(0, dl));
32664
32665 // Truncate the low bits of the result. This will become PSHUFD.
32666 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
32667
32668 SDValue HiCmp;
32669 if (IsSigned) {
32670 // SMULO overflows if the high bits don't match the sign of the low.
32671 HiCmp = DAG.getNode(ISD::SRA, dl, VT, Res, DAG.getConstant(31, dl, VT));
32672 } else {
32673 // UMULO overflows if the high bits are non-zero.
32674 HiCmp = DAG.getConstant(0, dl, VT);
32675 }
32676 SDValue Ovf = DAG.getSetCC(dl, N->getValueType(1), Hi, HiCmp, ISD::SETNE);
32677
32678 // Widen the result with by padding with undef.
32679 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
32680 DAG.getUNDEF(VT));
32681 Results.push_back(Res);
32682 Results.push_back(Ovf);
32683 return;
32684 }
32685 case X86ISD::VPMADDWD: {
32686 // Legalize types for X86ISD::VPMADDWD by widening.
32687 assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
32688
32689 EVT VT = N->getValueType(0);
32690 EVT InVT = N->getOperand(0).getValueType();
32691 assert(VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 &&
32692 "Expected a VT that divides into 128 bits.");
32693 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32694 "Unexpected type action!");
32695 unsigned NumConcat = 128 / InVT.getSizeInBits();
32696
32697 EVT InWideVT = EVT::getVectorVT(*DAG.getContext(),
32698 InVT.getVectorElementType(),
32699 NumConcat * InVT.getVectorNumElements());
32700 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
32701 VT.getVectorElementType(),
32702 NumConcat * VT.getVectorNumElements());
32703
32704 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
32705 Ops[0] = N->getOperand(0);
32706 SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
32707 Ops[0] = N->getOperand(1);
32708 SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
32709
32710 SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1);
32711 Results.push_back(Res);
32712 return;
32713 }
32714 // We might have generated v2f32 FMIN/FMAX operations. Widen them to v4f32.
32715 case X86ISD::FMINC:
32716 case X86ISD::FMIN:
32717 case X86ISD::FMAXC:
32718 case X86ISD::FMAX: {
32719 EVT VT = N->getValueType(0);
32720 assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX.");
32721 SDValue UNDEF = DAG.getUNDEF(VT);
32722 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
32723 N->getOperand(0), UNDEF);
32724 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
32725 N->getOperand(1), UNDEF);
32726 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS));
32727 return;
32728 }
32729 case ISD::SDIV:
32730 case ISD::UDIV:
32731 case ISD::SREM:
32732 case ISD::UREM: {
32733 EVT VT = N->getValueType(0);
32734 if (VT.isVector()) {
32735 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
32736 "Unexpected type action!");
32737 // If this RHS is a constant splat vector we can widen this and let
32738 // division/remainder by constant optimize it.
32739 // TODO: Can we do something for non-splat?
32740 APInt SplatVal;
32741 if (ISD::isConstantSplatVector(N->getOperand(1).getNode(), SplatVal)) {
32742 unsigned NumConcats = 128 / VT.getSizeInBits();
32743 SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
32744 Ops0[0] = N->getOperand(0);
32745 EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT);
32746 SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
32747 SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
32748 SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1);
32749 Results.push_back(Res);
32750 }
32751 return;
32752 }
32753
32754 SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
32755 Results.push_back(V);
32756 return;
32757 }
32758 case ISD::TRUNCATE: {
32759 MVT VT = N->getSimpleValueType(0);
32760 if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
32761 return;
32762
32763 // The generic legalizer will try to widen the input type to the same
32764 // number of elements as the widened result type. But this isn't always
32765 // the best thing so do some custom legalization to avoid some cases.
32766 MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT();
32767 SDValue In = N->getOperand(0);
32768 EVT InVT = In.getValueType();
32769 EVT InEltVT = InVT.getVectorElementType();
32770 EVT EltVT = VT.getVectorElementType();
32771 unsigned MinElts = VT.getVectorNumElements();
32772 unsigned WidenNumElts = WidenVT.getVectorNumElements();
32773 unsigned InBits = InVT.getSizeInBits();
32774
32775 // See if there are sufficient leading bits to perform a PACKUS/PACKSS.
32776 unsigned PackOpcode;
32777 if (SDValue Src =
32778 matchTruncateWithPACK(PackOpcode, VT, In, dl, DAG, Subtarget)) {
32779 if (SDValue Res = truncateVectorWithPACK(PackOpcode, VT, Src,
32780 dl, DAG, Subtarget)) {
32781 Res = widenSubVector(WidenVT, Res, false, Subtarget, DAG, dl);
32782 Results.push_back(Res);
32783 return;
32784 }
32785 }
32786
32787 if ((128 % InBits) == 0 && WidenVT.is128BitVector()) {
32788 // 128 bit and smaller inputs should avoid truncate all together and
32789 // use a shuffle.
32790 if ((InEltVT.getSizeInBits() % EltVT.getSizeInBits()) == 0) {
32791 int Scale = InEltVT.getSizeInBits() / EltVT.getSizeInBits();
32792 SmallVector<int, 16> TruncMask(WidenNumElts, -1);
32793 for (unsigned I = 0; I < MinElts; ++I)
32794 TruncMask[I] = Scale * I;
32795 SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl, 128);
32796 assert(isTypeLegal(WidenVT) && isTypeLegal(WidenIn.getValueType()) &&
32797 "Illegal vector type in truncation");
32798 WidenIn = DAG.getBitcast(WidenVT, WidenIn);
32799 Results.push_back(
32800 DAG.getVectorShuffle(WidenVT, dl, WidenIn, WidenIn, TruncMask));
32801 return;
32802 }
32803 }
32804
32805 // With AVX512 there are some cases that can use a target specific
32806 // truncate node to go from 256/512 to less than 128 with zeros in the
32807 // upper elements of the 128 bit result.
32808 if (Subtarget.hasAVX512() && isTypeLegal(InVT)) {
32809 // We can use VTRUNC directly if for 256 bits with VLX or for any 512.
32810 if ((InBits == 256 && Subtarget.hasVLX()) || InBits == 512) {
32811 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
32812 return;
32813 }
32814 // There's one case we can widen to 512 bits and use VTRUNC.
32815 if (InVT == MVT::v4i64 && VT == MVT::v4i8 && isTypeLegal(MVT::v8i64)) {
32816 In = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i64, In,
32817 DAG.getUNDEF(MVT::v4i64));
32818 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
32819 return;
32820 }
32821 }
32822 if (Subtarget.hasVLX() && InVT == MVT::v8i64 && VT == MVT::v8i8 &&
32823 getTypeAction(*DAG.getContext(), InVT) == TypeSplitVector &&
32824 isTypeLegal(MVT::v4i64)) {
32825 // Input needs to be split and output needs to widened. Let's use two
32826 // VTRUNCs, and shuffle their results together into the wider type.
32827 SDValue Lo, Hi;
32828 std::tie(Lo, Hi) = DAG.SplitVector(In, dl);
32829
32830 Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo);
32831 Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi);
32832 SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
32833 { 0, 1, 2, 3, 16, 17, 18, 19,
32834 -1, -1, -1, -1, -1, -1, -1, -1 });
32835 Results.push_back(Res);
32836 return;
32837 }
32838
32839 // Attempt to widen the truncation input vector to let LowerTRUNCATE handle
32840 // this via type legalization.
32841 if ((InEltVT == MVT::i16 || InEltVT == MVT::i32 || InEltVT == MVT::i64) &&
32842 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32) &&
32843 (!Subtarget.hasSSSE3() ||
32844 (!isTypeLegal(InVT) &&
32845 !(MinElts <= 4 && InEltVT == MVT::i64 && EltVT == MVT::i8)))) {
32846 SDValue WidenIn = widenSubVector(In, false, Subtarget, DAG, dl,
32847 InEltVT.getSizeInBits() * WidenNumElts);
32848 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, WidenVT, WidenIn));
32849 return;
32850 }
32851
32852 return;
32853 }
32854 case ISD::ANY_EXTEND:
32855 // Right now, only MVT::v8i8 has Custom action for an illegal type.
32856 // It's intended to custom handle the input type.
32857 assert(N->getValueType(0) == MVT::v8i8 &&
32858 "Do not know how to legalize this Node");
32859 return;
32860 case ISD::SIGN_EXTEND:
32861 case ISD::ZERO_EXTEND: {
32862 EVT VT = N->getValueType(0);
32863 SDValue In = N->getOperand(0);
32864 EVT InVT = In.getValueType();
32865 if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
32866 (InVT == MVT::v4i16 || InVT == MVT::v4i8)){
32867 assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
32868 "Unexpected type action!");
32869 assert(N->getOpcode() == ISD::SIGN_EXTEND && "Unexpected opcode");
32870 // Custom split this so we can extend i8/i16->i32 invec. This is better
32871 // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using
32872 // sra. Then extending from i32 to i64 using pcmpgt. By custom splitting
32873 // we allow the sra from the extend to i32 to be shared by the split.
32874 In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In);
32875
32876 // Fill a vector with sign bits for each element.
32877 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
32878 SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT);
32879
32880 // Create an unpackl and unpackh to interleave the sign bits then bitcast
32881 // to v2i64.
32882 SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
32883 {0, 4, 1, 5});
32884 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo);
32885 SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
32886 {2, 6, 3, 7});
32887 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi);
32888
32889 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32890 Results.push_back(Res);
32891 return;
32892 }
32893
32894 if (VT == MVT::v16i32 || VT == MVT::v8i64) {
32895 if (!InVT.is128BitVector()) {
32896 // Not a 128 bit vector, but maybe type legalization will promote
32897 // it to 128 bits.
32898 if (getTypeAction(*DAG.getContext(), InVT) != TypePromoteInteger)
32899 return;
32900 InVT = getTypeToTransformTo(*DAG.getContext(), InVT);
32901 if (!InVT.is128BitVector())
32902 return;
32903
32904 // Promote the input to 128 bits. Type legalization will turn this into
32905 // zext_inreg/sext_inreg.
32906 In = DAG.getNode(N->getOpcode(), dl, InVT, In);
32907 }
32908
32909 // Perform custom splitting instead of the two stage extend we would get
32910 // by default.
32911 EVT LoVT, HiVT;
32912 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
32913 assert(isTypeLegal(LoVT) && "Split VT not legal?");
32914
32915 SDValue Lo = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, LoVT, In, DAG);
32916
32917 // We need to shift the input over by half the number of elements.
32918 unsigned NumElts = InVT.getVectorNumElements();
32919 unsigned HalfNumElts = NumElts / 2;
32920 SmallVector<int, 16> ShufMask(NumElts, SM_SentinelUndef);
32921 for (unsigned i = 0; i != HalfNumElts; ++i)
32922 ShufMask[i] = i + HalfNumElts;
32923
32924 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
32925 Hi = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, HiVT, Hi, DAG);
32926
32927 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
32928 Results.push_back(Res);
32929 }
32930 return;
32931 }
32932 case ISD::FP_TO_SINT:
32933 case ISD::STRICT_FP_TO_SINT:
32934 case ISD::FP_TO_UINT:
32935 case ISD::STRICT_FP_TO_UINT: {
32936 bool IsStrict = N->isStrictFPOpcode();
32937 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
32938 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
32939 EVT VT = N->getValueType(0);
32940 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
32941 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
32942 EVT SrcVT = Src.getValueType();
32943
32944 SDValue Res;
32945 if (isSoftF16(SrcVT, Subtarget)) {
32946 EVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;
32947 if (IsStrict) {
32948 Res =
32949 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},
32950 {Chain, DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
32951 {NVT, MVT::Other}, {Chain, Src})});
32952 Chain = Res.getValue(1);
32953 } else {
32954 Res = DAG.getNode(N->getOpcode(), dl, VT,
32955 DAG.getNode(ISD::FP_EXTEND, dl, NVT, Src));
32956 }
32957 Results.push_back(Res);
32958 if (IsStrict)
32959 Results.push_back(Chain);
32960
32961 return;
32962 }
32963
32964 if (VT.isVector() && Subtarget.hasFP16() &&
32965 SrcVT.getVectorElementType() == MVT::f16) {
32966 EVT EleVT = VT.getVectorElementType();
32967 EVT ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;
32968
32969 if (SrcVT != MVT::v8f16) {
32970 SDValue Tmp =
32971 IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT);
32972 SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);
32973 Ops[0] = Src;
32974 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);
32975 }
32976
32977 if (IsStrict) {
32978 unsigned Opc =
32979 IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
32980 Res =
32981 DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src});
32982 Chain = Res.getValue(1);
32983 } else {
32984 unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
32985 Res = DAG.getNode(Opc, dl, ResVT, Src);
32986 }
32987
32988 // TODO: Need to add exception check code for strict FP.
32989 if (EleVT.getSizeInBits() < 16) {
32990 MVT TmpVT = MVT::getVectorVT(EleVT.getSimpleVT(), 8);
32991 Res = DAG.getNode(ISD::TRUNCATE, dl, TmpVT, Res);
32992
32993 // Now widen to 128 bits.
32994 unsigned NumConcats = 128 / TmpVT.getSizeInBits();
32995 MVT ConcatVT = MVT::getVectorVT(EleVT.getSimpleVT(), 8 * NumConcats);
32996 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(TmpVT));
32997 ConcatOps[0] = Res;
32998 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps);
32999 }
33000
33001 Results.push_back(Res);
33002 if (IsStrict)
33003 Results.push_back(Chain);
33004
33005 return;
33006 }
33007
33008 if (VT.isVector() && VT.getScalarSizeInBits() < 32) {
33009 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33010 "Unexpected type action!");
33011
33012 // Try to create a 128 bit vector, but don't exceed a 32 bit element.
33013 unsigned NewEltWidth = std::min(128 / VT.getVectorNumElements(), 32U);
33014 MVT PromoteVT = MVT::getVectorVT(MVT::getIntegerVT(NewEltWidth),
33015 VT.getVectorNumElements());
33016 SDValue Res;
33017 SDValue Chain;
33018 if (IsStrict) {
33019 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {PromoteVT, MVT::Other},
33020 {N->getOperand(0), Src});
33021 Chain = Res.getValue(1);
33022 } else
33023 Res = DAG.getNode(ISD::FP_TO_SINT, dl, PromoteVT, Src);
33024
33025 // Preserve what we know about the size of the original result. If the
33026 // result is v2i32, we have to manually widen the assert.
33027 if (PromoteVT == MVT::v2i32)
33028 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,
33029 DAG.getUNDEF(MVT::v2i32));
33030
33031 Res = DAG.getNode(!IsSigned ? ISD::AssertZext : ISD::AssertSext, dl,
33032 Res.getValueType(), Res,
33033 DAG.getValueType(VT.getVectorElementType()));
33034
33035 if (PromoteVT == MVT::v2i32)
33036 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res,
33037 DAG.getIntPtrConstant(0, dl));
33038
33039 // Truncate back to the original width.
33040 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
33041
33042 // Now widen to 128 bits.
33043 unsigned NumConcats = 128 / VT.getSizeInBits();
33044 MVT ConcatVT = MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(),
33045 VT.getVectorNumElements() * NumConcats);
33046 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
33047 ConcatOps[0] = Res;
33048 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps);
33049 Results.push_back(Res);
33050 if (IsStrict)
33051 Results.push_back(Chain);
33052 return;
33053 }
33054
33055
33056 if (VT == MVT::v2i32) {
33057 assert((!IsStrict || IsSigned || Subtarget.hasAVX512()) &&
33058 "Strict unsigned conversion requires AVX512");
33059 assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
33060 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33061 "Unexpected type action!");
33062 if (Src.getValueType() == MVT::v2f64) {
33063 if (!IsSigned && !Subtarget.hasAVX512()) {
33064 SDValue Res =
33065 expandFP_TO_UINT_SSE(MVT::v4i32, Src, dl, DAG, Subtarget);
33066 Results.push_back(Res);
33067 return;
33068 }
33069
33070 unsigned Opc;
33071 if (IsStrict)
33072 Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
33073 else
33074 Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
33075
33076 // If we have VLX we can emit a target specific FP_TO_UINT node,.
33077 if (!IsSigned && !Subtarget.hasVLX()) {
33078 // Otherwise we can defer to the generic legalizer which will widen
33079 // the input as well. This will be further widened during op
33080 // legalization to v8i32<-v8f64.
33081 // For strict nodes we'll need to widen ourselves.
33082 // FIXME: Fix the type legalizer to safely widen strict nodes?
33083 if (!IsStrict)
33084 return;
33085 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64, Src,
33086 DAG.getConstantFP(0.0, dl, MVT::v2f64));
33087 Opc = N->getOpcode();
33088 }
33089 SDValue Res;
33090 SDValue Chain;
33091 if (IsStrict) {
33092 Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other},
33093 {N->getOperand(0), Src});
33094 Chain = Res.getValue(1);
33095 } else {
33096 Res = DAG.getNode(Opc, dl, MVT::v4i32, Src);
33097 }
33098 Results.push_back(Res);
33099 if (IsStrict)
33100 Results.push_back(Chain);
33101 return;
33102 }
33103
33104 // Custom widen strict v2f32->v2i32 by padding with zeros.
33105 // FIXME: Should generic type legalizer do this?
33106 if (Src.getValueType() == MVT::v2f32 && IsStrict) {
33107 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
33108 DAG.getConstantFP(0.0, dl, MVT::v2f32));
33109 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other},
33110 {N->getOperand(0), Src});
33111 Results.push_back(Res);
33112 Results.push_back(Res.getValue(1));
33113 return;
33114 }
33115
33116 // The FP_TO_INTHelper below only handles f32/f64/f80 scalar inputs,
33117 // so early out here.
33118 return;
33119 }
33120
33121 assert(!VT.isVector() && "Vectors should have been handled above!");
33122
33123 if ((Subtarget.hasDQI() && VT == MVT::i64 &&
33124 (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
33125 (Subtarget.hasFP16() && SrcVT == MVT::f16)) {
33126 assert(!Subtarget.is64Bit() && "i64 should be legal");
33127 unsigned NumElts = Subtarget.hasVLX() ? 2 : 8;
33128 // If we use a 128-bit result we might need to use a target specific node.
33129 unsigned SrcElts =
33130 std::max(NumElts, 128U / (unsigned)SrcVT.getSizeInBits());
33131 MVT VecVT = MVT::getVectorVT(MVT::i64, NumElts);
33132 MVT VecInVT = MVT::getVectorVT(SrcVT.getSimpleVT(), SrcElts);
33133 unsigned Opc = N->getOpcode();
33134 if (NumElts != SrcElts) {
33135 if (IsStrict)
33136 Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
33137 else
33138 Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
33139 }
33140
33141 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
33142 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
33143 DAG.getConstantFP(0.0, dl, VecInVT), Src,
33144 ZeroIdx);
33145 SDValue Chain;
33146 if (IsStrict) {
33147 SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
33148 Res = DAG.getNode(Opc, SDLoc(N), Tys, N->getOperand(0), Res);
33149 Chain = Res.getValue(1);
33150 } else
33151 Res = DAG.getNode(Opc, SDLoc(N), VecVT, Res);
33152 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx);
33153 Results.push_back(Res);
33154 if (IsStrict)
33155 Results.push_back(Chain);
33156 return;
33157 }
33158
33159 if (VT == MVT::i128 && Subtarget.isTargetWin64()) {
33160 SDValue Chain;
33161 SDValue V = LowerWin64_FP_TO_INT128(SDValue(N, 0), DAG, Chain);
33162 Results.push_back(V);
33163 if (IsStrict)
33164 Results.push_back(Chain);
33165 return;
33166 }
33167
33168 if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, Chain)) {
33169 Results.push_back(V);
33170 if (IsStrict)
33171 Results.push_back(Chain);
33172 }
33173 return;
33174 }
33175 case ISD::LRINT:
33176 case ISD::LLRINT: {
33177 if (SDValue V = LRINT_LLRINTHelper(N, DAG))
33178 Results.push_back(V);
33179 return;
33180 }
33181
33182 case ISD::SINT_TO_FP:
33183 case ISD::STRICT_SINT_TO_FP:
33184 case ISD::UINT_TO_FP:
33185 case ISD::STRICT_UINT_TO_FP: {
33186 bool IsStrict = N->isStrictFPOpcode();
33187 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
33188 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
33189 EVT VT = N->getValueType(0);
33190 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
33191 if (VT.getVectorElementType() == MVT::f16 && Subtarget.hasFP16() &&
33192 Subtarget.hasVLX()) {
33193 if (Src.getValueType().getVectorElementType() == MVT::i16)
33194 return;
33195
33196 if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2i32)
33197 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
33198 IsStrict ? DAG.getConstant(0, dl, MVT::v2i32)
33199 : DAG.getUNDEF(MVT::v2i32));
33200 if (IsStrict) {
33201 unsigned Opc =
33202 IsSigned ? X86ISD::STRICT_CVTSI2P : X86ISD::STRICT_CVTUI2P;
33203 SDValue Res = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
33204 {N->getOperand(0), Src});
33205 Results.push_back(Res);
33206 Results.push_back(Res.getValue(1));
33207 } else {
33208 unsigned Opc = IsSigned ? X86ISD::CVTSI2P : X86ISD::CVTUI2P;
33209 Results.push_back(DAG.getNode(Opc, dl, MVT::v8f16, Src));
33210 }
33211 return;
33212 }
33213 if (VT != MVT::v2f32)
33214 return;
33215 EVT SrcVT = Src.getValueType();
33216 if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) {
33217 if (IsStrict) {
33218 unsigned Opc = IsSigned ? X86ISD::STRICT_CVTSI2P
33219 : X86ISD::STRICT_CVTUI2P;
33220 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
33221 {N->getOperand(0), Src});
33222 Results.push_back(Res);
33223 Results.push_back(Res.getValue(1));
33224 } else {
33225 unsigned Opc = IsSigned ? X86ISD::CVTSI2P : X86ISD::CVTUI2P;
33226 Results.push_back(DAG.getNode(Opc, dl, MVT::v4f32, Src));
33227 }
33228 return;
33229 }
33230 if (SrcVT == MVT::v2i64 && !IsSigned && Subtarget.is64Bit() &&
33231 Subtarget.hasSSE41() && !Subtarget.hasAVX512()) {
33232 SDValue Zero = DAG.getConstant(0, dl, SrcVT);
33233 SDValue One = DAG.getConstant(1, dl, SrcVT);
33234 SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT,
33235 DAG.getNode(ISD::SRL, dl, SrcVT, Src, One),
33236 DAG.getNode(ISD::AND, dl, SrcVT, Src, One));
33237 SDValue IsNeg = DAG.getSetCC(dl, MVT::v2i64, Src, Zero, ISD::SETLT);
33238 SDValue SignSrc = DAG.getSelect(dl, SrcVT, IsNeg, Sign, Src);
33239 SmallVector<SDValue, 4> SignCvts(4, DAG.getConstantFP(0.0, dl, MVT::f32));
33240 for (int i = 0; i != 2; ++i) {
33241 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
33242 SignSrc, DAG.getIntPtrConstant(i, dl));
33243 if (IsStrict)
33244 SignCvts[i] =
33245 DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {MVT::f32, MVT::Other},
33246 {N->getOperand(0), Elt});
33247 else
33248 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Elt);
33249 };
33250 SDValue SignCvt = DAG.getBuildVector(MVT::v4f32, dl, SignCvts);
33251 SDValue Slow, Chain;
33252 if (IsStrict) {
33253 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
33254 SignCvts[0].getValue(1), SignCvts[1].getValue(1));
33255 Slow = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v4f32, MVT::Other},
33256 {Chain, SignCvt, SignCvt});
33257 Chain = Slow.getValue(1);
33258 } else {
33259 Slow = DAG.getNode(ISD::FADD, dl, MVT::v4f32, SignCvt, SignCvt);
33260 }
33261 IsNeg = DAG.getBitcast(MVT::v4i32, IsNeg);
33262 IsNeg =
33263 DAG.getVectorShuffle(MVT::v4i32, dl, IsNeg, IsNeg, {1, 3, -1, -1});
33264 SDValue Cvt = DAG.getSelect(dl, MVT::v4f32, IsNeg, Slow, SignCvt);
33265 Results.push_back(Cvt);
33266 if (IsStrict)
33267 Results.push_back(Chain);
33268 return;
33269 }
33270
33271 if (SrcVT != MVT::v2i32)
33272 return;
33273
33274 if (IsSigned || Subtarget.hasAVX512()) {
33275 if (!IsStrict)
33276 return;
33277
33278 // Custom widen strict v2i32->v2f32 to avoid scalarization.
33279 // FIXME: Should generic type legalizer do this?
33280 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
33281 DAG.getConstant(0, dl, MVT::v2i32));
33282 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other},
33283 {N->getOperand(0), Src});
33284 Results.push_back(Res);
33285 Results.push_back(Res.getValue(1));
33286 return;
33287 }
33288
33289 assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
33290 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src);
33291 SDValue VBias = DAG.getConstantFP(
33292 llvm::bit_cast<double>(0x4330000000000000ULL), dl, MVT::v2f64);
33293 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
33294 DAG.getBitcast(MVT::v2i64, VBias));
33295 Or = DAG.getBitcast(MVT::v2f64, Or);
33296 if (IsStrict) {
33297 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
33298 {N->getOperand(0), Or, VBias});
33299 SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl,
33300 {MVT::v4f32, MVT::Other},
33301 {Sub.getValue(1), Sub});
33302 Results.push_back(Res);
33303 Results.push_back(Res.getValue(1));
33304 } else {
33305 // TODO: Are there any fast-math-flags to propagate here?
33306 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
33307 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
33308 }
33309 return;
33310 }
33311 case ISD::STRICT_FP_ROUND:
33312 case ISD::FP_ROUND: {
33313 bool IsStrict = N->isStrictFPOpcode();
33314 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
33315 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
33316 SDValue Rnd = N->getOperand(IsStrict ? 2 : 1);
33317 EVT SrcVT = Src.getValueType();
33318 EVT VT = N->getValueType(0);
33319 SDValue V;
33320 if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2f32) {
33321 SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f32)
33322 : DAG.getUNDEF(MVT::v2f32);
33323 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, Ext);
33324 }
33325 if (!Subtarget.hasFP16() && VT.getVectorElementType() == MVT::f16) {
33326 assert(Subtarget.hasF16C() && "Cannot widen f16 without F16C");
33327 if (SrcVT.getVectorElementType() != MVT::f32)
33328 return;
33329
33330 if (IsStrict)
33331 V = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},
33332 {Chain, Src, Rnd});
33333 else
33334 V = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Src, Rnd);
33335
33336 Results.push_back(DAG.getBitcast(MVT::v8f16, V));
33337 if (IsStrict)
33338 Results.push_back(V.getValue(1));
33339 return;
33340 }
33341 if (!isTypeLegal(Src.getValueType()))
33342 return;
33343 EVT NewVT = VT.getVectorElementType() == MVT::f16 ? MVT::v8f16 : MVT::v4f32;
33344 if (IsStrict)
33345 V = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {NewVT, MVT::Other},
33346 {Chain, Src});
33347 else
33348 V = DAG.getNode(X86ISD::VFPROUND, dl, NewVT, Src);
33349 Results.push_back(V);
33350 if (IsStrict)
33351 Results.push_back(V.getValue(1));
33352 return;
33353 }
33354 case ISD::FP_EXTEND:
33355 case ISD::STRICT_FP_EXTEND: {
33356 // Right now, only MVT::v2f32 has OperationAction for FP_EXTEND.
33357 // No other ValueType for FP_EXTEND should reach this point.
33358 assert(N->getValueType(0) == MVT::v2f32 &&
33359 "Do not know how to legalize this Node");
33360 if (!Subtarget.hasFP16() || !Subtarget.hasVLX())
33361 return;
33362 bool IsStrict = N->isStrictFPOpcode();
33363 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
33364 if (Src.getValueType().getVectorElementType() != MVT::f16)
33365 return;
33366 SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f16)
33367 : DAG.getUNDEF(MVT::v2f16);
33368 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f16, Src, Ext);
33369 if (IsStrict)
33370 V = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::v4f32, MVT::Other},
33371 {N->getOperand(0), V});
33372 else
33373 V = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, V);
33374 Results.push_back(V);
33375 if (IsStrict)
33376 Results.push_back(V.getValue(1));
33377 return;
33378 }
33379 case ISD::INTRINSIC_W_CHAIN: {
33380 unsigned IntNo = N->getConstantOperandVal(1);
33381 switch (IntNo) {
33382 default : llvm_unreachable("Do not know how to custom type "
33383 "legalize this intrinsic operation!");
33384 case Intrinsic::x86_rdtsc:
33385 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget,
33386 Results);
33387 case Intrinsic::x86_rdtscp:
33388 return getReadTimeStampCounter(N, dl, X86::RDTSCP, DAG, Subtarget,
33389 Results);
33390 case Intrinsic::x86_rdpmc:
33391 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPMC, X86::ECX, Subtarget,
33392 Results);
33393 return;
33394 case Intrinsic::x86_rdpru:
33395 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPRU, X86::ECX, Subtarget,
33396 Results);
33397 return;
33398 case Intrinsic::x86_xgetbv:
33399 expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget,
33400 Results);
33401 return;
33402 }
33403 }
33404 case ISD::READCYCLECOUNTER: {
33405 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, Results);
33406 }
33407 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
33408 EVT T = N->getValueType(0);
33409 assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair");
33410 bool Regs64bit = T == MVT::i128;
33411 assert((!Regs64bit || Subtarget.canUseCMPXCHG16B()) &&
33412 "64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS requires CMPXCHG16B");
33413 MVT HalfT = Regs64bit ? MVT::i64 : MVT::i32;
33414 SDValue cpInL, cpInH;
33415 std::tie(cpInL, cpInH) =
33416 DAG.SplitScalar(N->getOperand(2), dl, HalfT, HalfT);
33417 cpInL = DAG.getCopyToReg(N->getOperand(0), dl,
33418 Regs64bit ? X86::RAX : X86::EAX, cpInL, SDValue());
33419 cpInH =
33420 DAG.getCopyToReg(cpInL.getValue(0), dl, Regs64bit ? X86::RDX : X86::EDX,
33421 cpInH, cpInL.getValue(1));
33422 SDValue swapInL, swapInH;
33423 std::tie(swapInL, swapInH) =
33424 DAG.SplitScalar(N->getOperand(3), dl, HalfT, HalfT);
33425 swapInH =
33426 DAG.getCopyToReg(cpInH.getValue(0), dl, Regs64bit ? X86::RCX : X86::ECX,
33427 swapInH, cpInH.getValue(1));
33428
33429 // In 64-bit mode we might need the base pointer in RBX, but we can't know
33430 // until later. So we keep the RBX input in a vreg and use a custom
33431 // inserter.
33432 // Since RBX will be a reserved register the register allocator will not
33433 // make sure its value will be properly saved and restored around this
33434 // live-range.
33435 SDValue Result;
33436 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
33437 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
33438 if (Regs64bit) {
33439 SDValue Ops[] = {swapInH.getValue(0), N->getOperand(1), swapInL,
33440 swapInH.getValue(1)};
33441 Result =
33442 DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG16_DAG, dl, Tys, Ops, T, MMO);
33443 } else {
33444 swapInL = DAG.getCopyToReg(swapInH.getValue(0), dl, X86::EBX, swapInL,
33445 swapInH.getValue(1));
33446 SDValue Ops[] = {swapInL.getValue(0), N->getOperand(1),
33447 swapInL.getValue(1)};
33448 Result =
33449 DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, Ops, T, MMO);
33450 }
33451
33452 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
33453 Regs64bit ? X86::RAX : X86::EAX,
33454 HalfT, Result.getValue(1));
33455 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
33456 Regs64bit ? X86::RDX : X86::EDX,
33457 HalfT, cpOutL.getValue(2));
33458 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
33459
33460 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
33461 MVT::i32, cpOutH.getValue(2));
33462 SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG);
33463 Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1));
33464
33465 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF));
33466 Results.push_back(Success);
33467 Results.push_back(EFLAGS.getValue(1));
33468 return;
33469 }
33470 case ISD::ATOMIC_LOAD: {
33471 assert(
33472 (N->getValueType(0) == MVT::i64 || N->getValueType(0) == MVT::i128) &&
33473 "Unexpected VT!");
33474 bool NoImplicitFloatOps =
33475 DAG.getMachineFunction().getFunction().hasFnAttribute(
33476 Attribute::NoImplicitFloat);
33477 if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) {
33478 auto *Node = cast<AtomicSDNode>(N);
33479
33480 if (N->getValueType(0) == MVT::i128) {
33481 if (Subtarget.is64Bit() && Subtarget.hasAVX()) {
33482 SDValue Ld = DAG.getLoad(MVT::v2i64, dl, Node->getChain(),
33483 Node->getBasePtr(), Node->getMemOperand());
33484 SDValue ResL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
33485 DAG.getIntPtrConstant(0, dl));
33486 SDValue ResH = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
33487 DAG.getIntPtrConstant(1, dl));
33488 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, N->getValueType(0),
33489 {ResL, ResH}));
33490 Results.push_back(Ld.getValue(1));
33491 return;
33492 }
33493 break;
33494 }
33495 if (Subtarget.hasSSE1()) {
33496 // Use a VZEXT_LOAD which will be selected as MOVQ or XORPS+MOVLPS.
33497 // Then extract the lower 64-bits.
33498 MVT LdVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;
33499 SDVTList Tys = DAG.getVTList(LdVT, MVT::Other);
33500 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
33501 SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
33502 MVT::i64, Node->getMemOperand());
33503 if (Subtarget.hasSSE2()) {
33504 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
33505 DAG.getIntPtrConstant(0, dl));
33506 Results.push_back(Res);
33507 Results.push_back(Ld.getValue(1));
33508 return;
33509 }
33510 // We use an alternative sequence for SSE1 that extracts as v2f32 and
33511 // then casts to i64. This avoids a 128-bit stack temporary being
33512 // created by type legalization if we were to cast v4f32->v2i64.
33513 SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld,
33514 DAG.getIntPtrConstant(0, dl));
33515 Res = DAG.getBitcast(MVT::i64, Res);
33516 Results.push_back(Res);
33517 Results.push_back(Ld.getValue(1));
33518 return;
33519 }
33520 if (Subtarget.hasX87()) {
33521 // First load this into an 80-bit X87 register. This will put the whole
33522 // integer into the significand.
33523 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
33524 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
33525 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD,
33526 dl, Tys, Ops, MVT::i64,
33527 Node->getMemOperand());
33528 SDValue Chain = Result.getValue(1);
33529
33530 // Now store the X87 register to a stack temporary and convert to i64.
33531 // This store is not atomic and doesn't need to be.
33532 // FIXME: We don't need a stack temporary if the result of the load
33533 // is already being stored. We could just directly store there.
33534 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
33535 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
33536 MachinePointerInfo MPI =
33537 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
33538 SDValue StoreOps[] = { Chain, Result, StackPtr };
33539 Chain = DAG.getMemIntrinsicNode(
33540 X86ISD::FIST, dl, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,
33541 MPI, std::nullopt /*Align*/, MachineMemOperand::MOStore);
33542
33543 // Finally load the value back from the stack temporary and return it.
33544 // This load is not atomic and doesn't need to be.
33545 // This load will be further type legalized.
33546 Result = DAG.getLoad(MVT::i64, dl, Chain, StackPtr, MPI);
33547 Results.push_back(Result);
33548 Results.push_back(Result.getValue(1));
33549 return;
33550 }
33551 }
33552 // TODO: Use MOVLPS when SSE1 is available?
33553 // Delegate to generic TypeLegalization. Situations we can really handle
33554 // should have already been dealt with by AtomicExpandPass.cpp.
33555 break;
33556 }
33557 case ISD::ATOMIC_SWAP:
33558 case ISD::ATOMIC_LOAD_ADD:
33559 case ISD::ATOMIC_LOAD_SUB:
33560 case ISD::ATOMIC_LOAD_AND:
33561 case ISD::ATOMIC_LOAD_OR:
33562 case ISD::ATOMIC_LOAD_XOR:
33563 case ISD::ATOMIC_LOAD_NAND:
33564 case ISD::ATOMIC_LOAD_MIN:
33565 case ISD::ATOMIC_LOAD_MAX:
33566 case ISD::ATOMIC_LOAD_UMIN:
33567 case ISD::ATOMIC_LOAD_UMAX:
33568 // Delegate to generic TypeLegalization. Situations we can really handle
33569 // should have already been dealt with by AtomicExpandPass.cpp.
33570 break;
33571
33572 case ISD::BITCAST: {
33573 assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
33574 EVT DstVT = N->getValueType(0);
33575 EVT SrcVT = N->getOperand(0).getValueType();
33576
33577 // If this is a bitcast from a v64i1 k-register to a i64 on a 32-bit target
33578 // we can split using the k-register rather than memory.
33579 if (SrcVT == MVT::v64i1 && DstVT == MVT::i64 && Subtarget.hasBWI()) {
33580 assert(!Subtarget.is64Bit() && "Expected 32-bit mode");
33581 SDValue Lo, Hi;
33582 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
33583 Lo = DAG.getBitcast(MVT::i32, Lo);
33584 Hi = DAG.getBitcast(MVT::i32, Hi);
33585 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
33586 Results.push_back(Res);
33587 return;
33588 }
33589
33590 if (DstVT.isVector() && SrcVT == MVT::x86mmx) {
33591 // FIXME: Use v4f32 for SSE1?
33592 assert(Subtarget.hasSSE2() && "Requires SSE2");
33593 assert(getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&
33594 "Unexpected type action!");
33595 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), DstVT);
33596 SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64,
33597 N->getOperand(0));
33598 Res = DAG.getBitcast(WideVT, Res);
33599 Results.push_back(Res);
33600 return;
33601 }
33602
33603 return;
33604 }
33605 case ISD::MGATHER: {
33606 EVT VT = N->getValueType(0);
33607 if ((VT == MVT::v2f32 || VT == MVT::v2i32) &&
33608 (Subtarget.hasVLX() || !Subtarget.hasAVX512())) {
33609 auto *Gather = cast<MaskedGatherSDNode>(N);
33610 SDValue Index = Gather->getIndex();
33611 if (Index.getValueType() != MVT::v2i64)
33612 return;
33613 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33614 "Unexpected type action!");
33615 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
33616 SDValue Mask = Gather->getMask();
33617 assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
33618 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
33619 Gather->getPassThru(),
33620 DAG.getUNDEF(VT));
33621 if (!Subtarget.hasVLX()) {
33622 // We need to widen the mask, but the instruction will only use 2
33623 // of its elements. So we can use undef.
33624 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask,
33625 DAG.getUNDEF(MVT::v2i1));
33626 Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask);
33627 }
33628 SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
33629 Gather->getBasePtr(), Index, Gather->getScale() };
33630 SDValue Res = DAG.getMemIntrinsicNode(
33631 X86ISD::MGATHER, dl, DAG.getVTList(WideVT, MVT::Other), Ops,
33632 Gather->getMemoryVT(), Gather->getMemOperand());
33633 Results.push_back(Res);
33634 Results.push_back(Res.getValue(1));
33635 return;
33636 }
33637 return;
33638 }
33639 case ISD::LOAD: {
33640 // Use an f64/i64 load and a scalar_to_vector for v2f32/v2i32 loads. This
33641 // avoids scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp
33642 // cast since type legalization will try to use an i64 load.
33643 MVT VT = N->getSimpleValueType(0);
33644 assert(VT.isVector() && VT.getSizeInBits() == 64 && "Unexpected VT");
33645 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
33646 "Unexpected type action!");
33647 if (!ISD::isNON_EXTLoad(N))
33648 return;
33649 auto *Ld = cast<LoadSDNode>(N);
33650 if (Subtarget.hasSSE2()) {
33651 MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64;
33652 SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(),
33653 Ld->getPointerInfo(), Ld->getOriginalAlign(),
33654 Ld->getMemOperand()->getFlags());
33655 SDValue Chain = Res.getValue(1);
33656 MVT VecVT = MVT::getVectorVT(LdVT, 2);
33657 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Res);
33658 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
33659 Res = DAG.getBitcast(WideVT, Res);
33660 Results.push_back(Res);
33661 Results.push_back(Chain);
33662 return;
33663 }
33664 assert(Subtarget.hasSSE1() && "Expected SSE");
33665 SDVTList Tys = DAG.getVTList(MVT::v4f32, MVT::Other);
33666 SDValue Ops[] = {Ld->getChain(), Ld->getBasePtr()};
33667 SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
33668 MVT::i64, Ld->getMemOperand());
33669 Results.push_back(Res);
33670 Results.push_back(Res.getValue(1));
33671 return;
33672 }
33673 case ISD::ADDRSPACECAST: {
33674 SDValue V = LowerADDRSPACECAST(SDValue(N,0), DAG);
33675 Results.push_back(V);
33676 return;
33677 }
33678 case ISD::BITREVERSE: {
33679 assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");
33680 assert(Subtarget.hasXOP() && "Expected XOP");
33681 // We can use VPPERM by copying to a vector register and back. We'll need
33682 // to move the scalar in two i32 pieces.
33683 Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG));
33684 return;
33685 }
33686 case ISD::EXTRACT_VECTOR_ELT: {
33687 // f16 = extract vXf16 %vec, i64 %idx
33688 assert(N->getSimpleValueType(0) == MVT::f16 &&
33689 "Unexpected Value type of EXTRACT_VECTOR_ELT!");
33690 assert(Subtarget.hasFP16() && "Expected FP16");
33691 SDValue VecOp = N->getOperand(0);
33692 EVT ExtVT = VecOp.getValueType().changeVectorElementTypeToInteger();
33693 SDValue Split = DAG.getBitcast(ExtVT, N->getOperand(0));
33694 Split = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Split,
33695 N->getOperand(1));
33696 Split = DAG.getBitcast(MVT::f16, Split);
33697 Results.push_back(Split);
33698 return;
33699 }
33700 }
33701}
33702
33703const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
33704 switch ((X86ISD::NodeType)Opcode) {
33705 case X86ISD::FIRST_NUMBER: break;
33706#define NODE_NAME_CASE(NODE) case X86ISD::NODE: return "X86ISD::" #NODE;
33707 NODE_NAME_CASE(BSF)
33708 NODE_NAME_CASE(BSR)
33709 NODE_NAME_CASE(FSHL)
33710 NODE_NAME_CASE(FSHR)
33711 NODE_NAME_CASE(FAND)
33712 NODE_NAME_CASE(FANDN)
33713 NODE_NAME_CASE(FOR)
33714 NODE_NAME_CASE(FXOR)
33715 NODE_NAME_CASE(FILD)
33716 NODE_NAME_CASE(FIST)
33717 NODE_NAME_CASE(FP_TO_INT_IN_MEM)
33718 NODE_NAME_CASE(FLD)
33719 NODE_NAME_CASE(FST)
33720 NODE_NAME_CASE(CALL)
33721 NODE_NAME_CASE(CALL_RVMARKER)
33722 NODE_NAME_CASE(BT)
33723 NODE_NAME_CASE(CMP)
33724 NODE_NAME_CASE(FCMP)
33725 NODE_NAME_CASE(STRICT_FCMP)
33726 NODE_NAME_CASE(STRICT_FCMPS)
33727 NODE_NAME_CASE(COMI)
33728 NODE_NAME_CASE(UCOMI)
33729 NODE_NAME_CASE(CMPM)
33730 NODE_NAME_CASE(CMPMM)
33731 NODE_NAME_CASE(STRICT_CMPM)
33732 NODE_NAME_CASE(CMPMM_SAE)
33733 NODE_NAME_CASE(SETCC)
33734 NODE_NAME_CASE(SETCC_CARRY)
33735 NODE_NAME_CASE(FSETCC)
33736 NODE_NAME_CASE(FSETCCM)
33737 NODE_NAME_CASE(FSETCCM_SAE)
33738 NODE_NAME_CASE(CMOV)
33739 NODE_NAME_CASE(BRCOND)
33740 NODE_NAME_CASE(RET_GLUE)
33741 NODE_NAME_CASE(IRET)
33742 NODE_NAME_CASE(REP_STOS)
33743 NODE_NAME_CASE(REP_MOVS)
33744 NODE_NAME_CASE(GlobalBaseReg)
33745 NODE_NAME_CASE(Wrapper)
33746 NODE_NAME_CASE(WrapperRIP)
33747 NODE_NAME_CASE(MOVQ2DQ)
33748 NODE_NAME_CASE(MOVDQ2Q)
33749 NODE_NAME_CASE(MMX_MOVD2W)
33750 NODE_NAME_CASE(MMX_MOVW2D)
33751 NODE_NAME_CASE(PEXTRB)
33752 NODE_NAME_CASE(PEXTRW)
33753 NODE_NAME_CASE(INSERTPS)
33754 NODE_NAME_CASE(PINSRB)
33755 NODE_NAME_CASE(PINSRW)
33756 NODE_NAME_CASE(PSHUFB)
33757 NODE_NAME_CASE(ANDNP)
33758 NODE_NAME_CASE(BLENDI)
33759 NODE_NAME_CASE(BLENDV)
33760 NODE_NAME_CASE(HADD)
33761 NODE_NAME_CASE(HSUB)
33762 NODE_NAME_CASE(FHADD)
33763 NODE_NAME_CASE(FHSUB)
33764 NODE_NAME_CASE(CONFLICT)
33765 NODE_NAME_CASE(FMAX)
33766 NODE_NAME_CASE(FMAXS)
33767 NODE_NAME_CASE(FMAX_SAE)
33768 NODE_NAME_CASE(FMAXS_SAE)
33769 NODE_NAME_CASE(FMIN)
33770 NODE_NAME_CASE(FMINS)
33771 NODE_NAME_CASE(FMIN_SAE)
33772 NODE_NAME_CASE(FMINS_SAE)
33773 NODE_NAME_CASE(FMAXC)
33774 NODE_NAME_CASE(FMINC)
33775 NODE_NAME_CASE(FRSQRT)
33776 NODE_NAME_CASE(FRCP)
33777 NODE_NAME_CASE(EXTRQI)
33778 NODE_NAME_CASE(INSERTQI)
33779 NODE_NAME_CASE(TLSADDR)
33780 NODE_NAME_CASE(TLSBASEADDR)
33781 NODE_NAME_CASE(TLSCALL)
33782 NODE_NAME_CASE(TLSDESC)
33783 NODE_NAME_CASE(EH_SJLJ_SETJMP)
33784 NODE_NAME_CASE(EH_SJLJ_LONGJMP)
33785 NODE_NAME_CASE(EH_SJLJ_SETUP_DISPATCH)
33786 NODE_NAME_CASE(EH_RETURN)
33787 NODE_NAME_CASE(TC_RETURN)
33788 NODE_NAME_CASE(FNSTCW16m)
33789 NODE_NAME_CASE(FLDCW16m)
33790 NODE_NAME_CASE(FNSTENVm)
33791 NODE_NAME_CASE(FLDENVm)
33792 NODE_NAME_CASE(LCMPXCHG_DAG)
33793 NODE_NAME_CASE(LCMPXCHG8_DAG)
33794 NODE_NAME_CASE(LCMPXCHG16_DAG)
33795 NODE_NAME_CASE(LCMPXCHG16_SAVE_RBX_DAG)
33796 NODE_NAME_CASE(LADD)
33797 NODE_NAME_CASE(LSUB)
33798 NODE_NAME_CASE(LOR)
33799 NODE_NAME_CASE(LXOR)
33800 NODE_NAME_CASE(LAND)
33801 NODE_NAME_CASE(LBTS)
33802 NODE_NAME_CASE(LBTC)
33803 NODE_NAME_CASE(LBTR)
33804 NODE_NAME_CASE(LBTS_RM)
33805 NODE_NAME_CASE(LBTC_RM)
33806 NODE_NAME_CASE(LBTR_RM)
33807 NODE_NAME_CASE(AADD)
33808 NODE_NAME_CASE(AOR)
33809 NODE_NAME_CASE(AXOR)
33810 NODE_NAME_CASE(AAND)
33811 NODE_NAME_CASE(VZEXT_MOVL)
33812 NODE_NAME_CASE(VZEXT_LOAD)
33813 NODE_NAME_CASE(VEXTRACT_STORE)
33814 NODE_NAME_CASE(VTRUNC)
33815 NODE_NAME_CASE(VTRUNCS)
33816 NODE_NAME_CASE(VTRUNCUS)
33817 NODE_NAME_CASE(VMTRUNC)
33818 NODE_NAME_CASE(VMTRUNCS)
33819 NODE_NAME_CASE(VMTRUNCUS)
33820 NODE_NAME_CASE(VTRUNCSTORES)
33821 NODE_NAME_CASE(VTRUNCSTOREUS)
33822 NODE_NAME_CASE(VMTRUNCSTORES)
33823 NODE_NAME_CASE(VMTRUNCSTOREUS)
33824 NODE_NAME_CASE(VFPEXT)
33825 NODE_NAME_CASE(STRICT_VFPEXT)
33826 NODE_NAME_CASE(VFPEXT_SAE)
33827 NODE_NAME_CASE(VFPEXTS)
33828 NODE_NAME_CASE(VFPEXTS_SAE)
33829 NODE_NAME_CASE(VFPROUND)
33830 NODE_NAME_CASE(VFPROUND2)
33831 NODE_NAME_CASE(VFPROUND2_RND)
33832 NODE_NAME_CASE(STRICT_VFPROUND)
33833 NODE_NAME_CASE(VMFPROUND)
33834 NODE_NAME_CASE(VFPROUND_RND)
33835 NODE_NAME_CASE(VFPROUNDS)
33836 NODE_NAME_CASE(VFPROUNDS_RND)
33837 NODE_NAME_CASE(VSHLDQ)
33838 NODE_NAME_CASE(VSRLDQ)
33839 NODE_NAME_CASE(VSHL)
33840 NODE_NAME_CASE(VSRL)
33841 NODE_NAME_CASE(VSRA)
33842 NODE_NAME_CASE(VSHLI)
33843 NODE_NAME_CASE(VSRLI)
33844 NODE_NAME_CASE(VSRAI)
33845 NODE_NAME_CASE(VSHLV)
33846 NODE_NAME_CASE(VSRLV)
33847 NODE_NAME_CASE(VSRAV)
33848 NODE_NAME_CASE(VROTLI)
33849 NODE_NAME_CASE(VROTRI)
33850 NODE_NAME_CASE(VPPERM)
33851 NODE_NAME_CASE(CMPP)
33852 NODE_NAME_CASE(STRICT_CMPP)
33853 NODE_NAME_CASE(PCMPEQ)
33854 NODE_NAME_CASE(PCMPGT)
33855 NODE_NAME_CASE(PHMINPOS)
33856 NODE_NAME_CASE(ADD)
33857 NODE_NAME_CASE(SUB)
33858 NODE_NAME_CASE(ADC)
33859 NODE_NAME_CASE(SBB)
33860 NODE_NAME_CASE(SMUL)
33861 NODE_NAME_CASE(UMUL)
33862 NODE_NAME_CASE(OR)
33863 NODE_NAME_CASE(XOR)
33864 NODE_NAME_CASE(AND)
33865 NODE_NAME_CASE(BEXTR)
33866 NODE_NAME_CASE(BEXTRI)
33867 NODE_NAME_CASE(BZHI)
33868 NODE_NAME_CASE(PDEP)
33869 NODE_NAME_CASE(PEXT)
33870 NODE_NAME_CASE(MUL_IMM)
33871 NODE_NAME_CASE(MOVMSK)
33872 NODE_NAME_CASE(PTEST)
33873 NODE_NAME_CASE(TESTP)
33874 NODE_NAME_CASE(KORTEST)
33875 NODE_NAME_CASE(KTEST)
33876 NODE_NAME_CASE(KADD)
33877 NODE_NAME_CASE(KSHIFTL)
33878 NODE_NAME_CASE(KSHIFTR)
33879 NODE_NAME_CASE(PACKSS)
33880 NODE_NAME_CASE(PACKUS)
33881 NODE_NAME_CASE(PALIGNR)
33882 NODE_NAME_CASE(VALIGN)
33883 NODE_NAME_CASE(VSHLD)
33884 NODE_NAME_CASE(VSHRD)
33885 NODE_NAME_CASE(VSHLDV)
33886 NODE_NAME_CASE(VSHRDV)
33887 NODE_NAME_CASE(PSHUFD)
33888 NODE_NAME_CASE(PSHUFHW)
33889 NODE_NAME_CASE(PSHUFLW)
33890 NODE_NAME_CASE(SHUFP)
33891 NODE_NAME_CASE(SHUF128)
33892 NODE_NAME_CASE(MOVLHPS)
33893 NODE_NAME_CASE(MOVHLPS)
33894 NODE_NAME_CASE(MOVDDUP)
33895 NODE_NAME_CASE(MOVSHDUP)
33896 NODE_NAME_CASE(MOVSLDUP)
33897 NODE_NAME_CASE(MOVSD)
33898 NODE_NAME_CASE(MOVSS)
33899 NODE_NAME_CASE(MOVSH)
33900 NODE_NAME_CASE(UNPCKL)
33901 NODE_NAME_CASE(UNPCKH)
33902 NODE_NAME_CASE(VBROADCAST)
33903 NODE_NAME_CASE(VBROADCAST_LOAD)
33904 NODE_NAME_CASE(VBROADCASTM)
33905 NODE_NAME_CASE(SUBV_BROADCAST_LOAD)
33906 NODE_NAME_CASE(VPERMILPV)
33907 NODE_NAME_CASE(VPERMILPI)
33908 NODE_NAME_CASE(VPERM2X128)
33909 NODE_NAME_CASE(VPERMV)
33910 NODE_NAME_CASE(VPERMV3)
33911 NODE_NAME_CASE(VPERMI)
33912 NODE_NAME_CASE(VPTERNLOG)
33913 NODE_NAME_CASE(VFIXUPIMM)
33914 NODE_NAME_CASE(VFIXUPIMM_SAE)
33915 NODE_NAME_CASE(VFIXUPIMMS)
33916 NODE_NAME_CASE(VFIXUPIMMS_SAE)
33917 NODE_NAME_CASE(VRANGE)
33918 NODE_NAME_CASE(VRANGE_SAE)
33919 NODE_NAME_CASE(VRANGES)
33920 NODE_NAME_CASE(VRANGES_SAE)
33921 NODE_NAME_CASE(PMULUDQ)
33922 NODE_NAME_CASE(PMULDQ)
33923 NODE_NAME_CASE(PSADBW)
33924 NODE_NAME_CASE(DBPSADBW)
33925 NODE_NAME_CASE(VASTART_SAVE_XMM_REGS)
33926 NODE_NAME_CASE(VAARG_64)
33927 NODE_NAME_CASE(VAARG_X32)
33928 NODE_NAME_CASE(DYN_ALLOCA)
33929 NODE_NAME_CASE(MFENCE)
33930 NODE_NAME_CASE(SEG_ALLOCA)
33931 NODE_NAME_CASE(PROBED_ALLOCA)
33932 NODE_NAME_CASE(RDRAND)
33933 NODE_NAME_CASE(RDSEED)
33934 NODE_NAME_CASE(RDPKRU)
33935 NODE_NAME_CASE(WRPKRU)
33936 NODE_NAME_CASE(VPMADDUBSW)
33937 NODE_NAME_CASE(VPMADDWD)
33938 NODE_NAME_CASE(VPSHA)
33939 NODE_NAME_CASE(VPSHL)
33940 NODE_NAME_CASE(VPCOM)
33941 NODE_NAME_CASE(VPCOMU)
33942 NODE_NAME_CASE(VPERMIL2)
33943 NODE_NAME_CASE(FMSUB)
33944 NODE_NAME_CASE(STRICT_FMSUB)
33945 NODE_NAME_CASE(FNMADD)
33946 NODE_NAME_CASE(STRICT_FNMADD)
33947 NODE_NAME_CASE(FNMSUB)
33948 NODE_NAME_CASE(STRICT_FNMSUB)
33949 NODE_NAME_CASE(FMADDSUB)
33950 NODE_NAME_CASE(FMSUBADD)
33951 NODE_NAME_CASE(FMADD_RND)
33952 NODE_NAME_CASE(FNMADD_RND)
33953 NODE_NAME_CASE(FMSUB_RND)
33954 NODE_NAME_CASE(FNMSUB_RND)
33955 NODE_NAME_CASE(FMADDSUB_RND)
33956 NODE_NAME_CASE(FMSUBADD_RND)
33957 NODE_NAME_CASE(VFMADDC)
33958 NODE_NAME_CASE(VFMADDC_RND)
33959 NODE_NAME_CASE(VFCMADDC)
33960 NODE_NAME_CASE(VFCMADDC_RND)
33961 NODE_NAME_CASE(VFMULC)
33962 NODE_NAME_CASE(VFMULC_RND)
33963 NODE_NAME_CASE(VFCMULC)
33964 NODE_NAME_CASE(VFCMULC_RND)
33965 NODE_NAME_CASE(VFMULCSH)
33966 NODE_NAME_CASE(VFMULCSH_RND)
33967 NODE_NAME_CASE(VFCMULCSH)
33968 NODE_NAME_CASE(VFCMULCSH_RND)
33969 NODE_NAME_CASE(VFMADDCSH)
33970 NODE_NAME_CASE(VFMADDCSH_RND)
33971 NODE_NAME_CASE(VFCMADDCSH)
33972 NODE_NAME_CASE(VFCMADDCSH_RND)
33973 NODE_NAME_CASE(VPMADD52H)
33974 NODE_NAME_CASE(VPMADD52L)
33975 NODE_NAME_CASE(VRNDSCALE)
33976 NODE_NAME_CASE(STRICT_VRNDSCALE)
33977 NODE_NAME_CASE(VRNDSCALE_SAE)
33978 NODE_NAME_CASE(VRNDSCALES)
33979 NODE_NAME_CASE(VRNDSCALES_SAE)
33980 NODE_NAME_CASE(VREDUCE)
33981 NODE_NAME_CASE(VREDUCE_SAE)
33982 NODE_NAME_CASE(VREDUCES)
33983 NODE_NAME_CASE(VREDUCES_SAE)
33984 NODE_NAME_CASE(VGETMANT)
33985 NODE_NAME_CASE(VGETMANT_SAE)
33986 NODE_NAME_CASE(VGETMANTS)
33987 NODE_NAME_CASE(VGETMANTS_SAE)
33988 NODE_NAME_CASE(PCMPESTR)
33989 NODE_NAME_CASE(PCMPISTR)
33990 NODE_NAME_CASE(XTEST)
33991 NODE_NAME_CASE(COMPRESS)
33992 NODE_NAME_CASE(EXPAND)
33993 NODE_NAME_CASE(SELECTS)
33994 NODE_NAME_CASE(ADDSUB)
33995 NODE_NAME_CASE(RCP14)
33996 NODE_NAME_CASE(RCP14S)
33997 NODE_NAME_CASE(RSQRT14)
33998 NODE_NAME_CASE(RSQRT14S)
33999 NODE_NAME_CASE(FADD_RND)
34000 NODE_NAME_CASE(FADDS)
34001 NODE_NAME_CASE(FADDS_RND)
34002 NODE_NAME_CASE(FSUB_RND)
34003 NODE_NAME_CASE(FSUBS)
34004 NODE_NAME_CASE(FSUBS_RND)
34005 NODE_NAME_CASE(FMUL_RND)
34006 NODE_NAME_CASE(FMULS)
34007 NODE_NAME_CASE(FMULS_RND)
34008 NODE_NAME_CASE(FDIV_RND)
34009 NODE_NAME_CASE(FDIVS)
34010 NODE_NAME_CASE(FDIVS_RND)
34011 NODE_NAME_CASE(FSQRT_RND)
34012 NODE_NAME_CASE(FSQRTS)
34013 NODE_NAME_CASE(FSQRTS_RND)
34014 NODE_NAME_CASE(FGETEXP)
34015 NODE_NAME_CASE(FGETEXP_SAE)
34016 NODE_NAME_CASE(FGETEXPS)
34017 NODE_NAME_CASE(FGETEXPS_SAE)
34018 NODE_NAME_CASE(SCALEF)
34019 NODE_NAME_CASE(SCALEF_RND)
34020 NODE_NAME_CASE(SCALEFS)
34021 NODE_NAME_CASE(SCALEFS_RND)
34022 NODE_NAME_CASE(MULHRS)
34023 NODE_NAME_CASE(SINT_TO_FP_RND)
34024 NODE_NAME_CASE(UINT_TO_FP_RND)
34025 NODE_NAME_CASE(CVTTP2SI)
34026 NODE_NAME_CASE(CVTTP2UI)
34027 NODE_NAME_CASE(STRICT_CVTTP2SI)
34028 NODE_NAME_CASE(STRICT_CVTTP2UI)
34029 NODE_NAME_CASE(MCVTTP2SI)
34030 NODE_NAME_CASE(MCVTTP2UI)
34031 NODE_NAME_CASE(CVTTP2SI_SAE)
34032 NODE_NAME_CASE(CVTTP2UI_SAE)
34033 NODE_NAME_CASE(CVTTS2SI)
34034 NODE_NAME_CASE(CVTTS2UI)
34035 NODE_NAME_CASE(CVTTS2SI_SAE)
34036 NODE_NAME_CASE(CVTTS2UI_SAE)
34037 NODE_NAME_CASE(CVTSI2P)
34038 NODE_NAME_CASE(CVTUI2P)
34039 NODE_NAME_CASE(STRICT_CVTSI2P)
34040 NODE_NAME_CASE(STRICT_CVTUI2P)
34041 NODE_NAME_CASE(MCVTSI2P)
34042 NODE_NAME_CASE(MCVTUI2P)
34043 NODE_NAME_CASE(VFPCLASS)
34044 NODE_NAME_CASE(VFPCLASSS)
34045 NODE_NAME_CASE(MULTISHIFT)
34046 NODE_NAME_CASE(SCALAR_SINT_TO_FP)
34047 NODE_NAME_CASE(SCALAR_SINT_TO_FP_RND)
34048 NODE_NAME_CASE(SCALAR_UINT_TO_FP)
34049 NODE_NAME_CASE(SCALAR_UINT_TO_FP_RND)
34050 NODE_NAME_CASE(CVTPS2PH)
34051 NODE_NAME_CASE(STRICT_CVTPS2PH)
34052 NODE_NAME_CASE(CVTPS2PH_SAE)
34053 NODE_NAME_CASE(MCVTPS2PH)
34054 NODE_NAME_CASE(MCVTPS2PH_SAE)
34055 NODE_NAME_CASE(CVTPH2PS)
34056 NODE_NAME_CASE(STRICT_CVTPH2PS)
34057 NODE_NAME_CASE(CVTPH2PS_SAE)
34058 NODE_NAME_CASE(CVTP2SI)
34059 NODE_NAME_CASE(CVTP2UI)
34060 NODE_NAME_CASE(MCVTP2SI)
34061 NODE_NAME_CASE(MCVTP2UI)
34062 NODE_NAME_CASE(CVTP2SI_RND)
34063 NODE_NAME_CASE(CVTP2UI_RND)
34064 NODE_NAME_CASE(CVTS2SI)
34065 NODE_NAME_CASE(CVTS2UI)
34066 NODE_NAME_CASE(CVTS2SI_RND)
34067 NODE_NAME_CASE(CVTS2UI_RND)
34068 NODE_NAME_CASE(CVTNEPS2BF16)
34069 NODE_NAME_CASE(MCVTNEPS2BF16)
34070 NODE_NAME_CASE(DPBF16PS)
34071 NODE_NAME_CASE(DPFP16PS)
34072 NODE_NAME_CASE(MPSADBW)
34073 NODE_NAME_CASE(LWPINS)
34074 NODE_NAME_CASE(MGATHER)
34075 NODE_NAME_CASE(MSCATTER)
34076 NODE_NAME_CASE(VPDPBUSD)
34077 NODE_NAME_CASE(VPDPBUSDS)
34078 NODE_NAME_CASE(VPDPWSSD)
34079 NODE_NAME_CASE(VPDPWSSDS)
34080 NODE_NAME_CASE(VPSHUFBITQMB)
34081 NODE_NAME_CASE(GF2P8MULB)
34082 NODE_NAME_CASE(GF2P8AFFINEQB)
34083 NODE_NAME_CASE(GF2P8AFFINEINVQB)
34084 NODE_NAME_CASE(NT_CALL)
34085 NODE_NAME_CASE(NT_BRIND)
34086 NODE_NAME_CASE(UMWAIT)
34087 NODE_NAME_CASE(TPAUSE)
34088 NODE_NAME_CASE(ENQCMD)
34089 NODE_NAME_CASE(ENQCMDS)
34090 NODE_NAME_CASE(VP2INTERSECT)
34091 NODE_NAME_CASE(VPDPBSUD)
34092 NODE_NAME_CASE(VPDPBSUDS)
34093 NODE_NAME_CASE(VPDPBUUD)
34094 NODE_NAME_CASE(VPDPBUUDS)
34095 NODE_NAME_CASE(VPDPBSSD)
34096 NODE_NAME_CASE(VPDPBSSDS)
34097 NODE_NAME_CASE(VPDPWSUD)
34098 NODE_NAME_CASE(VPDPWSUDS)
34099 NODE_NAME_CASE(VPDPWUSD)
34100 NODE_NAME_CASE(VPDPWUSDS)
34101 NODE_NAME_CASE(VPDPWUUD)
34102 NODE_NAME_CASE(VPDPWUUDS)
34103 NODE_NAME_CASE(VMINMAX)
34104 NODE_NAME_CASE(VMINMAX_SAE)
34105 NODE_NAME_CASE(VMINMAXS)
34106 NODE_NAME_CASE(VMINMAXS_SAE)
34107 NODE_NAME_CASE(CVTP2IBS)
34108 NODE_NAME_CASE(CVTP2IUBS)
34109 NODE_NAME_CASE(CVTP2IBS_RND)
34110 NODE_NAME_CASE(CVTP2IUBS_RND)
34111 NODE_NAME_CASE(CVTTP2IBS)
34112 NODE_NAME_CASE(CVTTP2IUBS)
34113 NODE_NAME_CASE(CVTTP2IBS_SAE)
34114 NODE_NAME_CASE(CVTTP2IUBS_SAE)
34115 NODE_NAME_CASE(VCVTNE2PH2BF8)
34116 NODE_NAME_CASE(VCVTNE2PH2BF8S)
34117 NODE_NAME_CASE(VCVTNE2PH2HF8)
34118 NODE_NAME_CASE(VCVTNE2PH2HF8S)
34119 NODE_NAME_CASE(VCVTBIASPH2BF8)
34120 NODE_NAME_CASE(VCVTBIASPH2BF8S)
34121 NODE_NAME_CASE(VCVTBIASPH2HF8)
34122 NODE_NAME_CASE(VCVTBIASPH2HF8S)
34123 NODE_NAME_CASE(VCVTNEPH2BF8)
34124 NODE_NAME_CASE(VCVTNEPH2BF8S)
34125 NODE_NAME_CASE(VCVTNEPH2HF8)
34126 NODE_NAME_CASE(VCVTNEPH2HF8S)
34127 NODE_NAME_CASE(VMCVTBIASPH2BF8)
34128 NODE_NAME_CASE(VMCVTBIASPH2BF8S)
34129 NODE_NAME_CASE(VMCVTBIASPH2HF8)
34130 NODE_NAME_CASE(VMCVTBIASPH2HF8S)
34131 NODE_NAME_CASE(VMCVTNEPH2BF8)
34132 NODE_NAME_CASE(VMCVTNEPH2BF8S)
34133 NODE_NAME_CASE(VMCVTNEPH2HF8)
34134 NODE_NAME_CASE(VMCVTNEPH2HF8S)
34135 NODE_NAME_CASE(VCVTHF82PH)
34136 NODE_NAME_CASE(AESENC128KL)
34137 NODE_NAME_CASE(AESDEC128KL)
34138 NODE_NAME_CASE(AESENC256KL)
34139 NODE_NAME_CASE(AESDEC256KL)
34140 NODE_NAME_CASE(AESENCWIDE128KL)
34141 NODE_NAME_CASE(AESDECWIDE128KL)
34142 NODE_NAME_CASE(AESENCWIDE256KL)
34143 NODE_NAME_CASE(AESDECWIDE256KL)
34144 NODE_NAME_CASE(CMPCCXADD)
34145 NODE_NAME_CASE(TESTUI)
34146 NODE_NAME_CASE(FP80_ADD)
34147 NODE_NAME_CASE(STRICT_FP80_ADD)
34148 NODE_NAME_CASE(CCMP)
34149 NODE_NAME_CASE(CTEST)
34150 NODE_NAME_CASE(CLOAD)
34151 NODE_NAME_CASE(CSTORE)
34152 }
34153 return nullptr;
34154#undef NODE_NAME_CASE
34155}
34156
34157/// Return true if the addressing mode represented by AM is legal for this
34158/// target, for a load/store of the specified type.
34159bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
34160 const AddrMode &AM, Type *Ty,
34161 unsigned AS,
34162 Instruction *I) const {
34163 // X86 supports extremely general addressing modes.
34164 CodeModel::Model M = getTargetMachine().getCodeModel();
34165
34166 // X86 allows a sign-extended 32-bit immediate field as a displacement.
34167 if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != nullptr))
34168 return false;
34169
34170 if (AM.BaseGV) {
34171 unsigned GVFlags = Subtarget.classifyGlobalReference(AM.BaseGV);
34172
34173 // If a reference to this global requires an extra load, we can't fold it.
34174 if (isGlobalStubReference(GVFlags))
34175 return false;
34176
34177 // If BaseGV requires a register for the PIC base, we cannot also have a
34178 // BaseReg specified.
34179 if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags))
34180 return false;
34181
34182 // If lower 4G is not available, then we must use rip-relative addressing.
34183 if ((M != CodeModel::Small || isPositionIndependent()) &&
34184 Subtarget.is64Bit() && (AM.BaseOffs || AM.Scale > 1))
34185 return false;
34186 }
34187
34188 switch (AM.Scale) {
34189 case 0:
34190 case 1:
34191 case 2:
34192 case 4:
34193 case 8:
34194 // These scales always work.
34195 break;
34196 case 3:
34197 case 5:
34198 case 9:
34199 // These scales are formed with basereg+scalereg. Only accept if there is
34200 // no basereg yet.
34201 if (AM.HasBaseReg)
34202 return false;
34203 break;
34204 default: // Other stuff never works.
34205 return false;
34206 }
34207
34208 return true;
34209}
34210
34211bool X86TargetLowering::isVectorShiftByScalarCheap(Type *Ty) const {
34212 unsigned Bits = Ty->getScalarSizeInBits();
34213
34214 // XOP has v16i8/v8i16/v4i32/v2i64 variable vector shifts.
34215 // Splitting for v32i8/v16i16 on XOP+AVX2 targets is still preferred.
34216 if (Subtarget.hasXOP() &&
34217 (Bits == 8 || Bits == 16 || Bits == 32 || Bits == 64))
34218 return false;
34219
34220 // AVX2 has vpsllv[dq] instructions (and other shifts) that make variable
34221 // shifts just as cheap as scalar ones.
34222 if (Subtarget.hasAVX2() && (Bits == 32 || Bits == 64))
34223 return false;
34224
34225 // AVX512BW has shifts such as vpsllvw.
34226 if (Subtarget.hasBWI() && Bits == 16)
34227 return false;
34228
34229 // Otherwise, it's significantly cheaper to shift by a scalar amount than by a
34230 // fully general vector.
34231 return true;
34232}
34233
34234bool X86TargetLowering::isBinOp(unsigned Opcode) const {
34235 switch (Opcode) {
34236 // These are non-commutative binops.
34237 // TODO: Add more X86ISD opcodes once we have test coverage.
34238 case X86ISD::ANDNP:
34239 case X86ISD::PCMPGT:
34240 case X86ISD::FMAX:
34241 case X86ISD::FMIN:
34242 case X86ISD::FANDN:
34243 case X86ISD::VPSHA:
34244 case X86ISD::VPSHL:
34245 case X86ISD::VSHLV:
34246 case X86ISD::VSRLV:
34247 case X86ISD::VSRAV:
34248 return true;
34249 }
34250
34251 return TargetLoweringBase::isBinOp(Opcode);
34252}
34253
34254bool X86TargetLowering::isCommutativeBinOp(unsigned Opcode) const {
34255 switch (Opcode) {
34256 // TODO: Add more X86ISD opcodes once we have test coverage.
34257 case X86ISD::PCMPEQ:
34258 case X86ISD::PMULDQ:
34259 case X86ISD::PMULUDQ:
34260 case X86ISD::FMAXC:
34261 case X86ISD::FMINC:
34262 case X86ISD::FAND:
34263 case X86ISD::FOR:
34264 case X86ISD::FXOR:
34265 return true;
34266 }
34267
34268 return TargetLoweringBase::isCommutativeBinOp(Opcode);
34269}
34270
34271bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
34272 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
34273 return false;
34274 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
34275 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
34276 return NumBits1 > NumBits2;
34277}
34278
34279bool X86TargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
34280 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
34281 return false;
34282
34283 if (!isTypeLegal(EVT::getEVT(Ty1)))
34284 return false;
34285
34286 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
34287
34288 // Assuming the caller doesn't have a zeroext or signext return parameter,
34289 // truncation all the way down to i1 is valid.
34290 return true;
34291}
34292
34293bool X86TargetLowering::isLegalICmpImmediate(int64_t Imm) const {
34294 return isInt<32>(Imm);
34295}
34296
34297bool X86TargetLowering::isLegalAddImmediate(int64_t Imm) const {
34298 // Can also use sub to handle negated immediates.
34299 return isInt<32>(Imm);
34300}
34301
34302bool X86TargetLowering::isLegalStoreImmediate(int64_t Imm) const {
34303 return isInt<32>(Imm);
34304}
34305
34306bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
34307 if (!VT1.isScalarInteger() || !VT2.isScalarInteger())
34308 return false;
34309 unsigned NumBits1 = VT1.getSizeInBits();
34310 unsigned NumBits2 = VT2.getSizeInBits();
34311 return NumBits1 > NumBits2;
34312}
34313
34314bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
34315 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
34316 return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget.is64Bit();
34317}
34318
34319bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
34320 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
34321 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget.is64Bit();
34322}
34323
34324bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
34325 EVT VT1 = Val.getValueType();
34326 if (isZExtFree(VT1, VT2))
34327 return true;
34328
34329 if (Val.getOpcode() != ISD::LOAD)
34330 return false;
34331
34332 if (!VT1.isSimple() || !VT1.isInteger() ||
34333 !VT2.isSimple() || !VT2.isInteger())
34334 return false;
34335
34336 switch (VT1.getSimpleVT().SimpleTy) {
34337 default: break;
34338 case MVT::i8:
34339 case MVT::i16:
34340 case MVT::i32:
34341 // X86 has 8, 16, and 32-bit zero-extending loads.
34342 return true;
34343 }
34344
34345 return false;
34346}
34347
34348bool X86TargetLowering::shouldSinkOperands(Instruction *I,
34349 SmallVectorImpl<Use *> &Ops) const {
34350 using namespace llvm::PatternMatch;
34351
34352 FixedVectorType *VTy = dyn_cast<FixedVectorType>(I->getType());
34353 if (!VTy)
34354 return false;
34355
34356 if (I->getOpcode() == Instruction::Mul &&
34357 VTy->getElementType()->isIntegerTy(64)) {
34358 for (auto &Op : I->operands()) {
34359 // Make sure we are not already sinking this operand
34360 if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
34361 continue;
34362
34363 // Look for PMULDQ pattern where the input is a sext_inreg from vXi32 or
34364 // the PMULUDQ pattern where the input is a zext_inreg from vXi32.
34365 if (Subtarget.hasSSE41() &&
34366 match(Op.get(), m_AShr(m_Shl(m_Value(), m_SpecificInt(32)),
34367 m_SpecificInt(32)))) {
34368 Ops.push_back(&cast<Instruction>(Op)->getOperandUse(0));
34369 Ops.push_back(&Op);
34370 } else if (Subtarget.hasSSE2() &&
34371 match(Op.get(),
34372 m_And(m_Value(), m_SpecificInt(UINT64_C(0xffffffff))))) {
34373 Ops.push_back(&Op);
34374 }
34375 }
34376
34377 return !Ops.empty();
34378 }
34379
34380 // A uniform shift amount in a vector shift or funnel shift may be much
34381 // cheaper than a generic variable vector shift, so make that pattern visible
34382 // to SDAG by sinking the shuffle instruction next to the shift.
34383 int ShiftAmountOpNum = -1;
34384 if (I->isShift())
34385 ShiftAmountOpNum = 1;
34386 else if (auto *II = dyn_cast<IntrinsicInst>(I)) {
34387 if (II->getIntrinsicID() == Intrinsic::fshl ||
34388 II->getIntrinsicID() == Intrinsic::fshr)
34389 ShiftAmountOpNum = 2;
34390 }
34391
34392 if (ShiftAmountOpNum == -1)
34393 return false;
34394
34395 auto *Shuf = dyn_cast<ShuffleVectorInst>(I->getOperand(ShiftAmountOpNum));
34396 if (Shuf && getSplatIndex(Shuf->getShuffleMask()) >= 0 &&
34397 isVectorShiftByScalarCheap(I->getType())) {
34398 Ops.push_back(&I->getOperandUse(ShiftAmountOpNum));
34399 return true;
34400 }
34401
34402 return false;
34403}
34404
34405bool X86TargetLowering::shouldConvertPhiType(Type *From, Type *To) const {
34406 if (!Subtarget.is64Bit())
34407 return false;
34408 return TargetLowering::shouldConvertPhiType(From, To);
34409}
34410
34411bool X86TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
34412 if (isa<MaskedLoadSDNode>(ExtVal.getOperand(0)))
34413 return false;
34414
34415 EVT SrcVT = ExtVal.getOperand(0).getValueType();
34416
34417 // There is no extending load for vXi1.
34418 if (SrcVT.getScalarType() == MVT::i1)
34419 return false;
34420
34421 return true;
34422}
34423
34424bool X86TargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
34425 EVT VT) const {
34426 if (!Subtarget.hasAnyFMA())
34427 return false;
34428
34429 VT = VT.getScalarType();
34430
34431 if (!VT.isSimple())
34432 return false;
34433
34434 switch (VT.getSimpleVT().SimpleTy) {
34435 case MVT::f16:
34436 return Subtarget.hasFP16();
34437 case MVT::f32:
34438 case MVT::f64:
34439 return true;
34440 default:
34441 break;
34442 }
34443
34444 return false;
34445}
34446
34447bool X86TargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const {
34448 // i16 instructions are longer (0x66 prefix) and potentially slower.
34449 return !(SrcVT == MVT::i32 && DestVT == MVT::i16);
34450}
34451
34452bool X86TargetLowering::shouldFoldSelectWithIdentityConstant(unsigned Opcode,
34453 EVT VT) const {
34454 // TODO: This is too general. There are cases where pre-AVX512 codegen would
34455 // benefit. The transform may also be profitable for scalar code.
34456 if (!Subtarget.hasAVX512())
34457 return false;
34458 if (!Subtarget.hasVLX() && !VT.is512BitVector())
34459 return false;
34460 if (!VT.isVector() || VT.getScalarType() == MVT::i1)
34461 return false;
34462
34463 return true;
34464}
34465
34466/// Targets can use this to indicate that they only support *some*
34467/// VECTOR_SHUFFLE operations, those with specific masks.
34468/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
34469/// are assumed to be legal.
34470bool X86TargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const {
34471 if (!VT.isSimple())
34472 return false;
34473
34474 // Not for i1 vectors
34475 if (VT.getSimpleVT().getScalarType() == MVT::i1)
34476 return false;
34477
34478 // Very little shuffling can be done for 64-bit vectors right now.
34479 if (VT.getSimpleVT().getSizeInBits() == 64)
34480 return false;
34481
34482 // We only care that the types being shuffled are legal. The lowering can
34483 // handle any possible shuffle mask that results.
34484 return isTypeLegal(VT.getSimpleVT());
34485}
34486
34487bool X86TargetLowering::isVectorClearMaskLegal(ArrayRef<int> Mask,
34488 EVT VT) const {
34489 // Don't convert an 'and' into a shuffle that we don't directly support.
34490 // vpblendw and vpshufb for 256-bit vectors are not available on AVX1.
34491 if (!Subtarget.hasAVX2())
34492 if (VT == MVT::v32i8 || VT == MVT::v16i16)
34493 return false;
34494
34495 // Just delegate to the generic legality, clear masks aren't special.
34496 return isShuffleMaskLegal(Mask, VT);
34497}
34498
34499bool X86TargetLowering::areJTsAllowed(const Function *Fn) const {
34500 // If the subtarget is using thunks, we need to not generate jump tables.
34501 if (Subtarget.useIndirectThunkBranches())
34502 return false;
34503
34504 // Otherwise, fallback on the generic logic.
34505 return TargetLowering::areJTsAllowed(Fn);
34506}
34507
34508MVT X86TargetLowering::getPreferredSwitchConditionType(LLVMContext &Context,
34509 EVT ConditionVT) const {
34510 // Avoid 8 and 16 bit types because they increase the chance for unnecessary
34511 // zero-extensions.
34512 if (ConditionVT.getSizeInBits() < 32)
34513 return MVT::i32;
34514 return TargetLoweringBase::getPreferredSwitchConditionType(Context,
34515 ConditionVT);
34516}
34517
34518//===----------------------------------------------------------------------===//
34519// X86 Scheduler Hooks
34520//===----------------------------------------------------------------------===//
34521
34522// Returns true if EFLAG is consumed after this iterator in the rest of the
34523// basic block or any successors of the basic block.
34524static bool isEFLAGSLiveAfter(MachineBasicBlock::iterator Itr,
34525 MachineBasicBlock *BB) {
34526 // Scan forward through BB for a use/def of EFLAGS.
34527 for (const MachineInstr &mi : llvm::make_range(std::next(Itr), BB->end())) {
34528 if (mi.readsRegister(X86::EFLAGS, /*TRI=*/nullptr))
34529 return true;
34530 // If we found a def, we can stop searching.
34531 if (mi.definesRegister(X86::EFLAGS, /*TRI=*/nullptr))
34532 return false;
34533 }
34534
34535 // If we hit the end of the block, check whether EFLAGS is live into a
34536 // successor.
34537 for (MachineBasicBlock *Succ : BB->successors())
34538 if (Succ->isLiveIn(X86::EFLAGS))
34539 return true;
34540
34541 return false;
34542}
34543
34544/// Utility function to emit xbegin specifying the start of an RTM region.
34545static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB,
34546 const TargetInstrInfo *TII) {
34547 const MIMetadata MIMD(MI);
34548
34549 const BasicBlock *BB = MBB->getBasicBlock();
34550 MachineFunction::iterator I = ++MBB->getIterator();
34551
34552 // For the v = xbegin(), we generate
34553 //
34554 // thisMBB:
34555 // xbegin sinkMBB
34556 //
34557 // mainMBB:
34558 // s0 = -1
34559 //
34560 // fallBB:
34561 // eax = # XABORT_DEF
34562 // s1 = eax
34563 //
34564 // sinkMBB:
34565 // v = phi(s0/mainBB, s1/fallBB)
34566
34567 MachineBasicBlock *thisMBB = MBB;
34568 MachineFunction *MF = MBB->getParent();
34569 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
34570 MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB);
34571 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
34572 MF->insert(I, mainMBB);
34573 MF->insert(I, fallMBB);
34574 MF->insert(I, sinkMBB);
34575
34576 if (isEFLAGSLiveAfter(MI, MBB)) {
34577 mainMBB->addLiveIn(X86::EFLAGS);
34578 fallMBB->addLiveIn(X86::EFLAGS);
34579 sinkMBB->addLiveIn(X86::EFLAGS);
34580 }
34581
34582 // Transfer the remainder of BB and its successor edges to sinkMBB.
34583 sinkMBB->splice(sinkMBB->begin(), MBB,
34584 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
34585 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
34586
34587 MachineRegisterInfo &MRI = MF->getRegInfo();
34588 Register DstReg = MI.getOperand(0).getReg();
34589 const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
34590 Register mainDstReg = MRI.createVirtualRegister(RC);
34591 Register fallDstReg = MRI.createVirtualRegister(RC);
34592
34593 // thisMBB:
34594 // xbegin fallMBB
34595 // # fallthrough to mainMBB
34596 // # abortion to fallMBB
34597 BuildMI(thisMBB, MIMD, TII->get(X86::XBEGIN_4)).addMBB(fallMBB);
34598 thisMBB->addSuccessor(mainMBB);
34599 thisMBB->addSuccessor(fallMBB);
34600
34601 // mainMBB:
34602 // mainDstReg := -1
34603 BuildMI(mainMBB, MIMD, TII->get(X86::MOV32ri), mainDstReg).addImm(-1);
34604 BuildMI(mainMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
34605 mainMBB->addSuccessor(sinkMBB);
34606
34607 // fallMBB:
34608 // ; pseudo instruction to model hardware's definition from XABORT
34609 // EAX := XABORT_DEF
34610 // fallDstReg := EAX
34611 BuildMI(fallMBB, MIMD, TII->get(X86::XABORT_DEF));
34612 BuildMI(fallMBB, MIMD, TII->get(TargetOpcode::COPY), fallDstReg)
34613 .addReg(X86::EAX);
34614 fallMBB->addSuccessor(sinkMBB);
34615
34616 // sinkMBB:
34617 // DstReg := phi(mainDstReg/mainBB, fallDstReg/fallBB)
34618 BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
34619 .addReg(mainDstReg).addMBB(mainMBB)
34620 .addReg(fallDstReg).addMBB(fallMBB);
34621
34622 MI.eraseFromParent();
34623 return sinkMBB;
34624}
34625
34626MachineBasicBlock *
34627X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI,
34628 MachineBasicBlock *MBB) const {
34629 // Emit va_arg instruction on X86-64.
34630
34631 // Operands to this pseudo-instruction:
34632 // 0 ) Output : destination address (reg)
34633 // 1-5) Input : va_list address (addr, i64mem)
34634 // 6 ) ArgSize : Size (in bytes) of vararg type
34635 // 7 ) ArgMode : 0=overflow only, 1=use gp_offset, 2=use fp_offset
34636 // 8 ) Align : Alignment of type
34637 // 9 ) EFLAGS (implicit-def)
34638
34639 assert(MI.getNumOperands() == 10 && "VAARG should have 10 operands!");
34640 static_assert(X86::AddrNumOperands == 5, "VAARG assumes 5 address operands");
34641
34642 Register DestReg = MI.getOperand(0).getReg();
34643 MachineOperand &Base = MI.getOperand(1);
34644 MachineOperand &Scale = MI.getOperand(2);
34645 MachineOperand &Index = MI.getOperand(3);
34646 MachineOperand &Disp = MI.getOperand(4);
34647 MachineOperand &Segment = MI.getOperand(5);
34648 unsigned ArgSize = MI.getOperand(6).getImm();
34649 unsigned ArgMode = MI.getOperand(7).getImm();
34650 Align Alignment = Align(MI.getOperand(8).getImm());
34651
34652 MachineFunction *MF = MBB->getParent();
34653
34654 // Memory Reference
34655 assert(MI.hasOneMemOperand() && "Expected VAARG to have one memoperand");
34656
34657 MachineMemOperand *OldMMO = MI.memoperands().front();
34658
34659 // Clone the MMO into two separate MMOs for loading and storing
34660 MachineMemOperand *LoadOnlyMMO = MF->getMachineMemOperand(
34661 OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOStore);
34662 MachineMemOperand *StoreOnlyMMO = MF->getMachineMemOperand(
34663 OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOLoad);
34664
34665 // Machine Information
34666 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
34667 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
34668 const TargetRegisterClass *AddrRegClass =
34669 getRegClassFor(getPointerTy(MBB->getParent()->getDataLayout()));
34670 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32);
34671 const MIMetadata MIMD(MI);
34672
34673 // struct va_list {
34674 // i32 gp_offset
34675 // i32 fp_offset
34676 // i64 overflow_area (address)
34677 // i64 reg_save_area (address)
34678 // }
34679 // sizeof(va_list) = 24
34680 // alignment(va_list) = 8
34681
34682 unsigned TotalNumIntRegs = 6;
34683 unsigned TotalNumXMMRegs = 8;
34684 bool UseGPOffset = (ArgMode == 1);
34685 bool UseFPOffset = (ArgMode == 2);
34686 unsigned MaxOffset = TotalNumIntRegs * 8 +
34687 (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
34688
34689 /* Align ArgSize to a multiple of 8 */
34690 unsigned ArgSizeA8 = (ArgSize + 7) & ~7;
34691 bool NeedsAlign = (Alignment > 8);
34692
34693 MachineBasicBlock *thisMBB = MBB;
34694 MachineBasicBlock *overflowMBB;
34695 MachineBasicBlock *offsetMBB;
34696 MachineBasicBlock *endMBB;
34697
34698 unsigned OffsetDestReg = 0; // Argument address computed by offsetMBB
34699 unsigned OverflowDestReg = 0; // Argument address computed by overflowMBB
34700 unsigned OffsetReg = 0;
34701
34702 if (!UseGPOffset && !UseFPOffset) {
34703 // If we only pull from the overflow region, we don't create a branch.
34704 // We don't need to alter control flow.
34705 OffsetDestReg = 0; // unused
34706 OverflowDestReg = DestReg;
34707
34708 offsetMBB = nullptr;
34709 overflowMBB = thisMBB;
34710 endMBB = thisMBB;
34711 } else {
34712 // First emit code to check if gp_offset (or fp_offset) is below the bound.
34713 // If so, pull the argument from reg_save_area. (branch to offsetMBB)
34714 // If not, pull from overflow_area. (branch to overflowMBB)
34715 //
34716 // thisMBB
34717 // | .
34718 // | .
34719 // offsetMBB overflowMBB
34720 // | .
34721 // | .
34722 // endMBB
34723
34724 // Registers for the PHI in endMBB
34725 OffsetDestReg = MRI.createVirtualRegister(AddrRegClass);
34726 OverflowDestReg = MRI.createVirtualRegister(AddrRegClass);
34727
34728 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
34729 overflowMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34730 offsetMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34731 endMBB = MF->CreateMachineBasicBlock(LLVM_BB);
34732
34733 MachineFunction::iterator MBBIter = ++MBB->getIterator();
34734
34735 // Insert the new basic blocks
34736 MF->insert(MBBIter, offsetMBB);
34737 MF->insert(MBBIter, overflowMBB);
34738 MF->insert(MBBIter, endMBB);
34739
34740 // Transfer the remainder of MBB and its successor edges to endMBB.
34741 endMBB->splice(endMBB->begin(), thisMBB,
34742 std::next(MachineBasicBlock::iterator(MI)), thisMBB->end());
34743 endMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
34744
34745 // Make offsetMBB and overflowMBB successors of thisMBB
34746 thisMBB->addSuccessor(offsetMBB);
34747 thisMBB->addSuccessor(overflowMBB);
34748
34749 // endMBB is a successor of both offsetMBB and overflowMBB
34750 offsetMBB->addSuccessor(endMBB);
34751 overflowMBB->addSuccessor(endMBB);
34752
34753 // Load the offset value into a register
34754 OffsetReg = MRI.createVirtualRegister(OffsetRegClass);
34755 BuildMI(thisMBB, MIMD, TII->get(X86::MOV32rm), OffsetReg)
34756 .add(Base)
34757 .add(Scale)
34758 .add(Index)
34759 .addDisp(Disp, UseFPOffset ? 4 : 0)
34760 .add(Segment)
34761 .setMemRefs(LoadOnlyMMO);
34762
34763 // Check if there is enough room left to pull this argument.
34764 BuildMI(thisMBB, MIMD, TII->get(X86::CMP32ri))
34765 .addReg(OffsetReg)
34766 .addImm(MaxOffset + 8 - ArgSizeA8);
34767
34768 // Branch to "overflowMBB" if offset >= max
34769 // Fall through to "offsetMBB" otherwise
34770 BuildMI(thisMBB, MIMD, TII->get(X86::JCC_1))
34771 .addMBB(overflowMBB).addImm(X86::COND_AE);
34772 }
34773
34774 // In offsetMBB, emit code to use the reg_save_area.
34775 if (offsetMBB) {
34776 assert(OffsetReg != 0);
34777
34778 // Read the reg_save_area address.
34779 Register RegSaveReg = MRI.createVirtualRegister(AddrRegClass);
34780 BuildMI(
34781 offsetMBB, MIMD,
34782 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm),
34783 RegSaveReg)
34784 .add(Base)
34785 .add(Scale)
34786 .add(Index)
34787 .addDisp(Disp, Subtarget.isTarget64BitLP64() ? 16 : 12)
34788 .add(Segment)
34789 .setMemRefs(LoadOnlyMMO);
34790
34791 if (Subtarget.isTarget64BitLP64()) {
34792 // Zero-extend the offset
34793 Register OffsetReg64 = MRI.createVirtualRegister(AddrRegClass);
34794 BuildMI(offsetMBB, MIMD, TII->get(X86::SUBREG_TO_REG), OffsetReg64)
34795 .addImm(0)
34796 .addReg(OffsetReg)
34797 .addImm(X86::sub_32bit);
34798
34799 // Add the offset to the reg_save_area to get the final address.
34800 BuildMI(offsetMBB, MIMD, TII->get(X86::ADD64rr), OffsetDestReg)
34801 .addReg(OffsetReg64)
34802 .addReg(RegSaveReg);
34803 } else {
34804 // Add the offset to the reg_save_area to get the final address.
34805 BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32rr), OffsetDestReg)
34806 .addReg(OffsetReg)
34807 .addReg(RegSaveReg);
34808 }
34809
34810 // Compute the offset for the next argument
34811 Register NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass);
34812 BuildMI(offsetMBB, MIMD, TII->get(X86::ADD32ri), NextOffsetReg)
34813 .addReg(OffsetReg)
34814 .addImm(UseFPOffset ? 16 : 8);
34815
34816 // Store it back into the va_list.
34817 BuildMI(offsetMBB, MIMD, TII->get(X86::MOV32mr))
34818 .add(Base)
34819 .add(Scale)
34820 .add(Index)
34821 .addDisp(Disp, UseFPOffset ? 4 : 0)
34822 .add(Segment)
34823 .addReg(NextOffsetReg)
34824 .setMemRefs(StoreOnlyMMO);
34825
34826 // Jump to endMBB
34827 BuildMI(offsetMBB, MIMD, TII->get(X86::JMP_1))
34828 .addMBB(endMBB);
34829 }
34830
34831 //
34832 // Emit code to use overflow area
34833 //
34834
34835 // Load the overflow_area address into a register.
34836 Register OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass);
34837 BuildMI(overflowMBB, MIMD,
34838 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm),
34839 OverflowAddrReg)
34840 .add(Base)
34841 .add(Scale)
34842 .add(Index)
34843 .addDisp(Disp, 8)
34844 .add(Segment)
34845 .setMemRefs(LoadOnlyMMO);
34846
34847 // If we need to align it, do so. Otherwise, just copy the address
34848 // to OverflowDestReg.
34849 if (NeedsAlign) {
34850 // Align the overflow address
34851 Register TmpReg = MRI.createVirtualRegister(AddrRegClass);
34852
34853 // aligned_addr = (addr + (align-1)) & ~(align-1)
34854 BuildMI(
34855 overflowMBB, MIMD,
34856 TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri),
34857 TmpReg)
34858 .addReg(OverflowAddrReg)
34859 .addImm(Alignment.value() - 1);
34860
34861 BuildMI(
34862 overflowMBB, MIMD,
34863 TII->get(Subtarget.isTarget64BitLP64() ? X86::AND64ri32 : X86::AND32ri),
34864 OverflowDestReg)
34865 .addReg(TmpReg)
34866 .addImm(~(uint64_t)(Alignment.value() - 1));
34867 } else {
34868 BuildMI(overflowMBB, MIMD, TII->get(TargetOpcode::COPY), OverflowDestReg)
34869 .addReg(OverflowAddrReg);
34870 }
34871
34872 // Compute the next overflow address after this argument.
34873 // (the overflow address should be kept 8-byte aligned)
34874 Register NextAddrReg = MRI.createVirtualRegister(AddrRegClass);
34875 BuildMI(
34876 overflowMBB, MIMD,
34877 TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri),
34878 NextAddrReg)
34879 .addReg(OverflowDestReg)
34880 .addImm(ArgSizeA8);
34881
34882 // Store the new overflow address.
34883 BuildMI(overflowMBB, MIMD,
34884 TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64mr : X86::MOV32mr))
34885 .add(Base)
34886 .add(Scale)
34887 .add(Index)
34888 .addDisp(Disp, 8)
34889 .add(Segment)
34890 .addReg(NextAddrReg)
34891 .setMemRefs(StoreOnlyMMO);
34892
34893 // If we branched, emit the PHI to the front of endMBB.
34894 if (offsetMBB) {
34895 BuildMI(*endMBB, endMBB->begin(), MIMD,
34896 TII->get(X86::PHI), DestReg)
34897 .addReg(OffsetDestReg).addMBB(offsetMBB)
34898 .addReg(OverflowDestReg).addMBB(overflowMBB);
34899 }
34900
34901 // Erase the pseudo instruction
34902 MI.eraseFromParent();
34903
34904 return endMBB;
34905}
34906
34907// The EFLAGS operand of SelectItr might be missing a kill marker
34908// because there were multiple uses of EFLAGS, and ISel didn't know
34909// which to mark. Figure out whether SelectItr should have had a
34910// kill marker, and set it if it should. Returns the correct kill
34911// marker value.
34912static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr,
34913 MachineBasicBlock* BB,
34914 const TargetRegisterInfo* TRI) {
34915 if (isEFLAGSLiveAfter(SelectItr, BB))
34916 return false;
34917
34918 // We found a def, or hit the end of the basic block and EFLAGS wasn't live
34919 // out. SelectMI should have a kill flag on EFLAGS.
34920 SelectItr->addRegisterKilled(X86::EFLAGS, TRI);
34921 return true;
34922}
34923
34924// Return true if it is OK for this CMOV pseudo-opcode to be cascaded
34925// together with other CMOV pseudo-opcodes into a single basic-block with
34926// conditional jump around it.
34927static bool isCMOVPseudo(MachineInstr &MI) {
34928 switch (MI.getOpcode()) {
34929 case X86::CMOV_FR16:
34930 case X86::CMOV_FR16X:
34931 case X86::CMOV_FR32:
34932 case X86::CMOV_FR32X:
34933 case X86::CMOV_FR64:
34934 case X86::CMOV_FR64X:
34935 case X86::CMOV_GR8:
34936 case X86::CMOV_GR16:
34937 case X86::CMOV_GR32:
34938 case X86::CMOV_RFP32:
34939 case X86::CMOV_RFP64:
34940 case X86::CMOV_RFP80:
34941 case X86::CMOV_VR64:
34942 case X86::CMOV_VR128:
34943 case X86::CMOV_VR128X:
34944 case X86::CMOV_VR256:
34945 case X86::CMOV_VR256X:
34946 case X86::CMOV_VR512:
34947 case X86::CMOV_VK1:
34948 case X86::CMOV_VK2:
34949 case X86::CMOV_VK4:
34950 case X86::CMOV_VK8:
34951 case X86::CMOV_VK16:
34952 case X86::CMOV_VK32:
34953 case X86::CMOV_VK64:
34954 return true;
34955
34956 default:
34957 return false;
34958 }
34959}
34960
34961// Helper function, which inserts PHI functions into SinkMBB:
34962// %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
34963// where %FalseValue(i) and %TrueValue(i) are taken from the consequent CMOVs
34964// in [MIItBegin, MIItEnd) range. It returns the last MachineInstrBuilder for
34965// the last PHI function inserted.
34966static MachineInstrBuilder createPHIsForCMOVsInSinkBB(
34967 MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd,
34968 MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB,
34969 MachineBasicBlock *SinkMBB) {
34970 MachineFunction *MF = TrueMBB->getParent();
34971 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
34972 const MIMetadata MIMD(*MIItBegin);
34973
34974 X86::CondCode CC = X86::CondCode(MIItBegin->getOperand(3).getImm());
34975 X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
34976
34977 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
34978
34979 // As we are creating the PHIs, we have to be careful if there is more than
34980 // one. Later CMOVs may reference the results of earlier CMOVs, but later
34981 // PHIs have to reference the individual true/false inputs from earlier PHIs.
34982 // That also means that PHI construction must work forward from earlier to
34983 // later, and that the code must maintain a mapping from earlier PHI's
34984 // destination registers, and the registers that went into the PHI.
34985 DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
34986 MachineInstrBuilder MIB;
34987
34988 for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd; ++MIIt) {
34989 Register DestReg = MIIt->getOperand(0).getReg();
34990 Register Op1Reg = MIIt->getOperand(1).getReg();
34991 Register Op2Reg = MIIt->getOperand(2).getReg();
34992
34993 // If this CMOV we are generating is the opposite condition from
34994 // the jump we generated, then we have to swap the operands for the
34995 // PHI that is going to be generated.
34996 if (MIIt->getOperand(3).getImm() == OppCC)
34997 std::swap(Op1Reg, Op2Reg);
34998
34999 if (RegRewriteTable.contains(Op1Reg))
35000 Op1Reg = RegRewriteTable[Op1Reg].first;
35001
35002 if (RegRewriteTable.contains(Op2Reg))
35003 Op2Reg = RegRewriteTable[Op2Reg].second;
35004
35005 MIB =
35006 BuildMI(*SinkMBB, SinkInsertionPoint, MIMD, TII->get(X86::PHI), DestReg)
35007 .addReg(Op1Reg)
35008 .addMBB(FalseMBB)
35009 .addReg(Op2Reg)
35010 .addMBB(TrueMBB);
35011
35012 // Add this PHI to the rewrite table.
35013 RegRewriteTable[DestReg] = std::make_pair(Op1Reg, Op2Reg);
35014 }
35015
35016 return MIB;
35017}
35018
35019// Lower cascaded selects in form of (SecondCmov (FirstCMOV F, T, cc1), T, cc2).
35020MachineBasicBlock *
35021X86TargetLowering::EmitLoweredCascadedSelect(MachineInstr &FirstCMOV,
35022 MachineInstr &SecondCascadedCMOV,
35023 MachineBasicBlock *ThisMBB) const {
35024 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35025 const MIMetadata MIMD(FirstCMOV);
35026
35027 // We lower cascaded CMOVs such as
35028 //
35029 // (SecondCascadedCMOV (FirstCMOV F, T, cc1), T, cc2)
35030 //
35031 // to two successive branches.
35032 //
35033 // Without this, we would add a PHI between the two jumps, which ends up
35034 // creating a few copies all around. For instance, for
35035 //
35036 // (sitofp (zext (fcmp une)))
35037 //
35038 // we would generate:
35039 //
35040 // ucomiss %xmm1, %xmm0
35041 // movss <1.0f>, %xmm0
35042 // movaps %xmm0, %xmm1
35043 // jne .LBB5_2
35044 // xorps %xmm1, %xmm1
35045 // .LBB5_2:
35046 // jp .LBB5_4
35047 // movaps %xmm1, %xmm0
35048 // .LBB5_4:
35049 // retq
35050 //
35051 // because this custom-inserter would have generated:
35052 //
35053 // A
35054 // | \
35055 // | B
35056 // | /
35057 // C
35058 // | \
35059 // | D
35060 // | /
35061 // E
35062 //
35063 // A: X = ...; Y = ...
35064 // B: empty
35065 // C: Z = PHI [X, A], [Y, B]
35066 // D: empty
35067 // E: PHI [X, C], [Z, D]
35068 //
35069 // If we lower both CMOVs in a single step, we can instead generate:
35070 //
35071 // A
35072 // | \
35073 // | C
35074 // | /|
35075 // |/ |
35076 // | |
35077 // | D
35078 // | /
35079 // E
35080 //
35081 // A: X = ...; Y = ...
35082 // D: empty
35083 // E: PHI [X, A], [X, C], [Y, D]
35084 //
35085 // Which, in our sitofp/fcmp example, gives us something like:
35086 //
35087 // ucomiss %xmm1, %xmm0
35088 // movss <1.0f>, %xmm0
35089 // jne .LBB5_4
35090 // jp .LBB5_4
35091 // xorps %xmm0, %xmm0
35092 // .LBB5_4:
35093 // retq
35094 //
35095
35096 // We lower cascaded CMOV into two successive branches to the same block.
35097 // EFLAGS is used by both, so mark it as live in the second.
35098 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
35099 MachineFunction *F = ThisMBB->getParent();
35100 MachineBasicBlock *FirstInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB);
35101 MachineBasicBlock *SecondInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB);
35102 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
35103
35104 MachineFunction::iterator It = ++ThisMBB->getIterator();
35105 F->insert(It, FirstInsertedMBB);
35106 F->insert(It, SecondInsertedMBB);
35107 F->insert(It, SinkMBB);
35108
35109 // For a cascaded CMOV, we lower it to two successive branches to
35110 // the same block (SinkMBB). EFLAGS is used by both, so mark it as live in
35111 // the FirstInsertedMBB.
35112 FirstInsertedMBB->addLiveIn(X86::EFLAGS);
35113
35114 // If the EFLAGS register isn't dead in the terminator, then claim that it's
35115 // live into the sink and copy blocks.
35116 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
35117 if (!SecondCascadedCMOV.killsRegister(X86::EFLAGS, /*TRI=*/nullptr) &&
35118 !checkAndUpdateEFLAGSKill(SecondCascadedCMOV, ThisMBB, TRI)) {
35119 SecondInsertedMBB->addLiveIn(X86::EFLAGS);
35120 SinkMBB->addLiveIn(X86::EFLAGS);
35121 }
35122
35123 // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
35124 SinkMBB->splice(SinkMBB->begin(), ThisMBB,
35125 std::next(MachineBasicBlock::iterator(FirstCMOV)),
35126 ThisMBB->end());
35127 SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB);
35128
35129 // Fallthrough block for ThisMBB.
35130 ThisMBB->addSuccessor(FirstInsertedMBB);
35131 // The true block target of the first branch is always SinkMBB.
35132 ThisMBB->addSuccessor(SinkMBB);
35133 // Fallthrough block for FirstInsertedMBB.
35134 FirstInsertedMBB->addSuccessor(SecondInsertedMBB);
35135 // The true block for the branch of FirstInsertedMBB.
35136 FirstInsertedMBB->addSuccessor(SinkMBB);
35137 // This is fallthrough.
35138 SecondInsertedMBB->addSuccessor(SinkMBB);
35139
35140 // Create the conditional branch instructions.
35141 X86::CondCode FirstCC = X86::CondCode(FirstCMOV.getOperand(3).getImm());
35142 BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(FirstCC);
35143
35144 X86::CondCode SecondCC =
35145 X86::CondCode(SecondCascadedCMOV.getOperand(3).getImm());
35146 BuildMI(FirstInsertedMBB, MIMD, TII->get(X86::JCC_1))
35147 .addMBB(SinkMBB)
35148 .addImm(SecondCC);
35149
35150 // SinkMBB:
35151 // %Result = phi [ %FalseValue, SecondInsertedMBB ], [ %TrueValue, ThisMBB ]
35152 Register DestReg = SecondCascadedCMOV.getOperand(0).getReg();
35153 Register Op1Reg = FirstCMOV.getOperand(1).getReg();
35154 Register Op2Reg = FirstCMOV.getOperand(2).getReg();
35155 MachineInstrBuilder MIB =
35156 BuildMI(*SinkMBB, SinkMBB->begin(), MIMD, TII->get(X86::PHI), DestReg)
35157 .addReg(Op1Reg)
35158 .addMBB(SecondInsertedMBB)
35159 .addReg(Op2Reg)
35160 .addMBB(ThisMBB);
35161
35162 // The second SecondInsertedMBB provides the same incoming value as the
35163 // FirstInsertedMBB (the True operand of the SELECT_CC/CMOV nodes).
35164 MIB.addReg(FirstCMOV.getOperand(2).getReg()).addMBB(FirstInsertedMBB);
35165
35166 // Now remove the CMOVs.
35167 FirstCMOV.eraseFromParent();
35168 SecondCascadedCMOV.eraseFromParent();
35169
35170 return SinkMBB;
35171}
35172
35173MachineBasicBlock *
35174X86TargetLowering::EmitLoweredSelect(MachineInstr &MI,
35175 MachineBasicBlock *ThisMBB) const {
35176 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35177 const MIMetadata MIMD(MI);
35178
35179 // To "insert" a SELECT_CC instruction, we actually have to insert the
35180 // diamond control-flow pattern. The incoming instruction knows the
35181 // destination vreg to set, the condition code register to branch on, the
35182 // true/false values to select between and a branch opcode to use.
35183
35184 // ThisMBB:
35185 // ...
35186 // TrueVal = ...
35187 // cmpTY ccX, r1, r2
35188 // bCC copy1MBB
35189 // fallthrough --> FalseMBB
35190
35191 // This code lowers all pseudo-CMOV instructions. Generally it lowers these
35192 // as described above, by inserting a BB, and then making a PHI at the join
35193 // point to select the true and false operands of the CMOV in the PHI.
35194 //
35195 // The code also handles two different cases of multiple CMOV opcodes
35196 // in a row.
35197 //
35198 // Case 1:
35199 // In this case, there are multiple CMOVs in a row, all which are based on
35200 // the same condition setting (or the exact opposite condition setting).
35201 // In this case we can lower all the CMOVs using a single inserted BB, and
35202 // then make a number of PHIs at the join point to model the CMOVs. The only
35203 // trickiness here, is that in a case like:
35204 //
35205 // t2 = CMOV cond1 t1, f1
35206 // t3 = CMOV cond1 t2, f2
35207 //
35208 // when rewriting this into PHIs, we have to perform some renaming on the
35209 // temps since you cannot have a PHI operand refer to a PHI result earlier
35210 // in the same block. The "simple" but wrong lowering would be:
35211 //
35212 // t2 = PHI t1(BB1), f1(BB2)
35213 // t3 = PHI t2(BB1), f2(BB2)
35214 //
35215 // but clearly t2 is not defined in BB1, so that is incorrect. The proper
35216 // renaming is to note that on the path through BB1, t2 is really just a
35217 // copy of t1, and do that renaming, properly generating:
35218 //
35219 // t2 = PHI t1(BB1), f1(BB2)
35220 // t3 = PHI t1(BB1), f2(BB2)
35221 //
35222 // Case 2:
35223 // CMOV ((CMOV F, T, cc1), T, cc2) is checked here and handled by a separate
35224 // function - EmitLoweredCascadedSelect.
35225
35226 X86::CondCode CC = X86::CondCode(MI.getOperand(3).getImm());
35227 X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
35228 MachineInstr *LastCMOV = &MI;
35229 MachineBasicBlock::iterator NextMIIt = MachineBasicBlock::iterator(MI);
35230
35231 // Check for case 1, where there are multiple CMOVs with the same condition
35232 // first. Of the two cases of multiple CMOV lowerings, case 1 reduces the
35233 // number of jumps the most.
35234
35235 if (isCMOVPseudo(MI)) {
35236 // See if we have a string of CMOVS with the same condition. Skip over
35237 // intervening debug insts.
35238 while (NextMIIt != ThisMBB->end() && isCMOVPseudo(*NextMIIt) &&
35239 (NextMIIt->getOperand(3).getImm() == CC ||
35240 NextMIIt->getOperand(3).getImm() == OppCC)) {
35241 LastCMOV = &*NextMIIt;
35242 NextMIIt = next_nodbg(NextMIIt, ThisMBB->end());
35243 }
35244 }
35245
35246 // This checks for case 2, but only do this if we didn't already find
35247 // case 1, as indicated by LastCMOV == MI.
35248 if (LastCMOV == &MI && NextMIIt != ThisMBB->end() &&
35249 NextMIIt->getOpcode() == MI.getOpcode() &&
35250 NextMIIt->getOperand(2).getReg() == MI.getOperand(2).getReg() &&
35251 NextMIIt->getOperand(1).getReg() == MI.getOperand(0).getReg() &&
35252 NextMIIt->getOperand(1).isKill()) {
35253 return EmitLoweredCascadedSelect(MI, *NextMIIt, ThisMBB);
35254 }
35255
35256 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
35257 MachineFunction *F = ThisMBB->getParent();
35258 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
35259 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
35260
35261 MachineFunction::iterator It = ++ThisMBB->getIterator();
35262 F->insert(It, FalseMBB);
35263 F->insert(It, SinkMBB);
35264
35265 // Set the call frame size on entry to the new basic blocks.
35266 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
35267 FalseMBB->setCallFrameSize(CallFrameSize);
35268 SinkMBB->setCallFrameSize(CallFrameSize);
35269
35270 // If the EFLAGS register isn't dead in the terminator, then claim that it's
35271 // live into the sink and copy blocks.
35272 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
35273 if (!LastCMOV->killsRegister(X86::EFLAGS, /*TRI=*/nullptr) &&
35274 !checkAndUpdateEFLAGSKill(LastCMOV, ThisMBB, TRI)) {
35275 FalseMBB->addLiveIn(X86::EFLAGS);
35276 SinkMBB->addLiveIn(X86::EFLAGS);
35277 }
35278
35279 // Transfer any debug instructions inside the CMOV sequence to the sunk block.
35280 auto DbgRange = llvm::make_range(MachineBasicBlock::iterator(MI),
35281 MachineBasicBlock::iterator(LastCMOV));
35282 for (MachineInstr &MI : llvm::make_early_inc_range(DbgRange))
35283 if (MI.isDebugInstr())
35284 SinkMBB->push_back(MI.removeFromParent());
35285
35286 // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
35287 SinkMBB->splice(SinkMBB->end(), ThisMBB,
35288 std::next(MachineBasicBlock::iterator(LastCMOV)),
35289 ThisMBB->end());
35290 SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB);
35291
35292 // Fallthrough block for ThisMBB.
35293 ThisMBB->addSuccessor(FalseMBB);
35294 // The true block target of the first (or only) branch is always a SinkMBB.
35295 ThisMBB->addSuccessor(SinkMBB);
35296 // Fallthrough block for FalseMBB.
35297 FalseMBB->addSuccessor(SinkMBB);
35298
35299 // Create the conditional branch instruction.
35300 BuildMI(ThisMBB, MIMD, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(CC);
35301
35302 // SinkMBB:
35303 // %Result = phi [ %FalseValue, FalseMBB ], [ %TrueValue, ThisMBB ]
35304 // ...
35305 MachineBasicBlock::iterator MIItBegin = MachineBasicBlock::iterator(MI);
35306 MachineBasicBlock::iterator MIItEnd =
35307 std::next(MachineBasicBlock::iterator(LastCMOV));
35308 createPHIsForCMOVsInSinkBB(MIItBegin, MIItEnd, ThisMBB, FalseMBB, SinkMBB);
35309
35310 // Now remove the CMOV(s).
35311 ThisMBB->erase(MIItBegin, MIItEnd);
35312
35313 return SinkMBB;
35314}
35315
35316static unsigned getSUBriOpcode(bool IsLP64) {
35317 if (IsLP64)
35318 return X86::SUB64ri32;
35319 else
35320 return X86::SUB32ri;
35321}
35322
35323MachineBasicBlock *
35324X86TargetLowering::EmitLoweredProbedAlloca(MachineInstr &MI,
35325 MachineBasicBlock *MBB) const {
35326 MachineFunction *MF = MBB->getParent();
35327 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35328 const X86FrameLowering &TFI = *Subtarget.getFrameLowering();
35329 const MIMetadata MIMD(MI);
35330 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
35331
35332 const unsigned ProbeSize = getStackProbeSize(*MF);
35333
35334 MachineRegisterInfo &MRI = MF->getRegInfo();
35335 MachineBasicBlock *testMBB = MF->CreateMachineBasicBlock(LLVM_BB);
35336 MachineBasicBlock *tailMBB = MF->CreateMachineBasicBlock(LLVM_BB);
35337 MachineBasicBlock *blockMBB = MF->CreateMachineBasicBlock(LLVM_BB);
35338
35339 MachineFunction::iterator MBBIter = ++MBB->getIterator();
35340 MF->insert(MBBIter, testMBB);
35341 MF->insert(MBBIter, blockMBB);
35342 MF->insert(MBBIter, tailMBB);
35343
35344 Register sizeVReg = MI.getOperand(1).getReg();
35345
35346 Register physSPReg = TFI.Uses64BitFramePtr ? X86::RSP : X86::ESP;
35347
35348 Register TmpStackPtr = MRI.createVirtualRegister(
35349 TFI.Uses64BitFramePtr ? &X86::GR64RegClass : &X86::GR32RegClass);
35350 Register FinalStackPtr = MRI.createVirtualRegister(
35351 TFI.Uses64BitFramePtr ? &X86::GR64RegClass : &X86::GR32RegClass);
35352
35353 BuildMI(*MBB, {MI}, MIMD, TII->get(TargetOpcode::COPY), TmpStackPtr)
35354 .addReg(physSPReg);
35355 {
35356 const unsigned Opc = TFI.Uses64BitFramePtr ? X86::SUB64rr : X86::SUB32rr;
35357 BuildMI(*MBB, {MI}, MIMD, TII->get(Opc), FinalStackPtr)
35358 .addReg(TmpStackPtr)
35359 .addReg(sizeVReg);
35360 }
35361
35362 // test rsp size
35363
35364 BuildMI(testMBB, MIMD,
35365 TII->get(TFI.Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr))
35366 .addReg(FinalStackPtr)
35367 .addReg(physSPReg);
35368
35369 BuildMI(testMBB, MIMD, TII->get(X86::JCC_1))
35370 .addMBB(tailMBB)
35371 .addImm(X86::COND_GE);
35372 testMBB->addSuccessor(blockMBB);
35373 testMBB->addSuccessor(tailMBB);
35374
35375 // Touch the block then extend it. This is done on the opposite side of
35376 // static probe where we allocate then touch, to avoid the need of probing the
35377 // tail of the static alloca. Possible scenarios are:
35378 //
35379 // + ---- <- ------------ <- ------------- <- ------------ +
35380 // | |
35381 // [free probe] -> [page alloc] -> [alloc probe] -> [tail alloc] + -> [dyn probe] -> [page alloc] -> [dyn probe] -> [tail alloc] +
35382 // | |
35383 // + <- ----------- <- ------------ <- ----------- <- ------------ +
35384 //
35385 // The property we want to enforce is to never have more than [page alloc] between two probes.
35386
35387 const unsigned XORMIOpc =
35388 TFI.Uses64BitFramePtr ? X86::XOR64mi32 : X86::XOR32mi;
35389 addRegOffset(BuildMI(blockMBB, MIMD, TII->get(XORMIOpc)), physSPReg, false, 0)
35390 .addImm(0);
35391
35392 BuildMI(blockMBB, MIMD, TII->get(getSUBriOpcode(TFI.Uses64BitFramePtr)),
35393 physSPReg)
35394 .addReg(physSPReg)
35395 .addImm(ProbeSize);
35396
35397 BuildMI(blockMBB, MIMD, TII->get(X86::JMP_1)).addMBB(testMBB);
35398 blockMBB->addSuccessor(testMBB);
35399
35400 // Replace original instruction by the expected stack ptr
35401 BuildMI(tailMBB, MIMD, TII->get(TargetOpcode::COPY),
35402 MI.getOperand(0).getReg())
35403 .addReg(FinalStackPtr);
35404
35405 tailMBB->splice(tailMBB->end(), MBB,
35406 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
35407 tailMBB->transferSuccessorsAndUpdatePHIs(MBB);
35408 MBB->addSuccessor(testMBB);
35409
35410 // Delete the original pseudo instruction.
35411 MI.eraseFromParent();
35412
35413 // And we're done.
35414 return tailMBB;
35415}
35416
35417MachineBasicBlock *
35418X86TargetLowering::EmitLoweredSegAlloca(MachineInstr &MI,
35419 MachineBasicBlock *BB) const {
35420 MachineFunction *MF = BB->getParent();
35421 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35422 const MIMetadata MIMD(MI);
35423 const BasicBlock *LLVM_BB = BB->getBasicBlock();
35424
35425 assert(MF->shouldSplitStack());
35426
35427 const bool Is64Bit = Subtarget.is64Bit();
35428 const bool IsLP64 = Subtarget.isTarget64BitLP64();
35429
35430 const unsigned TlsReg = Is64Bit ? X86::FS : X86::GS;
35431 const unsigned TlsOffset = IsLP64 ? 0x70 : Is64Bit ? 0x40 : 0x30;
35432
35433 // BB:
35434 // ... [Till the alloca]
35435 // If stacklet is not large enough, jump to mallocMBB
35436 //
35437 // bumpMBB:
35438 // Allocate by subtracting from RSP
35439 // Jump to continueMBB
35440 //
35441 // mallocMBB:
35442 // Allocate by call to runtime
35443 //
35444 // continueMBB:
35445 // ...
35446 // [rest of original BB]
35447 //
35448
35449 MachineBasicBlock *mallocMBB = MF->CreateMachineBasicBlock(LLVM_BB);
35450 MachineBasicBlock *bumpMBB = MF->CreateMachineBasicBlock(LLVM_BB);
35451 MachineBasicBlock *continueMBB = MF->CreateMachineBasicBlock(LLVM_BB);
35452
35453 MachineRegisterInfo &MRI = MF->getRegInfo();
35454 const TargetRegisterClass *AddrRegClass =
35455 getRegClassFor(getPointerTy(MF->getDataLayout()));
35456
35457 Register mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass),
35458 bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass),
35459 tmpSPVReg = MRI.createVirtualRegister(AddrRegClass),
35460 SPLimitVReg = MRI.createVirtualRegister(AddrRegClass),
35461 sizeVReg = MI.getOperand(1).getReg(),
35462 physSPReg =
35463 IsLP64 || Subtarget.isTargetNaCl64() ? X86::RSP : X86::ESP;
35464
35465 MachineFunction::iterator MBBIter = ++BB->getIterator();
35466
35467 MF->insert(MBBIter, bumpMBB);
35468 MF->insert(MBBIter, mallocMBB);
35469 MF->insert(MBBIter, continueMBB);
35470
35471 continueMBB->splice(continueMBB->begin(), BB,
35472 std::next(MachineBasicBlock::iterator(MI)), BB->end());
35473 continueMBB->transferSuccessorsAndUpdatePHIs(BB);
35474
35475 // Add code to the main basic block to check if the stack limit has been hit,
35476 // and if so, jump to mallocMBB otherwise to bumpMBB.
35477 BuildMI(BB, MIMD, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
35478 BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
35479 .addReg(tmpSPVReg).addReg(sizeVReg);
35480 BuildMI(BB, MIMD, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
35481 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg)
35482 .addReg(SPLimitVReg);
35483 BuildMI(BB, MIMD, TII->get(X86::JCC_1)).addMBB(mallocMBB).addImm(X86::COND_G);
35484
35485 // bumpMBB simply decreases the stack pointer, since we know the current
35486 // stacklet has enough space.
35487 BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), physSPReg)
35488 .addReg(SPLimitVReg);
35489 BuildMI(bumpMBB, MIMD, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
35490 .addReg(SPLimitVReg);
35491 BuildMI(bumpMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
35492
35493 // Calls into a routine in libgcc to allocate more space from the heap.
35494 const uint32_t *RegMask =
35495 Subtarget.getRegisterInfo()->getCallPreservedMask(*MF, CallingConv::C);
35496 if (IsLP64) {
35497 BuildMI(mallocMBB, MIMD, TII->get(X86::MOV64rr), X86::RDI)
35498 .addReg(sizeVReg);
35499 BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
35500 .addExternalSymbol("__morestack_allocate_stack_space")
35501 .addRegMask(RegMask)
35502 .addReg(X86::RDI, RegState::Implicit)
35503 .addReg(X86::RAX, RegState::ImplicitDefine);
35504 } else if (Is64Bit) {
35505 BuildMI(mallocMBB, MIMD, TII->get(X86::MOV32rr), X86::EDI)
35506 .addReg(sizeVReg);
35507 BuildMI(mallocMBB, MIMD, TII->get(X86::CALL64pcrel32))
35508 .addExternalSymbol("__morestack_allocate_stack_space")
35509 .addRegMask(RegMask)
35510 .addReg(X86::EDI, RegState::Implicit)
35511 .addReg(X86::EAX, RegState::ImplicitDefine);
35512 } else {
35513 BuildMI(mallocMBB, MIMD, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
35514 .addImm(12);
35515 BuildMI(mallocMBB, MIMD, TII->get(X86::PUSH32r)).addReg(sizeVReg);
35516 BuildMI(mallocMBB, MIMD, TII->get(X86::CALLpcrel32))
35517 .addExternalSymbol("__morestack_allocate_stack_space")
35518 .addRegMask(RegMask)
35519 .addReg(X86::EAX, RegState::ImplicitDefine);
35520 }
35521
35522 if (!Is64Bit)
35523 BuildMI(mallocMBB, MIMD, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
35524 .addImm(16);
35525
35526 BuildMI(mallocMBB, MIMD, TII->get(TargetOpcode::COPY), mallocPtrVReg)
35527 .addReg(IsLP64 ? X86::RAX : X86::EAX);
35528 BuildMI(mallocMBB, MIMD, TII->get(X86::JMP_1)).addMBB(continueMBB);
35529
35530 // Set up the CFG correctly.
35531 BB->addSuccessor(bumpMBB);
35532 BB->addSuccessor(mallocMBB);
35533 mallocMBB->addSuccessor(continueMBB);
35534 bumpMBB->addSuccessor(continueMBB);
35535
35536 // Take care of the PHI nodes.
35537 BuildMI(*continueMBB, continueMBB->begin(), MIMD, TII->get(X86::PHI),
35538 MI.getOperand(0).getReg())
35539 .addReg(mallocPtrVReg)
35540 .addMBB(mallocMBB)
35541 .addReg(bumpSPPtrVReg)
35542 .addMBB(bumpMBB);
35543
35544 // Delete the original pseudo instruction.
35545 MI.eraseFromParent();
35546
35547 // And we're done.
35548 return continueMBB;
35549}
35550
35551MachineBasicBlock *
35552X86TargetLowering::EmitLoweredCatchRet(MachineInstr &MI,
35553 MachineBasicBlock *BB) const {
35554 MachineFunction *MF = BB->getParent();
35555 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
35556 MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
35557 const MIMetadata MIMD(MI);
35558
35559 assert(!isAsynchronousEHPersonality(
35560 classifyEHPersonality(MF->getFunction().getPersonalityFn())) &&
35561 "SEH does not use catchret!");
35562
35563 // Only 32-bit EH needs to worry about manually restoring stack pointers.
35564 if (!Subtarget.is32Bit())
35565 return BB;
35566
35567 // C++ EH creates a new target block to hold the restore code, and wires up
35568 // the new block to the return destination with a normal JMP_4.
35569 MachineBasicBlock *RestoreMBB =
35570 MF->CreateMachineBasicBlock(BB->getBasicBlock());
35571 assert(BB->succ_size() == 1);
35572 MF->insert(std::next(BB->getIterator()), RestoreMBB);
35573 RestoreMBB->transferSuccessorsAndUpdatePHIs(BB);
35574 BB->addSuccessor(RestoreMBB);
35575 MI.getOperand(0).setMBB(RestoreMBB);
35576
35577 // Marking this as an EH pad but not a funclet entry block causes PEI to
35578 // restore stack pointers in the block.
35579 RestoreMBB->setIsEHPad(true);
35580
35581 auto RestoreMBBI = RestoreMBB->begin();
35582 BuildMI(*RestoreMBB, RestoreMBBI, MIMD, TII.get(X86::JMP_4)).addMBB(TargetMBB);
35583 return BB;
35584}
35585
35586MachineBasicBlock *
35587X86TargetLowering::EmitLoweredTLSAddr(MachineInstr &MI,
35588 MachineBasicBlock *BB) const {
35589 // So, here we replace TLSADDR with the sequence:
35590 // adjust_stackdown -> TLSADDR -> adjust_stackup.
35591 // We need this because TLSADDR is lowered into calls
35592 // inside MC, therefore without the two markers shrink-wrapping
35593 // may push the prologue/epilogue pass them.
35594 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
35595 const MIMetadata MIMD(MI);
35596 MachineFunction &MF = *BB->getParent();
35597
35598 // Emit CALLSEQ_START right before the instruction.
35599 MF.getFrameInfo().setAdjustsStack(true);
35600 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
35601 MachineInstrBuilder CallseqStart =
35602 BuildMI(MF, MIMD, TII.get(AdjStackDown)).addImm(0).addImm(0).addImm(0);
35603 BB->insert(MachineBasicBlock::iterator(MI), CallseqStart);
35604
35605 // Emit CALLSEQ_END right after the instruction.
35606 // We don't call erase from parent because we want to keep the
35607 // original instruction around.
35608 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
35609 MachineInstrBuilder CallseqEnd =
35610 BuildMI(MF, MIMD, TII.get(AdjStackUp)).addImm(0).addImm(0);
35611 BB->insertAfter(MachineBasicBlock::iterator(MI), CallseqEnd);
35612
35613 return BB;
35614}
35615
35616MachineBasicBlock *
35617X86TargetLowering::EmitLoweredTLSCall(MachineInstr &MI,
35618 MachineBasicBlock *BB) const {
35619 // This is pretty easy. We're taking the value that we received from
35620 // our load from the relocation, sticking it in either RDI (x86-64)
35621 // or EAX and doing an indirect call. The return value will then
35622 // be in the normal return register.
35623 MachineFunction *F = BB->getParent();
35624 const X86InstrInfo *TII = Subtarget.getInstrInfo();
35625 const MIMetadata MIMD(MI);
35626
35627 assert(Subtarget.isTargetDarwin() && "Darwin only instr emitted?");
35628 assert(MI.getOperand(3).isGlobal() && "This should be a global");
35629
35630 // Get a register mask for the lowered call.
35631 // FIXME: The 32-bit calls have non-standard calling conventions. Use a
35632 // proper register mask.
35633 const uint32_t *RegMask =
35634 Subtarget.is64Bit() ?
35635 Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask() :
35636 Subtarget.getRegisterInfo()->getCallPreservedMask(*F, CallingConv::C);
35637 if (Subtarget.is64Bit()) {
35638 MachineInstrBuilder MIB =
35639 BuildMI(*BB, MI, MIMD, TII->get(X86::MOV64rm), X86::RDI)
35640 .addReg(X86::RIP)
35641 .addImm(0)
35642 .addReg(0)
35643 .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
35644 MI.getOperand(3).getTargetFlags())
35645 .addReg(0);
35646 MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL64m));
35647 addDirectMem(MIB, X86::RDI);
35648 MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask);
35649 } else if (!isPositionIndependent()) {
35650 MachineInstrBuilder MIB =
35651 BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX)
35652 .addReg(0)
35653 .addImm(0)
35654 .addReg(0)
35655 .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
35656 MI.getOperand(3).getTargetFlags())
35657 .addReg(0);
35658 MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m));
35659 addDirectMem(MIB, X86::EAX);
35660 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
35661 } else {
35662 MachineInstrBuilder MIB =
35663 BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), X86::EAX)
35664 .addReg(TII->getGlobalBaseReg(F))
35665 .addImm(0)
35666 .addReg(0)
35667 .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
35668 MI.getOperand(3).getTargetFlags())
35669 .addReg(0);
35670 MIB = BuildMI(*BB, MI, MIMD, TII->get(X86::CALL32m));
35671 addDirectMem(MIB, X86::EAX);
35672 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
35673 }
35674
35675 MI.eraseFromParent(); // The pseudo instruction is gone now.
35676 return BB;
35677}
35678
35679static unsigned getOpcodeForIndirectThunk(unsigned RPOpc) {
35680 switch (RPOpc) {
35681 case X86::INDIRECT_THUNK_CALL32:
35682 return X86::CALLpcrel32;
35683 case X86::INDIRECT_THUNK_CALL64:
35684 return X86::CALL64pcrel32;
35685 case X86::INDIRECT_THUNK_TCRETURN32:
35686 return X86::TCRETURNdi;
35687 case X86::INDIRECT_THUNK_TCRETURN64:
35688 return X86::TCRETURNdi64;
35689 }
35690 llvm_unreachable("not indirect thunk opcode");
35691}
35692
35693static const char *getIndirectThunkSymbol(const X86Subtarget &Subtarget,
35694 unsigned Reg) {
35695 if (Subtarget.useRetpolineExternalThunk()) {
35696 // When using an external thunk for retpolines, we pick names that match the
35697 // names GCC happens to use as well. This helps simplify the implementation
35698 // of the thunks for kernels where they have no easy ability to create
35699 // aliases and are doing non-trivial configuration of the thunk's body. For
35700 // example, the Linux kernel will do boot-time hot patching of the thunk
35701 // bodies and cannot easily export aliases of these to loaded modules.
35702 //
35703 // Note that at any point in the future, we may need to change the semantics
35704 // of how we implement retpolines and at that time will likely change the
35705 // name of the called thunk. Essentially, there is no hard guarantee that
35706 // LLVM will generate calls to specific thunks, we merely make a best-effort
35707 // attempt to help out kernels and other systems where duplicating the
35708 // thunks is costly.
35709 switch (Reg) {
35710 case X86::EAX:
35711 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35712 return "__x86_indirect_thunk_eax";
35713 case X86::ECX:
35714 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35715 return "__x86_indirect_thunk_ecx";
35716 case X86::EDX:
35717 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35718 return "__x86_indirect_thunk_edx";
35719 case X86::EDI:
35720 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35721 return "__x86_indirect_thunk_edi";
35722 case X86::R11:
35723 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
35724 return "__x86_indirect_thunk_r11";
35725 }
35726 llvm_unreachable("unexpected reg for external indirect thunk");
35727 }
35728
35729 if (Subtarget.useRetpolineIndirectCalls() ||
35730 Subtarget.useRetpolineIndirectBranches()) {
35731 // When targeting an internal COMDAT thunk use an LLVM-specific name.
35732 switch (Reg) {
35733 case X86::EAX:
35734 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35735 return "__llvm_retpoline_eax";
35736 case X86::ECX:
35737 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35738 return "__llvm_retpoline_ecx";
35739 case X86::EDX:
35740 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35741 return "__llvm_retpoline_edx";
35742 case X86::EDI:
35743 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!");
35744 return "__llvm_retpoline_edi";
35745 case X86::R11:
35746 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
35747 return "__llvm_retpoline_r11";
35748 }
35749 llvm_unreachable("unexpected reg for retpoline");
35750 }
35751
35752 if (Subtarget.useLVIControlFlowIntegrity()) {
35753 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!");
35754 return "__llvm_lvi_thunk_r11";
35755 }
35756 llvm_unreachable("getIndirectThunkSymbol() invoked without thunk feature");
35757}
35758
35759MachineBasicBlock *
35760X86TargetLowering::EmitLoweredIndirectThunk(MachineInstr &MI,
35761 MachineBasicBlock *BB) const {
35762 // Copy the virtual register into the R11 physical register and
35763 // call the retpoline thunk.
35764 const MIMetadata MIMD(MI);
35765 const X86InstrInfo *TII = Subtarget.getInstrInfo();
35766 Register CalleeVReg = MI.getOperand(0).getReg();
35767 unsigned Opc = getOpcodeForIndirectThunk(MI.getOpcode());
35768
35769 // Find an available scratch register to hold the callee. On 64-bit, we can
35770 // just use R11, but we scan for uses anyway to ensure we don't generate
35771 // incorrect code. On 32-bit, we use one of EAX, ECX, or EDX that isn't
35772 // already a register use operand to the call to hold the callee. If none
35773 // are available, use EDI instead. EDI is chosen because EBX is the PIC base
35774 // register and ESI is the base pointer to realigned stack frames with VLAs.
35775 SmallVector<unsigned, 3> AvailableRegs;
35776 if (Subtarget.is64Bit())
35777 AvailableRegs.push_back(X86::R11);
35778 else
35779 AvailableRegs.append({X86::EAX, X86::ECX, X86::EDX, X86::EDI});
35780
35781 // Zero out any registers that are already used.
35782 for (const auto &MO : MI.operands()) {
35783 if (MO.isReg() && MO.isUse())
35784 for (unsigned &Reg : AvailableRegs)
35785 if (Reg == MO.getReg())
35786 Reg = 0;
35787 }
35788
35789 // Choose the first remaining non-zero available register.
35790 unsigned AvailableReg = 0;
35791 for (unsigned MaybeReg : AvailableRegs) {
35792 if (MaybeReg) {
35793 AvailableReg = MaybeReg;
35794 break;
35795 }
35796 }
35797 if (!AvailableReg)
35798 report_fatal_error("calling convention incompatible with retpoline, no "
35799 "available registers");
35800
35801 const char *Symbol = getIndirectThunkSymbol(Subtarget, AvailableReg);
35802
35803 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), AvailableReg)
35804 .addReg(CalleeVReg);
35805 MI.getOperand(0).ChangeToES(Symbol);
35806 MI.setDesc(TII->get(Opc));
35807 MachineInstrBuilder(*BB->getParent(), &MI)
35808 .addReg(AvailableReg, RegState::Implicit | RegState::Kill);
35809 return BB;
35810}
35811
35812/// SetJmp implies future control flow change upon calling the corresponding
35813/// LongJmp.
35814/// Instead of using the 'return' instruction, the long jump fixes the stack and
35815/// performs an indirect branch. To do so it uses the registers that were stored
35816/// in the jump buffer (when calling SetJmp).
35817/// In case the shadow stack is enabled we need to fix it as well, because some
35818/// return addresses will be skipped.
35819/// The function will save the SSP for future fixing in the function
35820/// emitLongJmpShadowStackFix.
35821/// \sa emitLongJmpShadowStackFix
35822/// \param [in] MI The temporary Machine Instruction for the builtin.
35823/// \param [in] MBB The Machine Basic Block that will be modified.
35824void X86TargetLowering::emitSetJmpShadowStackFix(MachineInstr &MI,
35825 MachineBasicBlock *MBB) const {
35826 const MIMetadata MIMD(MI);
35827 MachineFunction *MF = MBB->getParent();
35828 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35829 MachineRegisterInfo &MRI = MF->getRegInfo();
35830 MachineInstrBuilder MIB;
35831
35832 // Memory Reference.
35833 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands());
35834
35835 // Initialize a register with zero.
35836 MVT PVT = getPointerTy(MF->getDataLayout());
35837 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
35838 Register ZReg = MRI.createVirtualRegister(PtrRC);
35839 unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr;
35840 BuildMI(*MBB, MI, MIMD, TII->get(XorRROpc))
35841 .addDef(ZReg)
35842 .addReg(ZReg, RegState::Undef)
35843 .addReg(ZReg, RegState::Undef);
35844
35845 // Read the current SSP Register value to the zeroed register.
35846 Register SSPCopyReg = MRI.createVirtualRegister(PtrRC);
35847 unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
35848 BuildMI(*MBB, MI, MIMD, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
35849
35850 // Write the SSP register value to offset 3 in input memory buffer.
35851 unsigned PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
35852 MIB = BuildMI(*MBB, MI, MIMD, TII->get(PtrStoreOpc));
35853 const int64_t SSPOffset = 3 * PVT.getStoreSize();
35854 const unsigned MemOpndSlot = 1;
35855 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35856 if (i == X86::AddrDisp)
35857 MIB.addDisp(MI.getOperand(MemOpndSlot + i), SSPOffset);
35858 else
35859 MIB.add(MI.getOperand(MemOpndSlot + i));
35860 }
35861 MIB.addReg(SSPCopyReg);
35862 MIB.setMemRefs(MMOs);
35863}
35864
35865MachineBasicBlock *
35866X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
35867 MachineBasicBlock *MBB) const {
35868 const MIMetadata MIMD(MI);
35869 MachineFunction *MF = MBB->getParent();
35870 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
35871 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
35872 MachineRegisterInfo &MRI = MF->getRegInfo();
35873
35874 const BasicBlock *BB = MBB->getBasicBlock();
35875 MachineFunction::iterator I = ++MBB->getIterator();
35876
35877 // Memory Reference
35878 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands());
35879
35880 unsigned DstReg;
35881 unsigned MemOpndSlot = 0;
35882
35883 unsigned CurOp = 0;
35884
35885 DstReg = MI.getOperand(CurOp++).getReg();
35886 const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
35887 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
35888 (void)TRI;
35889 Register mainDstReg = MRI.createVirtualRegister(RC);
35890 Register restoreDstReg = MRI.createVirtualRegister(RC);
35891
35892 MemOpndSlot = CurOp;
35893
35894 MVT PVT = getPointerTy(MF->getDataLayout());
35895 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
35896 "Invalid Pointer Size!");
35897
35898 // For v = setjmp(buf), we generate
35899 //
35900 // thisMBB:
35901 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB
35902 // SjLjSetup restoreMBB
35903 //
35904 // mainMBB:
35905 // v_main = 0
35906 //
35907 // sinkMBB:
35908 // v = phi(main, restore)
35909 //
35910 // restoreMBB:
35911 // if base pointer being used, load it from frame
35912 // v_restore = 1
35913
35914 MachineBasicBlock *thisMBB = MBB;
35915 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
35916 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
35917 MachineBasicBlock *restoreMBB = MF->CreateMachineBasicBlock(BB);
35918 MF->insert(I, mainMBB);
35919 MF->insert(I, sinkMBB);
35920 MF->push_back(restoreMBB);
35921 restoreMBB->setMachineBlockAddressTaken();
35922
35923 MachineInstrBuilder MIB;
35924
35925 // Transfer the remainder of BB and its successor edges to sinkMBB.
35926 sinkMBB->splice(sinkMBB->begin(), MBB,
35927 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
35928 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
35929
35930 // thisMBB:
35931 unsigned PtrStoreOpc = 0;
35932 unsigned LabelReg = 0;
35933 const int64_t LabelOffset = 1 * PVT.getStoreSize();
35934 bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
35935 !isPositionIndependent();
35936
35937 // Prepare IP either in reg or imm.
35938 if (!UseImmLabel) {
35939 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
35940 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
35941 LabelReg = MRI.createVirtualRegister(PtrRC);
35942 if (Subtarget.is64Bit()) {
35943 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA64r), LabelReg)
35944 .addReg(X86::RIP)
35945 .addImm(0)
35946 .addReg(0)
35947 .addMBB(restoreMBB)
35948 .addReg(0);
35949 } else {
35950 const X86InstrInfo *XII = static_cast<const X86InstrInfo*>(TII);
35951 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::LEA32r), LabelReg)
35952 .addReg(XII->getGlobalBaseReg(MF))
35953 .addImm(0)
35954 .addReg(0)
35955 .addMBB(restoreMBB, Subtarget.classifyBlockAddressReference())
35956 .addReg(0);
35957 }
35958 } else
35959 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
35960 // Store IP
35961 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrStoreOpc));
35962 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
35963 if (i == X86::AddrDisp)
35964 MIB.addDisp(MI.getOperand(MemOpndSlot + i), LabelOffset);
35965 else
35966 MIB.add(MI.getOperand(MemOpndSlot + i));
35967 }
35968 if (!UseImmLabel)
35969 MIB.addReg(LabelReg);
35970 else
35971 MIB.addMBB(restoreMBB);
35972 MIB.setMemRefs(MMOs);
35973
35974 if (MF->getFunction().getParent()->getModuleFlag("cf-protection-return")) {
35975 emitSetJmpShadowStackFix(MI, thisMBB);
35976 }
35977
35978 // Setup
35979 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(X86::EH_SjLj_Setup))
35980 .addMBB(restoreMBB);
35981
35982 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
35983 MIB.addRegMask(RegInfo->getNoPreservedMask());
35984 thisMBB->addSuccessor(mainMBB);
35985 thisMBB->addSuccessor(restoreMBB);
35986
35987 // mainMBB:
35988 // EAX = 0
35989 BuildMI(mainMBB, MIMD, TII->get(X86::MOV32r0), mainDstReg);
35990 mainMBB->addSuccessor(sinkMBB);
35991
35992 // sinkMBB:
35993 BuildMI(*sinkMBB, sinkMBB->begin(), MIMD, TII->get(X86::PHI), DstReg)
35994 .addReg(mainDstReg)
35995 .addMBB(mainMBB)
35996 .addReg(restoreDstReg)
35997 .addMBB(restoreMBB);
35998
35999 // restoreMBB:
36000 if (RegInfo->hasBasePointer(*MF)) {
36001 const bool Uses64BitFramePtr =
36002 Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64();
36003 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
36004 X86FI->setRestoreBasePointer(MF);
36005 Register FramePtr = RegInfo->getFrameRegister(*MF);
36006 Register BasePtr = RegInfo->getBaseRegister();
36007 unsigned Opm = Uses64BitFramePtr ? X86::MOV64rm : X86::MOV32rm;
36008 addRegOffset(BuildMI(restoreMBB, MIMD, TII->get(Opm), BasePtr),
36009 FramePtr, true, X86FI->getRestoreBasePointerOffset())
36010 .setMIFlag(MachineInstr::FrameSetup);
36011 }
36012 BuildMI(restoreMBB, MIMD, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
36013 BuildMI(restoreMBB, MIMD, TII->get(X86::JMP_1)).addMBB(sinkMBB);
36014 restoreMBB->addSuccessor(sinkMBB);
36015
36016 MI.eraseFromParent();
36017 return sinkMBB;
36018}
36019
36020/// Fix the shadow stack using the previously saved SSP pointer.
36021/// \sa emitSetJmpShadowStackFix
36022/// \param [in] MI The temporary Machine Instruction for the builtin.
36023/// \param [in] MBB The Machine Basic Block that will be modified.
36024/// \return The sink MBB that will perform the future indirect branch.
36025MachineBasicBlock *
36026X86TargetLowering::emitLongJmpShadowStackFix(MachineInstr &MI,
36027 MachineBasicBlock *MBB) const {
36028 const MIMetadata MIMD(MI);
36029 MachineFunction *MF = MBB->getParent();
36030 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
36031 MachineRegisterInfo &MRI = MF->getRegInfo();
36032
36033 // Memory Reference
36034 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands());
36035
36036 MVT PVT = getPointerTy(MF->getDataLayout());
36037 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
36038
36039 // checkSspMBB:
36040 // xor vreg1, vreg1
36041 // rdssp vreg1
36042 // test vreg1, vreg1
36043 // je sinkMBB # Jump if Shadow Stack is not supported
36044 // fallMBB:
36045 // mov buf+24/12(%rip), vreg2
36046 // sub vreg1, vreg2
36047 // jbe sinkMBB # No need to fix the Shadow Stack
36048 // fixShadowMBB:
36049 // shr 3/2, vreg2
36050 // incssp vreg2 # fix the SSP according to the lower 8 bits
36051 // shr 8, vreg2
36052 // je sinkMBB
36053 // fixShadowLoopPrepareMBB:
36054 // shl vreg2
36055 // mov 128, vreg3
36056 // fixShadowLoopMBB:
36057 // incssp vreg3
36058 // dec vreg2
36059 // jne fixShadowLoopMBB # Iterate until you finish fixing
36060 // # the Shadow Stack
36061 // sinkMBB:
36062
36063 MachineFunction::iterator I = ++MBB->getIterator();
36064 const BasicBlock *BB = MBB->getBasicBlock();
36065
36066 MachineBasicBlock *checkSspMBB = MF->CreateMachineBasicBlock(BB);
36067 MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB);
36068 MachineBasicBlock *fixShadowMBB = MF->CreateMachineBasicBlock(BB);
36069 MachineBasicBlock *fixShadowLoopPrepareMBB = MF->CreateMachineBasicBlock(BB);
36070 MachineBasicBlock *fixShadowLoopMBB = MF->CreateMachineBasicBlock(BB);
36071 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
36072 MF->insert(I, checkSspMBB);
36073 MF->insert(I, fallMBB);
36074 MF->insert(I, fixShadowMBB);
36075 MF->insert(I, fixShadowLoopPrepareMBB);
36076 MF->insert(I, fixShadowLoopMBB);
36077 MF->insert(I, sinkMBB);
36078
36079 // Transfer the remainder of BB and its successor edges to sinkMBB.
36080 sinkMBB->splice(sinkMBB->begin(), MBB, MachineBasicBlock::iterator(MI),
36081 MBB->end());
36082 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
36083
36084 MBB->addSuccessor(checkSspMBB);
36085
36086 // Initialize a register with zero.
36087 Register ZReg = MRI.createVirtualRegister(&X86::GR32RegClass);
36088 BuildMI(checkSspMBB, MIMD, TII->get(X86::MOV32r0), ZReg);
36089
36090 if (PVT == MVT::i64) {
36091 Register TmpZReg = MRI.createVirtualRegister(PtrRC);
36092 BuildMI(checkSspMBB, MIMD, TII->get(X86::SUBREG_TO_REG), TmpZReg)
36093 .addImm(0)
36094 .addReg(ZReg)
36095 .addImm(X86::sub_32bit);
36096 ZReg = TmpZReg;
36097 }
36098
36099 // Read the current SSP Register value to the zeroed register.
36100 Register SSPCopyReg = MRI.createVirtualRegister(PtrRC);
36101 unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
36102 BuildMI(checkSspMBB, MIMD, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
36103
36104 // Check whether the result of the SSP register is zero and jump directly
36105 // to the sink.
36106 unsigned TestRROpc = (PVT == MVT::i64) ? X86::TEST64rr : X86::TEST32rr;
36107 BuildMI(checkSspMBB, MIMD, TII->get(TestRROpc))
36108 .addReg(SSPCopyReg)
36109 .addReg(SSPCopyReg);
36110 BuildMI(checkSspMBB, MIMD, TII->get(X86::JCC_1))
36111 .addMBB(sinkMBB)
36112 .addImm(X86::COND_E);
36113 checkSspMBB->addSuccessor(sinkMBB);
36114 checkSspMBB->addSuccessor(fallMBB);
36115
36116 // Reload the previously saved SSP register value.
36117 Register PrevSSPReg = MRI.createVirtualRegister(PtrRC);
36118 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
36119 const int64_t SPPOffset = 3 * PVT.getStoreSize();
36120 MachineInstrBuilder MIB =
36121 BuildMI(fallMBB, MIMD, TII->get(PtrLoadOpc), PrevSSPReg);
36122 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
36123 const MachineOperand &MO = MI.getOperand(i);
36124 if (i == X86::AddrDisp)
36125 MIB.addDisp(MO, SPPOffset);
36126 else if (MO.isReg()) // Don't add the whole operand, we don't want to
36127 // preserve kill flags.
36128 MIB.addReg(MO.getReg());
36129 else
36130 MIB.add(MO);
36131 }
36132 MIB.setMemRefs(MMOs);
36133
36134 // Subtract the current SSP from the previous SSP.
36135 Register SspSubReg = MRI.createVirtualRegister(PtrRC);
36136 unsigned SubRROpc = (PVT == MVT::i64) ? X86::SUB64rr : X86::SUB32rr;
36137 BuildMI(fallMBB, MIMD, TII->get(SubRROpc), SspSubReg)
36138 .addReg(PrevSSPReg)
36139 .addReg(SSPCopyReg);
36140
36141 // Jump to sink in case PrevSSPReg <= SSPCopyReg.
36142 BuildMI(fallMBB, MIMD, TII->get(X86::JCC_1))
36143 .addMBB(sinkMBB)
36144 .addImm(X86::COND_BE);
36145 fallMBB->addSuccessor(sinkMBB);
36146 fallMBB->addSuccessor(fixShadowMBB);
36147
36148 // Shift right by 2/3 for 32/64 because incssp multiplies the argument by 4/8.
36149 unsigned ShrRIOpc = (PVT == MVT::i64) ? X86::SHR64ri : X86::SHR32ri;
36150 unsigned Offset = (PVT == MVT::i64) ? 3 : 2;
36151 Register SspFirstShrReg = MRI.createVirtualRegister(PtrRC);
36152 BuildMI(fixShadowMBB, MIMD, TII->get(ShrRIOpc), SspFirstShrReg)
36153 .addReg(SspSubReg)
36154 .addImm(Offset);
36155
36156 // Increase SSP when looking only on the lower 8 bits of the delta.
36157 unsigned IncsspOpc = (PVT == MVT::i64) ? X86::INCSSPQ : X86::INCSSPD;
36158 BuildMI(fixShadowMBB, MIMD, TII->get(IncsspOpc)).addReg(SspFirstShrReg);
36159
36160 // Reset the lower 8 bits.
36161 Register SspSecondShrReg = MRI.createVirtualRegister(PtrRC);
36162 BuildMI(fixShadowMBB, MIMD, TII->get(ShrRIOpc), SspSecondShrReg)
36163 .addReg(SspFirstShrReg)
36164 .addImm(8);
36165
36166 // Jump if the result of the shift is zero.
36167 BuildMI(fixShadowMBB, MIMD, TII->get(X86::JCC_1))
36168 .addMBB(sinkMBB)
36169 .addImm(X86::COND_E);
36170 fixShadowMBB->addSuccessor(sinkMBB);
36171 fixShadowMBB->addSuccessor(fixShadowLoopPrepareMBB);
36172
36173 // Do a single shift left.
36174 unsigned ShlR1Opc = (PVT == MVT::i64) ? X86::SHL64ri : X86::SHL32ri;
36175 Register SspAfterShlReg = MRI.createVirtualRegister(PtrRC);
36176 BuildMI(fixShadowLoopPrepareMBB, MIMD, TII->get(ShlR1Opc), SspAfterShlReg)
36177 .addReg(SspSecondShrReg)
36178 .addImm(1);
36179
36180 // Save the value 128 to a register (will be used next with incssp).
36181 Register Value128InReg = MRI.createVirtualRegister(PtrRC);
36182 unsigned MovRIOpc = (PVT == MVT::i64) ? X86::MOV64ri32 : X86::MOV32ri;
36183 BuildMI(fixShadowLoopPrepareMBB, MIMD, TII->get(MovRIOpc), Value128InReg)
36184 .addImm(128);
36185 fixShadowLoopPrepareMBB->addSuccessor(fixShadowLoopMBB);
36186
36187 // Since incssp only looks at the lower 8 bits, we might need to do several
36188 // iterations of incssp until we finish fixing the shadow stack.
36189 Register DecReg = MRI.createVirtualRegister(PtrRC);
36190 Register CounterReg = MRI.createVirtualRegister(PtrRC);
36191 BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::PHI), CounterReg)
36192 .addReg(SspAfterShlReg)
36193 .addMBB(fixShadowLoopPrepareMBB)
36194 .addReg(DecReg)
36195 .addMBB(fixShadowLoopMBB);
36196
36197 // Every iteration we increase the SSP by 128.
36198 BuildMI(fixShadowLoopMBB, MIMD, TII->get(IncsspOpc)).addReg(Value128InReg);
36199
36200 // Every iteration we decrement the counter by 1.
36201 unsigned DecROpc = (PVT == MVT::i64) ? X86::DEC64r : X86::DEC32r;
36202 BuildMI(fixShadowLoopMBB, MIMD, TII->get(DecROpc), DecReg).addReg(CounterReg);
36203
36204 // Jump if the counter is not zero yet.
36205 BuildMI(fixShadowLoopMBB, MIMD, TII->get(X86::JCC_1))
36206 .addMBB(fixShadowLoopMBB)
36207 .addImm(X86::COND_NE);
36208 fixShadowLoopMBB->addSuccessor(sinkMBB);
36209 fixShadowLoopMBB->addSuccessor(fixShadowLoopMBB);
36210
36211 return sinkMBB;
36212}
36213
36214MachineBasicBlock *
36215X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
36216 MachineBasicBlock *MBB) const {
36217 const MIMetadata MIMD(MI);
36218 MachineFunction *MF = MBB->getParent();
36219 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
36220 MachineRegisterInfo &MRI = MF->getRegInfo();
36221
36222 // Memory Reference
36223 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands());
36224
36225 MVT PVT = getPointerTy(MF->getDataLayout());
36226 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
36227 "Invalid Pointer Size!");
36228
36229 const TargetRegisterClass *RC =
36230 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
36231 Register Tmp = MRI.createVirtualRegister(RC);
36232 // Since FP is only updated here but NOT referenced, it's treated as GPR.
36233 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
36234 Register FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
36235 Register SP = RegInfo->getStackRegister();
36236
36237 MachineInstrBuilder MIB;
36238
36239 const int64_t LabelOffset = 1 * PVT.getStoreSize();
36240 const int64_t SPOffset = 2 * PVT.getStoreSize();
36241
36242 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
36243 unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r;
36244
36245 MachineBasicBlock *thisMBB = MBB;
36246
36247 // When CET and shadow stack is enabled, we need to fix the Shadow Stack.
36248 if (MF->getFunction().getParent()->getModuleFlag("cf-protection-return")) {
36249 thisMBB = emitLongJmpShadowStackFix(MI, thisMBB);
36250 }
36251
36252 // Reload FP
36253 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), FP);
36254 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
36255 const MachineOperand &MO = MI.getOperand(i);
36256 if (MO.isReg()) // Don't add the whole operand, we don't want to
36257 // preserve kill flags.
36258 MIB.addReg(MO.getReg());
36259 else
36260 MIB.add(MO);
36261 }
36262 MIB.setMemRefs(MMOs);
36263 MIB.setMIFlag(MachineInstr::FrameDestroy);
36264
36265 // Reload IP
36266 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), Tmp);
36267 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
36268 const MachineOperand &MO = MI.getOperand(i);
36269 if (i == X86::AddrDisp)
36270 MIB.addDisp(MO, LabelOffset);
36271 else if (MO.isReg()) // Don't add the whole operand, we don't want to
36272 // preserve kill flags.
36273 MIB.addReg(MO.getReg());
36274 else
36275 MIB.add(MO);
36276 }
36277 MIB.setMemRefs(MMOs);
36278
36279 // Reload SP
36280 MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), SP);
36281 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
36282 if (i == X86::AddrDisp)
36283 MIB.addDisp(MI.getOperand(i), SPOffset);
36284 else
36285 MIB.add(MI.getOperand(i)); // We can preserve the kill flags here, it's
36286 // the last instruction of the expansion.
36287 }
36288 MIB.setMemRefs(MMOs);
36289 MIB.setMIFlag(MachineInstr::FrameDestroy);
36290
36291 // Jump
36292 BuildMI(*thisMBB, MI, MIMD, TII->get(IJmpOpc)).addReg(Tmp);
36293
36294 MI.eraseFromParent();
36295 return thisMBB;
36296}
36297
36298void X86TargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
36299 MachineBasicBlock *MBB,
36300 MachineBasicBlock *DispatchBB,
36301 int FI) const {
36302 const MIMetadata MIMD(MI);
36303 MachineFunction *MF = MBB->getParent();
36304 MachineRegisterInfo *MRI = &MF->getRegInfo();
36305 const X86InstrInfo *TII = Subtarget.getInstrInfo();
36306
36307 MVT PVT = getPointerTy(MF->getDataLayout());
36308 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
36309
36310 unsigned Op = 0;
36311 unsigned VR = 0;
36312
36313 bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
36314 !isPositionIndependent();
36315
36316 if (UseImmLabel) {
36317 Op = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
36318 } else {
36319 const TargetRegisterClass *TRC =
36320 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
36321 VR = MRI->createVirtualRegister(TRC);
36322 Op = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
36323
36324 if (Subtarget.is64Bit())
36325 BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA64r), VR)
36326 .addReg(X86::RIP)
36327 .addImm(1)
36328 .addReg(0)
36329 .addMBB(DispatchBB)
36330 .addReg(0);
36331 else
36332 BuildMI(*MBB, MI, MIMD, TII->get(X86::LEA32r), VR)
36333 .addReg(0) /* TII->getGlobalBaseReg(MF) */
36334 .addImm(1)
36335 .addReg(0)
36336 .addMBB(DispatchBB, Subtarget.classifyBlockAddressReference())
36337 .addReg(0);
36338 }
36339
36340 MachineInstrBuilder MIB = BuildMI(*MBB, MI, MIMD, TII->get(Op));
36341 addFrameReference(MIB, FI, Subtarget.is64Bit() ? 56 : 36);
36342 if (UseImmLabel)
36343 MIB.addMBB(DispatchBB);
36344 else
36345 MIB.addReg(VR);
36346}
36347
36348MachineBasicBlock *
36349X86TargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
36350 MachineBasicBlock *BB) const {
36351 const MIMetadata MIMD(MI);
36352 MachineFunction *MF = BB->getParent();
36353 MachineRegisterInfo *MRI = &MF->getRegInfo();
36354 const X86InstrInfo *TII = Subtarget.getInstrInfo();
36355 int FI = MF->getFrameInfo().getFunctionContextIndex();
36356
36357 // Get a mapping of the call site numbers to all of the landing pads they're
36358 // associated with.
36359 DenseMap<unsigned, SmallVector<MachineBasicBlock *, 2>> CallSiteNumToLPad;
36360 unsigned MaxCSNum = 0;
36361 for (auto &MBB : *MF) {
36362 if (!MBB.isEHPad())
36363 continue;
36364
36365 MCSymbol *Sym = nullptr;
36366 for (const auto &MI : MBB) {
36367 if (MI.isDebugInstr())
36368 continue;
36369
36370 assert(MI.isEHLabel() && "expected EH_LABEL");
36371 Sym = MI.getOperand(0).getMCSymbol();
36372 break;
36373 }
36374
36375 if (!MF->hasCallSiteLandingPad(Sym))
36376 continue;
36377
36378 for (unsigned CSI : MF->getCallSiteLandingPad(Sym)) {
36379 CallSiteNumToLPad[CSI].push_back(&MBB);
36380 MaxCSNum = std::max(MaxCSNum, CSI);
36381 }
36382 }
36383
36384 // Get an ordered list of the machine basic blocks for the jump table.
36385 std::vector<MachineBasicBlock *> LPadList;
36386 SmallPtrSet<MachineBasicBlock *, 32> InvokeBBs;
36387 LPadList.reserve(CallSiteNumToLPad.size());
36388
36389 for (unsigned CSI = 1; CSI <= MaxCSNum; ++CSI) {
36390 for (auto &LP : CallSiteNumToLPad[CSI]) {
36391 LPadList.push_back(LP);
36392 InvokeBBs.insert(LP->pred_begin(), LP->pred_end());
36393 }
36394 }
36395
36396 assert(!LPadList.empty() &&
36397 "No landing pad destinations for the dispatch jump table!");
36398
36399 // Create the MBBs for the dispatch code.
36400
36401 // Shove the dispatch's address into the return slot in the function context.
36402 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
36403 DispatchBB->setIsEHPad(true);
36404
36405 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
36406 BuildMI(TrapBB, MIMD, TII->get(X86::TRAP));
36407 DispatchBB->addSuccessor(TrapBB);
36408
36409 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
36410 DispatchBB->addSuccessor(DispContBB);
36411
36412 // Insert MBBs.
36413 MF->push_back(DispatchBB);
36414 MF->push_back(DispContBB);
36415 MF->push_back(TrapBB);
36416
36417 // Insert code into the entry block that creates and registers the function
36418 // context.
36419 SetupEntryBlockForSjLj(MI, BB, DispatchBB, FI);
36420
36421 // Create the jump table and associated information
36422 unsigned JTE = getJumpTableEncoding();
36423 MachineJumpTableInfo *JTI = MF->getOrCreateJumpTableInfo(JTE);
36424 unsigned MJTI = JTI->createJumpTableIndex(LPadList);
36425
36426 const X86RegisterInfo &RI = TII->getRegisterInfo();
36427 // Add a register mask with no preserved registers. This results in all
36428 // registers being marked as clobbered.
36429 if (RI.hasBasePointer(*MF)) {
36430 const bool FPIs64Bit =
36431 Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64();
36432 X86MachineFunctionInfo *MFI = MF->getInfo<X86MachineFunctionInfo>();
36433 MFI->setRestoreBasePointer(MF);
36434
36435 Register FP = RI.getFrameRegister(*MF);
36436 Register BP = RI.getBaseRegister();
36437 unsigned Op = FPIs64Bit ? X86::MOV64rm : X86::MOV32rm;
36438 addRegOffset(BuildMI(DispatchBB, MIMD, TII->get(Op), BP), FP, true,
36439 MFI->getRestoreBasePointerOffset())
36440 .addRegMask(RI.getNoPreservedMask());
36441 } else {
36442 BuildMI(DispatchBB, MIMD, TII->get(X86::NOOP))
36443 .addRegMask(RI.getNoPreservedMask());
36444 }
36445
36446 // IReg is used as an index in a memory operand and therefore can't be SP
36447 Register IReg = MRI->createVirtualRegister(&X86::GR32_NOSPRegClass);
36448 addFrameReference(BuildMI(DispatchBB, MIMD, TII->get(X86::MOV32rm), IReg), FI,
36449 Subtarget.is64Bit() ? 8 : 4);
36450 BuildMI(DispatchBB, MIMD, TII->get(X86::CMP32ri))
36451 .addReg(IReg)
36452 .addImm(LPadList.size());
36453 BuildMI(DispatchBB, MIMD, TII->get(X86::JCC_1))
36454 .addMBB(TrapBB)
36455 .addImm(X86::COND_AE);
36456
36457 if (Subtarget.is64Bit()) {
36458 Register BReg = MRI->createVirtualRegister(&X86::GR64RegClass);
36459 Register IReg64 = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
36460
36461 // leaq .LJTI0_0(%rip), BReg
36462 BuildMI(DispContBB, MIMD, TII->get(X86::LEA64r), BReg)
36463 .addReg(X86::RIP)
36464 .addImm(1)
36465 .addReg(0)
36466 .addJumpTableIndex(MJTI)
36467 .addReg(0);
36468 // movzx IReg64, IReg
36469 BuildMI(DispContBB, MIMD, TII->get(TargetOpcode::SUBREG_TO_REG), IReg64)
36470 .addImm(0)
36471 .addReg(IReg)
36472 .addImm(X86::sub_32bit);
36473
36474 switch (JTE) {
36475 case MachineJumpTableInfo::EK_BlockAddress:
36476 // jmpq *(BReg,IReg64,8)
36477 BuildMI(DispContBB, MIMD, TII->get(X86::JMP64m))
36478 .addReg(BReg)
36479 .addImm(8)
36480 .addReg(IReg64)
36481 .addImm(0)
36482 .addReg(0);
36483 break;
36484 case MachineJumpTableInfo::EK_LabelDifference32: {
36485 Register OReg = MRI->createVirtualRegister(&X86::GR32RegClass);
36486 Register OReg64 = MRI->createVirtualRegister(&X86::GR64RegClass);
36487 Register TReg = MRI->createVirtualRegister(&X86::GR64RegClass);
36488
36489 // movl (BReg,IReg64,4), OReg
36490 BuildMI(DispContBB, MIMD, TII->get(X86::MOV32rm), OReg)
36491 .addReg(BReg)
36492 .addImm(4)
36493 .addReg(IReg64)
36494 .addImm(0)
36495 .addReg(0);
36496 // movsx OReg64, OReg
36497 BuildMI(DispContBB, MIMD, TII->get(X86::MOVSX64rr32), OReg64)
36498 .addReg(OReg);
36499 // addq BReg, OReg64, TReg
36500 BuildMI(DispContBB, MIMD, TII->get(X86::ADD64rr), TReg)
36501 .addReg(OReg64)
36502 .addReg(BReg);
36503 // jmpq *TReg
36504 BuildMI(DispContBB, MIMD, TII->get(X86::JMP64r)).addReg(TReg);
36505 break;
36506 }
36507 default:
36508 llvm_unreachable("Unexpected jump table encoding");
36509 }
36510 } else {
36511 // jmpl *.LJTI0_0(,IReg,4)
36512 BuildMI(DispContBB, MIMD, TII->get(X86::JMP32m))
36513 .addReg(0)
36514 .addImm(4)
36515 .addReg(IReg)
36516 .addJumpTableIndex(MJTI)
36517 .addReg(0);
36518 }
36519
36520 // Add the jump table entries as successors to the MBB.
36521 SmallPtrSet<MachineBasicBlock *, 8> SeenMBBs;
36522 for (auto &LP : LPadList)
36523 if (SeenMBBs.insert(LP).second)
36524 DispContBB->addSuccessor(LP);
36525
36526 // N.B. the order the invoke BBs are processed in doesn't matter here.
36527 SmallVector<MachineBasicBlock *, 64> MBBLPads;
36528 const MCPhysReg *SavedRegs = MF->getRegInfo().getCalleeSavedRegs();
36529 for (MachineBasicBlock *MBB : InvokeBBs) {
36530 // Remove the landing pad successor from the invoke block and replace it
36531 // with the new dispatch block.
36532 // Keep a copy of Successors since it's modified inside the loop.
36533 SmallVector<MachineBasicBlock *, 8> Successors(MBB->succ_rbegin(),
36534 MBB->succ_rend());
36535 // FIXME: Avoid quadratic complexity.
36536 for (auto *MBBS : Successors) {
36537 if (MBBS->isEHPad()) {
36538 MBB->removeSuccessor(MBBS);
36539 MBBLPads.push_back(MBBS);
36540 }
36541 }
36542
36543 MBB->addSuccessor(DispatchBB);
36544
36545 // Find the invoke call and mark all of the callee-saved registers as
36546 // 'implicit defined' so that they're spilled. This prevents code from
36547 // moving instructions to before the EH block, where they will never be
36548 // executed.
36549 for (auto &II : reverse(*MBB)) {
36550 if (!II.isCall())
36551 continue;
36552
36553 DenseMap<unsigned, bool> DefRegs;
36554 for (auto &MOp : II.operands())
36555 if (MOp.isReg())
36556 DefRegs[MOp.getReg()] = true;
36557
36558 MachineInstrBuilder MIB(*MF, &II);
36559 for (unsigned RegIdx = 0; SavedRegs[RegIdx]; ++RegIdx) {
36560 unsigned Reg = SavedRegs[RegIdx];
36561 if (!DefRegs[Reg])
36562 MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead);
36563 }
36564
36565 break;
36566 }
36567 }
36568
36569 // Mark all former landing pads as non-landing pads. The dispatch is the only
36570 // landing pad now.
36571 for (auto &LP : MBBLPads)
36572 LP->setIsEHPad(false);
36573
36574 // The instruction is gone now.
36575 MI.eraseFromParent();
36576 return BB;
36577}
36578
36579MachineBasicBlock *
36580X86TargetLowering::emitPatchableEventCall(MachineInstr &MI,
36581 MachineBasicBlock *BB) const {
36582 // Wrap patchable event calls in CALLSEQ_START/CALLSEQ_END, as tracing
36583 // calls may require proper stack alignment.
36584 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
36585 const MIMetadata MIMD(MI);
36586 MachineFunction &MF = *BB->getParent();
36587
36588 // Emit CALLSEQ_START right before the instruction.
36589 MF.getFrameInfo().setAdjustsStack(true);
36590 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
36591 MachineInstrBuilder CallseqStart =
36592 BuildMI(MF, MIMD, TII.get(AdjStackDown)).addImm(0).addImm(0).addImm(0);
36593 BB->insert(MachineBasicBlock::iterator(MI), CallseqStart);
36594
36595 // Emit CALLSEQ_END right after the instruction.
36596 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
36597 MachineInstrBuilder CallseqEnd =
36598 BuildMI(MF, MIMD, TII.get(AdjStackUp)).addImm(0).addImm(0);
36599 BB->insertAfter(MachineBasicBlock::iterator(MI), CallseqEnd);
36600
36601 return BB;
36602}
36603
36604MachineBasicBlock *
36605X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
36606 MachineBasicBlock *BB) const {
36607 MachineFunction *MF = BB->getParent();
36608 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
36609 const MIMetadata MIMD(MI);
36610
36611 auto TMMImmToTMMReg = [](unsigned Imm) {
36612 assert (Imm < 8 && "Illegal tmm index");
36613 return X86::TMM0 + Imm;
36614 };
36615 switch (MI.getOpcode()) {
36616 default: llvm_unreachable("Unexpected instr type to insert");
36617 case X86::TLS_addr32:
36618 case X86::TLS_addr64:
36619 case X86::TLS_addrX32:
36620 case X86::TLS_base_addr32:
36621 case X86::TLS_base_addr64:
36622 case X86::TLS_base_addrX32:
36623 case X86::TLS_desc32:
36624 case X86::TLS_desc64:
36625 return EmitLoweredTLSAddr(MI, BB);
36626 case X86::INDIRECT_THUNK_CALL32:
36627 case X86::INDIRECT_THUNK_CALL64:
36628 case X86::INDIRECT_THUNK_TCRETURN32:
36629 case X86::INDIRECT_THUNK_TCRETURN64:
36630 return EmitLoweredIndirectThunk(MI, BB);
36631 case X86::CATCHRET:
36632 return EmitLoweredCatchRet(MI, BB);
36633 case X86::SEG_ALLOCA_32:
36634 case X86::SEG_ALLOCA_64:
36635 return EmitLoweredSegAlloca(MI, BB);
36636 case X86::PROBED_ALLOCA_32:
36637 case X86::PROBED_ALLOCA_64:
36638 return EmitLoweredProbedAlloca(MI, BB);
36639 case X86::TLSCall_32:
36640 case X86::TLSCall_64:
36641 return EmitLoweredTLSCall(MI, BB);
36642 case X86::CMOV_FR16:
36643 case X86::CMOV_FR16X:
36644 case X86::CMOV_FR32:
36645 case X86::CMOV_FR32X:
36646 case X86::CMOV_FR64:
36647 case X86::CMOV_FR64X:
36648 case X86::CMOV_GR8:
36649 case X86::CMOV_GR16:
36650 case X86::CMOV_GR32:
36651 case X86::CMOV_RFP32:
36652 case X86::CMOV_RFP64:
36653 case X86::CMOV_RFP80:
36654 case X86::CMOV_VR64:
36655 case X86::CMOV_VR128:
36656 case X86::CMOV_VR128X:
36657 case X86::CMOV_VR256:
36658 case X86::CMOV_VR256X:
36659 case X86::CMOV_VR512:
36660 case X86::CMOV_VK1:
36661 case X86::CMOV_VK2:
36662 case X86::CMOV_VK4:
36663 case X86::CMOV_VK8:
36664 case X86::CMOV_VK16:
36665 case X86::CMOV_VK32:
36666 case X86::CMOV_VK64:
36667 return EmitLoweredSelect(MI, BB);
36668
36669 case X86::FP80_ADDr:
36670 case X86::FP80_ADDm32: {
36671 // Change the floating point control register to use double extended
36672 // precision when performing the addition.
36673 int OrigCWFrameIdx =
36674 MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36675 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)),
36676 OrigCWFrameIdx);
36677
36678 // Load the old value of the control word...
36679 Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36680 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW),
36681 OrigCWFrameIdx);
36682
36683 // OR 0b11 into bit 8 and 9. 0b11 is the encoding for double extended
36684 // precision.
36685 Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36686 BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
36687 .addReg(OldCW, RegState::Kill)
36688 .addImm(0x300);
36689
36690 // Extract to 16 bits.
36691 Register NewCW16 =
36692 MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
36693 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16)
36694 .addReg(NewCW, RegState::Kill, X86::sub_16bit);
36695
36696 // Prepare memory for FLDCW.
36697 int NewCWFrameIdx =
36698 MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36699 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
36700 NewCWFrameIdx)
36701 .addReg(NewCW16, RegState::Kill);
36702
36703 // Reload the modified control word now...
36704 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
36705 NewCWFrameIdx);
36706
36707 // Do the addition.
36708 if (MI.getOpcode() == X86::FP80_ADDr) {
36709 BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80))
36710 .add(MI.getOperand(0))
36711 .add(MI.getOperand(1))
36712 .add(MI.getOperand(2));
36713 } else {
36714 BuildMI(*BB, MI, MIMD, TII->get(X86::ADD_Fp80m32))
36715 .add(MI.getOperand(0))
36716 .add(MI.getOperand(1))
36717 .add(MI.getOperand(2))
36718 .add(MI.getOperand(3))
36719 .add(MI.getOperand(4))
36720 .add(MI.getOperand(5))
36721 .add(MI.getOperand(6));
36722 }
36723
36724 // Reload the original control word now.
36725 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
36726 OrigCWFrameIdx);
36727
36728 MI.eraseFromParent(); // The pseudo instruction is gone now.
36729 return BB;
36730 }
36731
36732 case X86::FP32_TO_INT16_IN_MEM:
36733 case X86::FP32_TO_INT32_IN_MEM:
36734 case X86::FP32_TO_INT64_IN_MEM:
36735 case X86::FP64_TO_INT16_IN_MEM:
36736 case X86::FP64_TO_INT32_IN_MEM:
36737 case X86::FP64_TO_INT64_IN_MEM:
36738 case X86::FP80_TO_INT16_IN_MEM:
36739 case X86::FP80_TO_INT32_IN_MEM:
36740 case X86::FP80_TO_INT64_IN_MEM: {
36741 // Change the floating point control register to use "round towards zero"
36742 // mode when truncating to an integer value.
36743 int OrigCWFrameIdx =
36744 MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36745 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FNSTCW16m)),
36746 OrigCWFrameIdx);
36747
36748 // Load the old value of the control word...
36749 Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36750 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOVZX32rm16), OldCW),
36751 OrigCWFrameIdx);
36752
36753 // OR 0b11 into bit 10 and 11. 0b11 is the encoding for round toward zero.
36754 Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
36755 BuildMI(*BB, MI, MIMD, TII->get(X86::OR32ri), NewCW)
36756 .addReg(OldCW, RegState::Kill).addImm(0xC00);
36757
36758 // Extract to 16 bits.
36759 Register NewCW16 =
36760 MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
36761 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), NewCW16)
36762 .addReg(NewCW, RegState::Kill, X86::sub_16bit);
36763
36764 // Prepare memory for FLDCW.
36765 int NewCWFrameIdx =
36766 MF->getFrameInfo().CreateStackObject(2, Align(2), false);
36767 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::MOV16mr)),
36768 NewCWFrameIdx)
36769 .addReg(NewCW16, RegState::Kill);
36770
36771 // Reload the modified control word now...
36772 addFrameReference(BuildMI(*BB, MI, MIMD,
36773 TII->get(X86::FLDCW16m)), NewCWFrameIdx);
36774
36775 // Get the X86 opcode to use.
36776 unsigned Opc;
36777 switch (MI.getOpcode()) {
36778 // clang-format off
36779 default: llvm_unreachable("illegal opcode!");
36780 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
36781 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
36782 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break;
36783 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break;
36784 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break;
36785 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break;
36786 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
36787 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
36788 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
36789 // clang-format on
36790 }
36791
36792 X86AddressMode AM = getAddressFromInstr(&MI, 0);
36793 addFullAddress(BuildMI(*BB, MI, MIMD, TII->get(Opc)), AM)
36794 .addReg(MI.getOperand(X86::AddrNumOperands).getReg());
36795
36796 // Reload the original control word now.
36797 addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::FLDCW16m)),
36798 OrigCWFrameIdx);
36799
36800 MI.eraseFromParent(); // The pseudo instruction is gone now.
36801 return BB;
36802 }
36803
36804 // xbegin
36805 case X86::XBEGIN:
36806 return emitXBegin(MI, BB, Subtarget.getInstrInfo());
36807
36808 case X86::VAARG_64:
36809 case X86::VAARG_X32:
36810 return EmitVAARGWithCustomInserter(MI, BB);
36811
36812 case X86::EH_SjLj_SetJmp32:
36813 case X86::EH_SjLj_SetJmp64:
36814 return emitEHSjLjSetJmp(MI, BB);
36815
36816 case X86::EH_SjLj_LongJmp32:
36817 case X86::EH_SjLj_LongJmp64:
36818 return emitEHSjLjLongJmp(MI, BB);
36819
36820 case X86::Int_eh_sjlj_setup_dispatch:
36821 return EmitSjLjDispatchBlock(MI, BB);
36822
36823 case TargetOpcode::STATEPOINT:
36824 // As an implementation detail, STATEPOINT shares the STACKMAP format at
36825 // this point in the process. We diverge later.
36826 return emitPatchPoint(MI, BB);
36827
36828 case TargetOpcode::STACKMAP:
36829 case TargetOpcode::PATCHPOINT:
36830 return emitPatchPoint(MI, BB);
36831
36832 case TargetOpcode::PATCHABLE_EVENT_CALL:
36833 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
36834 return emitPatchableEventCall(MI, BB);
36835
36836 case X86::LCMPXCHG8B: {
36837 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
36838 // In addition to 4 E[ABCD] registers implied by encoding, CMPXCHG8B
36839 // requires a memory operand. If it happens that current architecture is
36840 // i686 and for current function we need a base pointer
36841 // - which is ESI for i686 - register allocator would not be able to
36842 // allocate registers for an address in form of X(%reg, %reg, Y)
36843 // - there never would be enough unreserved registers during regalloc
36844 // (without the need for base ptr the only option would be X(%edi, %esi, Y).
36845 // We are giving a hand to register allocator by precomputing the address in
36846 // a new vreg using LEA.
36847
36848 // If it is not i686 or there is no base pointer - nothing to do here.
36849 if (!Subtarget.is32Bit() || !TRI->hasBasePointer(*MF))
36850 return BB;
36851
36852 // Even though this code does not necessarily needs the base pointer to
36853 // be ESI, we check for that. The reason: if this assert fails, there are
36854 // some changes happened in the compiler base pointer handling, which most
36855 // probably have to be addressed somehow here.
36856 assert(TRI->getBaseRegister() == X86::ESI &&
36857 "LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a "
36858 "base pointer in mind");
36859
36860 MachineRegisterInfo &MRI = MF->getRegInfo();
36861 MVT SPTy = getPointerTy(MF->getDataLayout());
36862 const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
36863 Register computedAddrVReg = MRI.createVirtualRegister(AddrRegClass);
36864
36865 X86AddressMode AM = getAddressFromInstr(&MI, 0);
36866 // Regalloc does not need any help when the memory operand of CMPXCHG8B
36867 // does not use index register.
36868 if (AM.IndexReg == X86::NoRegister)
36869 return BB;
36870
36871 // After X86TargetLowering::ReplaceNodeResults CMPXCHG8B is glued to its
36872 // four operand definitions that are E[ABCD] registers. We skip them and
36873 // then insert the LEA.
36874 MachineBasicBlock::reverse_iterator RMBBI(MI.getReverseIterator());
36875 while (RMBBI != BB->rend() &&
36876 (RMBBI->definesRegister(X86::EAX, /*TRI=*/nullptr) ||
36877 RMBBI->definesRegister(X86::EBX, /*TRI=*/nullptr) ||
36878 RMBBI->definesRegister(X86::ECX, /*TRI=*/nullptr) ||
36879 RMBBI->definesRegister(X86::EDX, /*TRI=*/nullptr))) {
36880 ++RMBBI;
36881 }
36882 MachineBasicBlock::iterator MBBI(RMBBI);
36883 addFullAddress(
36884 BuildMI(*BB, *MBBI, MIMD, TII->get(X86::LEA32r), computedAddrVReg), AM);
36885
36886 setDirectAddressInInstr(&MI, 0, computedAddrVReg);
36887
36888 return BB;
36889 }
36890 case X86::LCMPXCHG16B_NO_RBX: {
36891 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
36892 Register BasePtr = TRI->getBaseRegister();
36893 if (TRI->hasBasePointer(*MF) &&
36894 (BasePtr == X86::RBX || BasePtr == X86::EBX)) {
36895 if (!BB->isLiveIn(BasePtr))
36896 BB->addLiveIn(BasePtr);
36897 // Save RBX into a virtual register.
36898 Register SaveRBX =
36899 MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36900 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), SaveRBX)
36901 .addReg(X86::RBX);
36902 Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36903 MachineInstrBuilder MIB =
36904 BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B_SAVE_RBX), Dst);
36905 for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx)
36906 MIB.add(MI.getOperand(Idx));
36907 MIB.add(MI.getOperand(X86::AddrNumOperands));
36908 MIB.addReg(SaveRBX);
36909 } else {
36910 // Simple case, just copy the virtual register to RBX.
36911 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::RBX)
36912 .add(MI.getOperand(X86::AddrNumOperands));
36913 MachineInstrBuilder MIB =
36914 BuildMI(*BB, MI, MIMD, TII->get(X86::LCMPXCHG16B));
36915 for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx)
36916 MIB.add(MI.getOperand(Idx));
36917 }
36918 MI.eraseFromParent();
36919 return BB;
36920 }
36921 case X86::MWAITX: {
36922 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
36923 Register BasePtr = TRI->getBaseRegister();
36924 bool IsRBX = (BasePtr == X86::RBX || BasePtr == X86::EBX);
36925 // If no need to save the base pointer, we generate MWAITXrrr,
36926 // else we generate pseudo MWAITX_SAVE_RBX.
36927 if (!IsRBX || !TRI->hasBasePointer(*MF)) {
36928 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX)
36929 .addReg(MI.getOperand(0).getReg());
36930 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX)
36931 .addReg(MI.getOperand(1).getReg());
36932 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EBX)
36933 .addReg(MI.getOperand(2).getReg());
36934 BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITXrrr));
36935 MI.eraseFromParent();
36936 } else {
36937 if (!BB->isLiveIn(BasePtr)) {
36938 BB->addLiveIn(BasePtr);
36939 }
36940 // Parameters can be copied into ECX and EAX but not EBX yet.
36941 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::ECX)
36942 .addReg(MI.getOperand(0).getReg());
36943 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), X86::EAX)
36944 .addReg(MI.getOperand(1).getReg());
36945 assert(Subtarget.is64Bit() && "Expected 64-bit mode!");
36946 // Save RBX into a virtual register.
36947 Register SaveRBX =
36948 MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36949 BuildMI(*BB, MI, MIMD, TII->get(TargetOpcode::COPY), SaveRBX)
36950 .addReg(X86::RBX);
36951 // Generate mwaitx pseudo.
36952 Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass);
36953 BuildMI(*BB, MI, MIMD, TII->get(X86::MWAITX_SAVE_RBX))
36954 .addDef(Dst) // Destination tied in with SaveRBX.
36955 .addReg(MI.getOperand(2).getReg()) // input value of EBX.
36956 .addUse(SaveRBX); // Save of base pointer.
36957 MI.eraseFromParent();
36958 }
36959 return BB;
36960 }
36961 case TargetOpcode::PREALLOCATED_SETUP: {
36962 assert(Subtarget.is32Bit() && "preallocated only used in 32-bit");
36963 auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
36964 MFI->setHasPreallocatedCall(true);
36965 int64_t PreallocatedId = MI.getOperand(0).getImm();
36966 size_t StackAdjustment = MFI->getPreallocatedStackSize(PreallocatedId);
36967 assert(StackAdjustment != 0 && "0 stack adjustment");
36968 LLVM_DEBUG(dbgs() << "PREALLOCATED_SETUP stack adjustment "
36969 << StackAdjustment << "\n");
36970 BuildMI(*BB, MI, MIMD, TII->get(X86::SUB32ri), X86::ESP)
36971 .addReg(X86::ESP)
36972 .addImm(StackAdjustment);
36973 MI.eraseFromParent();
36974 return BB;
36975 }
36976 case TargetOpcode::PREALLOCATED_ARG: {
36977 assert(Subtarget.is32Bit() && "preallocated calls only used in 32-bit");
36978 int64_t PreallocatedId = MI.getOperand(1).getImm();
36979 int64_t ArgIdx = MI.getOperand(2).getImm();
36980 auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
36981 size_t ArgOffset = MFI->getPreallocatedArgOffsets(PreallocatedId)[ArgIdx];
36982 LLVM_DEBUG(dbgs() << "PREALLOCATED_ARG arg index " << ArgIdx
36983 << ", arg offset " << ArgOffset << "\n");
36984 // stack pointer + offset
36985 addRegOffset(BuildMI(*BB, MI, MIMD, TII->get(X86::LEA32r),
36986 MI.getOperand(0).getReg()),
36987 X86::ESP, false, ArgOffset);
36988 MI.eraseFromParent();
36989 return BB;
36990 }
36991 case X86::PTDPBSSD:
36992 case X86::PTDPBSUD:
36993 case X86::PTDPBUSD:
36994 case X86::PTDPBUUD:
36995 case X86::PTDPBF16PS:
36996 case X86::PTDPFP16PS: {
36997 unsigned Opc;
36998 switch (MI.getOpcode()) {
36999 // clang-format off
37000 default: llvm_unreachable("illegal opcode!");
37001 case X86::PTDPBSSD: Opc = X86::TDPBSSD; break;
37002 case X86::PTDPBSUD: Opc = X86::TDPBSUD; break;
37003 case X86::PTDPBUSD: Opc = X86::TDPBUSD; break;
37004 case X86::PTDPBUUD: Opc = X86::TDPBUUD; break;
37005 case X86::PTDPBF16PS: Opc = X86::TDPBF16PS; break;
37006 case X86::PTDPFP16PS: Opc = X86::TDPFP16PS; break;
37007 // clang-format on
37008 }
37009
37010 MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc));
37011 MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Define);
37012 MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Undef);
37013 MIB.addReg(TMMImmToTMMReg(MI.getOperand(1).getImm()), RegState::Undef);
37014 MIB.addReg(TMMImmToTMMReg(MI.getOperand(2).getImm()), RegState::Undef);
37015
37016 MI.eraseFromParent(); // The pseudo is gone now.
37017 return BB;
37018 }
37019 case X86::PTILEZERO: {
37020 unsigned Imm = MI.getOperand(0).getImm();
37021 BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
37022 MI.eraseFromParent(); // The pseudo is gone now.
37023 auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
37024 MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
37025 return BB;
37026 }
37027 case X86::PTILEZEROV: {
37028 auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
37029 MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
37030 return BB;
37031 }
37032 case X86::PTILELOADD:
37033 case X86::PTILELOADDT1:
37034 case X86::PTILESTORED: {
37035 unsigned Opc;
37036 switch (MI.getOpcode()) {
37037 default: llvm_unreachable("illegal opcode!");
37038#define GET_EGPR_IF_ENABLED(OPC) (Subtarget.hasEGPR() ? OPC##_EVEX : OPC)
37039 case X86::PTILELOADD:
37040 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
37041 break;
37042 case X86::PTILELOADDT1:
37043 Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDT1);
37044 break;
37045 case X86::PTILESTORED:
37046 Opc = GET_EGPR_IF_ENABLED(X86::TILESTORED);
37047 break;
37048#undef GET_EGPR_IF_ENABLED
37049 }
37050
37051 MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc));
37052 unsigned CurOp = 0;
37053 if (Opc != X86::TILESTORED && Opc != X86::TILESTORED_EVEX)
37054 MIB.addReg(TMMImmToTMMReg(MI.getOperand(CurOp++).getImm()),
37055 RegState::Define);
37056
37057 MIB.add(MI.getOperand(CurOp++)); // base
37058 MIB.add(MI.getOperand(CurOp++)); // scale
37059 MIB.add(MI.getOperand(CurOp++)); // index -- stride
37060 MIB.add(MI.getOperand(CurOp++)); // displacement
37061 MIB.add(MI.getOperand(CurOp++)); // segment
37062
37063 if (Opc == X86::TILESTORED || Opc == X86::TILESTORED_EVEX)
37064 MIB.addReg(TMMImmToTMMReg(MI.getOperand(CurOp++).getImm()),
37065 RegState::Undef);
37066
37067 MI.eraseFromParent(); // The pseudo is gone now.
37068 return BB;
37069 }
37070 case X86::PTCMMIMFP16PS:
37071 case X86::PTCMMRLFP16PS: {
37072 const MIMetadata MIMD(MI);
37073 unsigned Opc;
37074 switch (MI.getOpcode()) {
37075 // clang-format off
37076 default: llvm_unreachable("Unexpected instruction!");
37077 case X86::PTCMMIMFP16PS: Opc = X86::TCMMIMFP16PS; break;
37078 case X86::PTCMMRLFP16PS: Opc = X86::TCMMRLFP16PS; break;
37079 // clang-format on
37080 }
37081 MachineInstrBuilder MIB = BuildMI(*BB, MI, MIMD, TII->get(Opc));
37082 MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Define);
37083 MIB.addReg(TMMImmToTMMReg(MI.getOperand(0).getImm()), RegState::Undef);
37084 MIB.addReg(TMMImmToTMMReg(MI.getOperand(1).getImm()), RegState::Undef);
37085 MIB.addReg(TMMImmToTMMReg(MI.getOperand(2).getImm()), RegState::Undef);
37086 MI.eraseFromParent(); // The pseudo is gone now.
37087 return BB;
37088 }
37089 }
37090}
37091
37092//===----------------------------------------------------------------------===//
37093// X86 Optimization Hooks
37094//===----------------------------------------------------------------------===//
37095
37096bool
37097X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
37098 const APInt &DemandedBits,
37099 const APInt &DemandedElts,
37100 TargetLoweringOpt &TLO) const {
37101 EVT VT = Op.getValueType();
37102 unsigned Opcode = Op.getOpcode();
37103 unsigned EltSize = VT.getScalarSizeInBits();
37104
37105 if (VT.isVector()) {
37106 // If the constant is only all signbits in the active bits, then we should
37107 // extend it to the entire constant to allow it act as a boolean constant
37108 // vector.
37109 auto NeedsSignExtension = [&](SDValue V, unsigned ActiveBits) {
37110 if (!ISD::isBuildVectorOfConstantSDNodes(V.getNode()))
37111 return false;
37112 for (unsigned i = 0, e = V.getNumOperands(); i != e; ++i) {
37113 if (!DemandedElts[i] || V.getOperand(i).isUndef())
37114 continue;
37115 const APInt &Val = V.getConstantOperandAPInt(i);
37116 if (Val.getBitWidth() > Val.getNumSignBits() &&
37117 Val.trunc(ActiveBits).getNumSignBits() == ActiveBits)
37118 return true;
37119 }
37120 return false;
37121 };
37122 // For vectors - if we have a constant, then try to sign extend.
37123 // TODO: Handle AND cases.
37124 unsigned ActiveBits = DemandedBits.getActiveBits();
37125 if (EltSize > ActiveBits && EltSize > 1 && isTypeLegal(VT) &&
37126 (Opcode == ISD::OR || Opcode == ISD::XOR || Opcode == X86ISD::ANDNP) &&
37127 NeedsSignExtension(Op.getOperand(1), ActiveBits)) {
37128 EVT ExtSVT = EVT::getIntegerVT(*TLO.DAG.getContext(), ActiveBits);
37129 EVT ExtVT = EVT::getVectorVT(*TLO.DAG.getContext(), ExtSVT,
37130 VT.getVectorNumElements());
37131 SDValue NewC =
37132 TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(Op), VT,
37133 Op.getOperand(1), TLO.DAG.getValueType(ExtVT));
37134 SDValue NewOp =
37135 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, Op.getOperand(0), NewC);
37136 return TLO.CombineTo(Op, NewOp);
37137 }
37138 return false;
37139 }
37140
37141 // Only optimize Ands to prevent shrinking a constant that could be
37142 // matched by movzx.
37143 if (Opcode != ISD::AND)
37144 return false;
37145
37146 // Make sure the RHS really is a constant.
37147 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
37148 if (!C)
37149 return false;
37150
37151 const APInt &Mask = C->getAPIntValue();
37152
37153 // Clear all non-demanded bits initially.
37154 APInt ShrunkMask = Mask & DemandedBits;
37155
37156 // Find the width of the shrunk mask.
37157 unsigned Width = ShrunkMask.getActiveBits();
37158
37159 // If the mask is all 0s there's nothing to do here.
37160 if (Width == 0)
37161 return false;
37162
37163 // Find the next power of 2 width, rounding up to a byte.
37164 Width = llvm::bit_ceil(std::max(Width, 8U));
37165 // Truncate the width to size to handle illegal types.
37166 Width = std::min(Width, EltSize);
37167
37168 // Calculate a possible zero extend mask for this constant.
37169 APInt ZeroExtendMask = APInt::getLowBitsSet(EltSize, Width);
37170
37171 // If we aren't changing the mask, just return true to keep it and prevent
37172 // the caller from optimizing.
37173 if (ZeroExtendMask == Mask)
37174 return true;
37175
37176 // Make sure the new mask can be represented by a combination of mask bits
37177 // and non-demanded bits.
37178 if (!ZeroExtendMask.isSubsetOf(Mask | ~DemandedBits))
37179 return false;
37180
37181 // Replace the constant with the zero extend mask.
37182 SDLoc DL(Op);
37183 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);
37184 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
37185 return TLO.CombineTo(Op, NewOp);
37186}
37187
37188static void computeKnownBitsForPSADBW(SDValue LHS, SDValue RHS,
37189 KnownBits &Known,
37190 const APInt &DemandedElts,
37191 const SelectionDAG &DAG, unsigned Depth) {
37192 KnownBits Known2;
37193 unsigned NumSrcElts = LHS.getValueType().getVectorNumElements();
37194 APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
37195 Known = DAG.computeKnownBits(RHS, DemandedSrcElts, Depth + 1);
37196 Known2 = DAG.computeKnownBits(LHS, DemandedSrcElts, Depth + 1);
37197 Known = KnownBits::abdu(Known, Known2).zext(16);
37198 // Known = (((D0 + D1) + (D2 + D3)) + ((D4 + D5) + (D6 + D7)))
37199 Known = KnownBits::add(Known, Known, /*NSW=*/true, /*NUW=*/true);
37200 Known = KnownBits::add(Known, Known, /*NSW=*/true, /*NUW=*/true);
37201 Known = KnownBits::add(Known, Known, /*NSW=*/true, /*NUW=*/true);
37202 Known = Known.zext(64);
37203}
37204
37205static void computeKnownBitsForPMADDWD(SDValue LHS, SDValue RHS,
37206 KnownBits &Known,
37207 const APInt &DemandedElts,
37208 const SelectionDAG &DAG,
37209 unsigned Depth) {
37210 unsigned NumSrcElts = LHS.getValueType().getVectorNumElements();
37211
37212 // Multiply signed i16 elements to create i32 values and add Lo/Hi pairs.
37213 APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
37214 APInt DemandedLoElts =
37215 DemandedSrcElts & APInt::getSplat(NumSrcElts, APInt(2, 0b01));
37216 APInt DemandedHiElts =
37217 DemandedSrcElts & APInt::getSplat(NumSrcElts, APInt(2, 0b10));
37218 KnownBits LHSLo = DAG.computeKnownBits(LHS, DemandedLoElts, Depth + 1);
37219 KnownBits LHSHi = DAG.computeKnownBits(LHS, DemandedHiElts, Depth + 1);
37220 KnownBits RHSLo = DAG.computeKnownBits(RHS, DemandedLoElts, Depth + 1);
37221 KnownBits RHSHi = DAG.computeKnownBits(RHS, DemandedHiElts, Depth + 1);
37222 KnownBits Lo = KnownBits::mul(LHSLo.sext(32), RHSLo.sext(32));
37223 KnownBits Hi = KnownBits::mul(LHSHi.sext(32), RHSHi.sext(32));
37224 Known = KnownBits::add(Lo, Hi, /*NSW=*/false, /*NUW=*/false);
37225}
37226
37227static void computeKnownBitsForPMADDUBSW(SDValue LHS, SDValue RHS,
37228 KnownBits &Known,
37229 const APInt &DemandedElts,
37230 const SelectionDAG &DAG,
37231 unsigned Depth) {
37232 unsigned NumSrcElts = LHS.getValueType().getVectorNumElements();
37233
37234 // Multiply unsigned/signed i8 elements to create i16 values and add_sat Lo/Hi
37235 // pairs.
37236 APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
37237 APInt DemandedLoElts =
37238 DemandedSrcElts & APInt::getSplat(NumSrcElts, APInt(2, 0b01));
37239 APInt DemandedHiElts =
37240 DemandedSrcElts & APInt::getSplat(NumSrcElts, APInt(2, 0b10));
37241 KnownBits LHSLo = DAG.computeKnownBits(LHS, DemandedLoElts, Depth + 1);
37242 KnownBits LHSHi = DAG.computeKnownBits(LHS, DemandedHiElts, Depth + 1);
37243 KnownBits RHSLo = DAG.computeKnownBits(RHS, DemandedLoElts, Depth + 1);
37244 KnownBits RHSHi = DAG.computeKnownBits(RHS, DemandedHiElts, Depth + 1);
37245 KnownBits Lo = KnownBits::mul(LHSLo.zext(16), RHSLo.sext(16));
37246 KnownBits Hi = KnownBits::mul(LHSHi.zext(16), RHSHi.sext(16));
37247 Known = KnownBits::sadd_sat(Lo, Hi);
37248}
37249
37250static KnownBits computeKnownBitsForHorizontalOperation(
37251 const SDValue Op, const APInt &DemandedElts, unsigned Depth,
37252 const SelectionDAG &DAG,
37253 const function_ref<KnownBits(const KnownBits &, const KnownBits &)>
37254 KnownBitsFunc) {
37255 APInt DemandedEltsLHS, DemandedEltsRHS;
37256 getHorizDemandedEltsForFirstOperand(Op.getValueType().getSizeInBits(),
37257 DemandedElts, DemandedEltsLHS,
37258 DemandedEltsRHS);
37259
37260 const auto ComputeForSingleOpFunc =
37261 [&DAG, Depth, KnownBitsFunc](SDValue Op, APInt &DemandedEltsOp) {
37262 return KnownBitsFunc(
37263 DAG.computeKnownBits(Op, DemandedEltsOp, Depth + 1),
37264 DAG.computeKnownBits(Op, DemandedEltsOp << 1, Depth + 1));
37265 };
37266
37267 if (DemandedEltsRHS.isZero())
37268 return ComputeForSingleOpFunc(Op.getOperand(0), DemandedEltsLHS);
37269 if (DemandedEltsLHS.isZero())
37270 return ComputeForSingleOpFunc(Op.getOperand(1), DemandedEltsRHS);
37271
37272 return ComputeForSingleOpFunc(Op.getOperand(0), DemandedEltsLHS)
37273 .intersectWith(ComputeForSingleOpFunc(Op.getOperand(1), DemandedEltsRHS));
37274}
37275
37276void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
37277 KnownBits &Known,
37278 const APInt &DemandedElts,
37279 const SelectionDAG &DAG,
37280 unsigned Depth) const {
37281 unsigned BitWidth = Known.getBitWidth();
37282 unsigned NumElts = DemandedElts.getBitWidth();
37283 unsigned Opc = Op.getOpcode();
37284 EVT VT = Op.getValueType();
37285 assert((Opc >= ISD::BUILTIN_OP_END ||
37286 Opc == ISD::INTRINSIC_WO_CHAIN ||
37287 Opc == ISD::INTRINSIC_W_CHAIN ||
37288 Opc == ISD::INTRINSIC_VOID) &&
37289 "Should use MaskedValueIsZero if you don't know whether Op"
37290 " is a target node!");
37291
37292 Known.resetAll();
37293 switch (Opc) {
37294 default: break;
37295 case X86ISD::MUL_IMM: {
37296 KnownBits Known2;
37297 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
37298 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
37299 Known = KnownBits::mul(Known, Known2);
37300 break;
37301 }
37302 case X86ISD::BSR:
37303 // BSR(0) is undef, but any use of BSR already accounts for non-zero inputs.
37304 // Similar KnownBits behaviour to CTLZ_ZERO_UNDEF.
37305 // TODO: Bound with input known bits?
37306 Known.Zero.setBitsFrom(Log2_32(BitWidth));
37307 break;
37308 case X86ISD::SETCC:
37309 Known.Zero.setBitsFrom(1);
37310 break;
37311 case X86ISD::MOVMSK: {
37312 unsigned NumLoBits = Op.getOperand(0).getValueType().getVectorNumElements();
37313 Known.Zero.setBitsFrom(NumLoBits);
37314 break;
37315 }
37316 case X86ISD::PEXTRB:
37317 case X86ISD::PEXTRW: {
37318 SDValue Src = Op.getOperand(0);
37319 EVT SrcVT = Src.getValueType();
37320 APInt DemandedElt = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
37321 Op.getConstantOperandVal(1));
37322 Known = DAG.computeKnownBits(Src, DemandedElt, Depth + 1);
37323 Known = Known.anyextOrTrunc(BitWidth);
37324 Known.Zero.setBitsFrom(SrcVT.getScalarSizeInBits());
37325 break;
37326 }
37327 case X86ISD::VSRAI:
37328 case X86ISD::VSHLI:
37329 case X86ISD::VSRLI: {
37330 unsigned ShAmt = Op.getConstantOperandVal(1);
37331 if (ShAmt >= VT.getScalarSizeInBits()) {
37332 // Out of range logical bit shifts are guaranteed to be zero.
37333 // Out of range arithmetic bit shifts splat the sign bit.
37334 if (Opc != X86ISD::VSRAI) {
37335 Known.setAllZero();
37336 break;
37337 }
37338
37339 ShAmt = VT.getScalarSizeInBits() - 1;
37340 }
37341
37342 Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
37343 if (Opc == X86ISD::VSHLI) {
37344 Known.Zero <<= ShAmt;
37345 Known.One <<= ShAmt;
37346 // Low bits are known zero.
37347 Known.Zero.setLowBits(ShAmt);
37348 } else if (Opc == X86ISD::VSRLI) {
37349 Known.Zero.lshrInPlace(ShAmt);
37350 Known.One.lshrInPlace(ShAmt);
37351 // High bits are known zero.
37352 Known.Zero.setHighBits(ShAmt);
37353 } else {
37354 Known.Zero.ashrInPlace(ShAmt);
37355 Known.One.ashrInPlace(ShAmt);
37356 }
37357 break;
37358 }
37359 case X86ISD::PACKUS: {
37360 // PACKUS is just a truncation if the upper half is zero.
37361 APInt DemandedLHS, DemandedRHS;
37362 getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
37363
37364 Known.One = APInt::getAllOnes(BitWidth * 2);
37365 Known.Zero = APInt::getAllOnes(BitWidth * 2);
37366
37367 KnownBits Known2;
37368 if (!!DemandedLHS) {
37369 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedLHS, Depth + 1);
37370 Known = Known.intersectWith(Known2);
37371 }
37372 if (!!DemandedRHS) {
37373 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedRHS, Depth + 1);
37374 Known = Known.intersectWith(Known2);
37375 }
37376
37377 if (Known.countMinLeadingZeros() < BitWidth)
37378 Known.resetAll();
37379 Known = Known.trunc(BitWidth);
37380 break;
37381 }
37382 case X86ISD::PSHUFB: {
37383 SDValue Src = Op.getOperand(0);
37384 SDValue Idx = Op.getOperand(1);
37385
37386 // If the index vector is never negative (MSB is zero), then all elements
37387 // come from the source vector. This is useful for cases where
37388 // PSHUFB is being used as a LUT (ctpop etc.) - the target shuffle handling
37389 // below will handle the more common constant shuffle mask case.
37390 KnownBits KnownIdx = DAG.computeKnownBits(Idx, DemandedElts, Depth + 1);
37391 if (KnownIdx.isNonNegative())
37392 Known = DAG.computeKnownBits(Src, Depth + 1);
37393 break;
37394 }
37395 case X86ISD::VBROADCAST: {
37396 SDValue Src = Op.getOperand(0);
37397 if (!Src.getSimpleValueType().isVector()) {
37398 Known = DAG.computeKnownBits(Src, Depth + 1);
37399 return;
37400 }
37401 break;
37402 }
37403 case X86ISD::AND: {
37404 if (Op.getResNo() == 0) {
37405 KnownBits Known2;
37406 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
37407 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
37408 Known &= Known2;
37409 }
37410 break;
37411 }
37412 case X86ISD::ANDNP: {
37413 KnownBits Known2;
37414 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
37415 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
37416
37417 // ANDNP = (~X & Y);
37418 Known.One &= Known2.Zero;
37419 Known.Zero |= Known2.One;
37420 break;
37421 }
37422 case X86ISD::FOR: {
37423 KnownBits Known2;
37424 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
37425 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
37426
37427 Known |= Known2;
37428 break;
37429 }
37430 case X86ISD::PSADBW: {
37431 SDValue LHS = Op.getOperand(0);
37432 SDValue RHS = Op.getOperand(1);
37433 assert(VT.getScalarType() == MVT::i64 &&
37434 LHS.getValueType() == RHS.getValueType() &&
37435 LHS.getValueType().getScalarType() == MVT::i8 &&
37436 "Unexpected PSADBW types");
37437 computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth);
37438 break;
37439 }
37440 case X86ISD::PCMPGT:
37441 case X86ISD::PCMPEQ: {
37442 KnownBits KnownLhs =
37443 DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
37444 KnownBits KnownRhs =
37445 DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
37446 std::optional<bool> Res = Opc == X86ISD::PCMPEQ
37447 ? KnownBits::eq(KnownLhs, KnownRhs)
37448 : KnownBits::sgt(KnownLhs, KnownRhs);
37449 if (Res) {
37450 if (*Res)
37451 Known.setAllOnes();
37452 else
37453 Known.setAllZero();
37454 }
37455 break;
37456 }
37457 case X86ISD::VPMADDWD: {
37458 SDValue LHS = Op.getOperand(0);
37459 SDValue RHS = Op.getOperand(1);
37460 assert(VT.getVectorElementType() == MVT::i32 &&
37461 LHS.getValueType() == RHS.getValueType() &&
37462 LHS.getValueType().getVectorElementType() == MVT::i16 &&
37463 "Unexpected PMADDWD types");
37464 computeKnownBitsForPMADDWD(LHS, RHS, Known, DemandedElts, DAG, Depth);
37465 break;
37466 }
37467 case X86ISD::VPMADDUBSW: {
37468 SDValue LHS = Op.getOperand(0);
37469 SDValue RHS = Op.getOperand(1);
37470 assert(VT.getVectorElementType() == MVT::i16 &&
37471 LHS.getValueType() == RHS.getValueType() &&
37472 LHS.getValueType().getVectorElementType() == MVT::i8 &&
37473 "Unexpected PMADDUBSW types");
37474 computeKnownBitsForPMADDUBSW(LHS, RHS, Known, DemandedElts, DAG, Depth);
37475 break;
37476 }
37477 case X86ISD::PMULUDQ: {
37478 KnownBits Known2;
37479 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
37480 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
37481
37482 Known = Known.trunc(BitWidth / 2).zext(BitWidth);
37483 Known2 = Known2.trunc(BitWidth / 2).zext(BitWidth);
37484 Known = KnownBits::mul(Known, Known2);
37485 break;
37486 }
37487 case X86ISD::CMOV: {
37488 Known = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
37489 // If we don't know any bits, early out.
37490 if (Known.isUnknown())
37491 break;
37492 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
37493
37494 // Only known if known in both the LHS and RHS.
37495 Known = Known.intersectWith(Known2);
37496 break;
37497 }
37498 case X86ISD::BEXTR:
37499 case X86ISD::BEXTRI: {
37500 SDValue Op0 = Op.getOperand(0);
37501 SDValue Op1 = Op.getOperand(1);
37502
37503 if (auto* Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
37504 unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0);
37505 unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8);
37506
37507 // If the length is 0, the result is 0.
37508 if (Length == 0) {
37509 Known.setAllZero();
37510 break;
37511 }
37512
37513 if ((Shift + Length) <= BitWidth) {
37514 Known = DAG.computeKnownBits(Op0, Depth + 1);
37515 Known = Known.extractBits(Length, Shift);
37516 Known = Known.zextOrTrunc(BitWidth);
37517 }
37518 }
37519 break;
37520 }
37521 case X86ISD::PDEP: {
37522 KnownBits Known2;
37523 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
37524 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
37525 // Zeros are retained from the mask operand. But not ones.
37526 Known.One.clearAllBits();
37527 // The result will have at least as many trailing zeros as the non-mask
37528 // operand since bits can only map to the same or higher bit position.
37529 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
37530 break;
37531 }
37532 case X86ISD::PEXT: {
37533 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
37534 // The result has as many leading zeros as the number of zeroes in the mask.
37535 unsigned Count = Known.Zero.popcount();
37536 Known.Zero = APInt::getHighBitsSet(BitWidth, Count);
37537 Known.One.clearAllBits();
37538 break;
37539 }
37540 case X86ISD::VTRUNC:
37541 case X86ISD::VTRUNCS:
37542 case X86ISD::VTRUNCUS:
37543 case X86ISD::CVTSI2P:
37544 case X86ISD::CVTUI2P:
37545 case X86ISD::CVTP2SI:
37546 case X86ISD::CVTP2UI:
37547 case X86ISD::MCVTP2SI:
37548 case X86ISD::MCVTP2UI:
37549 case X86ISD::CVTTP2SI:
37550 case X86ISD::CVTTP2UI:
37551 case X86ISD::MCVTTP2SI:
37552 case X86ISD::MCVTTP2UI:
37553 case X86ISD::MCVTSI2P:
37554 case X86ISD::MCVTUI2P:
37555 case X86ISD::VFPROUND:
37556 case X86ISD::VMFPROUND:
37557 case X86ISD::CVTPS2PH:
37558 case X86ISD::MCVTPS2PH: {
37559 // Truncations/Conversions - upper elements are known zero.
37560 EVT SrcVT = Op.getOperand(0).getValueType();
37561 if (SrcVT.isVector()) {
37562 unsigned NumSrcElts = SrcVT.getVectorNumElements();
37563 if (NumElts > NumSrcElts && DemandedElts.countr_zero() >= NumSrcElts)
37564 Known.setAllZero();
37565 }
37566 break;
37567 }
37568 case X86ISD::STRICT_CVTTP2SI:
37569 case X86ISD::STRICT_CVTTP2UI:
37570 case X86ISD::STRICT_CVTSI2P:
37571 case X86ISD::STRICT_CVTUI2P:
37572 case X86ISD::STRICT_VFPROUND:
37573 case X86ISD::STRICT_CVTPS2PH: {
37574 // Strict Conversions - upper elements are known zero.
37575 EVT SrcVT = Op.getOperand(1).getValueType();
37576 if (SrcVT.isVector()) {
37577 unsigned NumSrcElts = SrcVT.getVectorNumElements();
37578 if (NumElts > NumSrcElts && DemandedElts.countr_zero() >= NumSrcElts)
37579 Known.setAllZero();
37580 }
37581 break;
37582 }
37583 case X86ISD::MOVQ2DQ: {
37584 // Move from MMX to XMM. Upper half of XMM should be 0.
37585 if (DemandedElts.countr_zero() >= (NumElts / 2))
37586 Known.setAllZero();
37587 break;
37588 }
37589 case X86ISD::VBROADCAST_LOAD: {
37590 APInt UndefElts;
37591 SmallVector<APInt, 16> EltBits;
37592 if (getTargetConstantBitsFromNode(Op, BitWidth, UndefElts, EltBits,
37593 /*AllowWholeUndefs*/ false,
37594 /*AllowPartialUndefs*/ false)) {
37595 Known.Zero.setAllBits();
37596 Known.One.setAllBits();
37597 for (unsigned I = 0; I != NumElts; ++I) {
37598 if (!DemandedElts[I])
37599 continue;
37600 if (UndefElts[I]) {
37601 Known.resetAll();
37602 break;
37603 }
37604 KnownBits Known2 = KnownBits::makeConstant(EltBits[I]);
37605 Known = Known.intersectWith(Known2);
37606 }
37607 return;
37608 }
37609 break;
37610 }
37611 case X86ISD::HADD:
37612 case X86ISD::HSUB: {
37613 Known = computeKnownBitsForHorizontalOperation(
37614 Op, DemandedElts, Depth, DAG,
37615 [Opc](const KnownBits &KnownLHS, const KnownBits &KnownRHS) {
37616 return KnownBits::computeForAddSub(
37617 /*Add=*/Opc == X86ISD::HADD, /*NSW=*/false, /*NUW=*/false,
37618 KnownLHS, KnownRHS);
37619 });
37620 break;
37621 }
37622 case ISD::INTRINSIC_WO_CHAIN: {
37623 switch (Op->getConstantOperandVal(0)) {
37624 case Intrinsic::x86_sse2_pmadd_wd:
37625 case Intrinsic::x86_avx2_pmadd_wd:
37626 case Intrinsic::x86_avx512_pmaddw_d_512: {
37627 SDValue LHS = Op.getOperand(1);
37628 SDValue RHS = Op.getOperand(2);
37629 assert(VT.getScalarType() == MVT::i32 &&
37630 LHS.getValueType() == RHS.getValueType() &&
37631 LHS.getValueType().getScalarType() == MVT::i16 &&
37632 "Unexpected PMADDWD types");
37633 computeKnownBitsForPMADDWD(LHS, RHS, Known, DemandedElts, DAG, Depth);
37634 break;
37635 }
37636 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
37637 case Intrinsic::x86_avx2_pmadd_ub_sw:
37638 case Intrinsic::x86_avx512_pmaddubs_w_512: {
37639 SDValue LHS = Op.getOperand(1);
37640 SDValue RHS = Op.getOperand(2);
37641 assert(VT.getScalarType() == MVT::i16 &&
37642 LHS.getValueType() == RHS.getValueType() &&
37643 LHS.getValueType().getScalarType() == MVT::i8 &&
37644 "Unexpected PMADDUBSW types");
37645 computeKnownBitsForPMADDUBSW(LHS, RHS, Known, DemandedElts, DAG, Depth);
37646 break;
37647 }
37648 case Intrinsic::x86_sse2_psad_bw:
37649 case Intrinsic::x86_avx2_psad_bw:
37650 case Intrinsic::x86_avx512_psad_bw_512: {
37651 SDValue LHS = Op.getOperand(1);
37652 SDValue RHS = Op.getOperand(2);
37653 assert(VT.getScalarType() == MVT::i64 &&
37654 LHS.getValueType() == RHS.getValueType() &&
37655 LHS.getValueType().getScalarType() == MVT::i8 &&
37656 "Unexpected PSADBW types");
37657 computeKnownBitsForPSADBW(LHS, RHS, Known, DemandedElts, DAG, Depth);
37658 break;
37659 }
37660 }
37661 break;
37662 }
37663 }
37664
37665 // Handle target shuffles.
37666 // TODO - use resolveTargetShuffleInputs once we can limit recursive depth.
37667 if (isTargetShuffle(Opc)) {
37668 SmallVector<int, 64> Mask;
37669 SmallVector<SDValue, 2> Ops;
37670 if (getTargetShuffleMask(Op, true, Ops, Mask)) {
37671 unsigned NumOps = Ops.size();
37672 unsigned NumElts = VT.getVectorNumElements();
37673 if (Mask.size() == NumElts) {
37674 SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
37675 Known.Zero.setAllBits(); Known.One.setAllBits();
37676 for (unsigned i = 0; i != NumElts; ++i) {
37677 if (!DemandedElts[i])
37678 continue;
37679 int M = Mask[i];
37680 if (M == SM_SentinelUndef) {
37681 // For UNDEF elements, we don't know anything about the common state
37682 // of the shuffle result.
37683 Known.resetAll();
37684 break;
37685 }
37686 if (M == SM_SentinelZero) {
37687 Known.One.clearAllBits();
37688 continue;
37689 }
37690 assert(0 <= M && (unsigned)M < (NumOps * NumElts) &&
37691 "Shuffle index out of range");
37692
37693 unsigned OpIdx = (unsigned)M / NumElts;
37694 unsigned EltIdx = (unsigned)M % NumElts;
37695 if (Ops[OpIdx].getValueType() != VT) {
37696 // TODO - handle target shuffle ops with different value types.
37697 Known.resetAll();
37698 break;
37699 }
37700 DemandedOps[OpIdx].setBit(EltIdx);
37701 }
37702 // Known bits are the values that are shared by every demanded element.
37703 for (unsigned i = 0; i != NumOps && !Known.isUnknown(); ++i) {
37704 if (!DemandedOps[i])
37705 continue;
37706 KnownBits Known2 =
37707 DAG.computeKnownBits(Ops[i], DemandedOps[i], Depth + 1);
37708 Known = Known.intersectWith(Known2);
37709 }
37710 }
37711 }
37712 }
37713}
37714
37715unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
37716 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
37717 unsigned Depth) const {
37718 EVT VT = Op.getValueType();
37719 unsigned VTBits = VT.getScalarSizeInBits();
37720 unsigned Opcode = Op.getOpcode();
37721 switch (Opcode) {
37722 case X86ISD::SETCC_CARRY:
37723 // SETCC_CARRY sets the dest to ~0 for true or 0 for false.
37724 return VTBits;
37725
37726 case X86ISD::VTRUNC: {
37727 SDValue Src = Op.getOperand(0);
37728 MVT SrcVT = Src.getSimpleValueType();
37729 unsigned NumSrcBits = SrcVT.getScalarSizeInBits();
37730 assert(VTBits < NumSrcBits && "Illegal truncation input type");
37731 APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
37732 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
37733 if (Tmp > (NumSrcBits - VTBits))
37734 return Tmp - (NumSrcBits - VTBits);
37735 return 1;
37736 }
37737
37738 case X86ISD::PACKSS: {
37739 // PACKSS is just a truncation if the sign bits extend to the packed size.
37740 APInt DemandedLHS, DemandedRHS;
37741 getPackDemandedElts(Op.getValueType(), DemandedElts, DemandedLHS,
37742 DemandedRHS);
37743
37744 // Helper to detect PACKSSDW(BITCAST(PACKSSDW(X)),BITCAST(PACKSSDW(Y)))
37745 // patterns often used to compact vXi64 allsignbit patterns.
37746 auto NumSignBitsPACKSS = [&](SDValue V, const APInt &Elts) -> unsigned {
37747 SDValue BC = peekThroughBitcasts(V);
37748 if (BC.getOpcode() == X86ISD::PACKSS &&
37749 BC.getScalarValueSizeInBits() == 16 &&
37750 V.getScalarValueSizeInBits() == 32) {
37751 SDValue BC0 = peekThroughBitcasts(BC.getOperand(0));
37752 SDValue BC1 = peekThroughBitcasts(BC.getOperand(1));
37753 if (BC0.getScalarValueSizeInBits() == 64 &&
37754 BC1.getScalarValueSizeInBits() == 64 &&
37755 DAG.ComputeNumSignBits(BC0, Depth + 1) == 64 &&
37756 DAG.ComputeNumSignBits(BC1, Depth + 1) == 64)
37757 return 32;
37758 }
37759 return DAG.ComputeNumSignBits(V, Elts, Depth + 1);
37760 };
37761
37762 unsigned SrcBits = Op.getOperand(0).getScalarValueSizeInBits();
37763 unsigned Tmp0 = SrcBits, Tmp1 = SrcBits;
37764 if (!!DemandedLHS)
37765 Tmp0 = NumSignBitsPACKSS(Op.getOperand(0), DemandedLHS);
37766 if (!!DemandedRHS)
37767 Tmp1 = NumSignBitsPACKSS(Op.getOperand(1), DemandedRHS);
37768 unsigned Tmp = std::min(Tmp0, Tmp1);
37769 if (Tmp > (SrcBits - VTBits))
37770 return Tmp - (SrcBits - VTBits);
37771 return 1;
37772 }
37773
37774 case X86ISD::VBROADCAST: {
37775 SDValue Src = Op.getOperand(0);
37776 if (!Src.getSimpleValueType().isVector())
37777 return DAG.ComputeNumSignBits(Src, Depth + 1);
37778 break;
37779 }
37780
37781 case X86ISD::VSHLI: {
37782 SDValue Src = Op.getOperand(0);
37783 const APInt &ShiftVal = Op.getConstantOperandAPInt(1);
37784 if (ShiftVal.uge(VTBits))
37785 return VTBits; // Shifted all bits out --> zero.
37786 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
37787 if (ShiftVal.uge(Tmp))
37788 return 1; // Shifted all sign bits out --> unknown.
37789 return Tmp - ShiftVal.getZExtValue();
37790 }
37791
37792 case X86ISD::VSRAI: {
37793 SDValue Src = Op.getOperand(0);
37794 APInt ShiftVal = Op.getConstantOperandAPInt(1);
37795 if (ShiftVal.uge(VTBits - 1))
37796 return VTBits; // Sign splat.
37797 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
37798 ShiftVal += Tmp;
37799 return ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
37800 }
37801
37802 case X86ISD::FSETCC:
37803 // cmpss/cmpsd return zero/all-bits result values in the bottom element.
37804 if (VT == MVT::f32 || VT == MVT::f64 ||
37805 ((VT == MVT::v4f32 || VT == MVT::v2f64) && DemandedElts == 1))
37806 return VTBits;
37807 break;
37808
37809 case X86ISD::PCMPGT:
37810 case X86ISD::PCMPEQ:
37811 case X86ISD::CMPP:
37812 case X86ISD::VPCOM:
37813 case X86ISD::VPCOMU:
37814 // Vector compares return zero/all-bits result values.
37815 return VTBits;
37816
37817 case X86ISD::ANDNP: {
37818 unsigned Tmp0 =
37819 DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
37820 if (Tmp0 == 1) return 1; // Early out.
37821 unsigned Tmp1 =
37822 DAG.ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
37823 return std::min(Tmp0, Tmp1);
37824 }
37825
37826 case X86ISD::CMOV: {
37827 unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth+1);
37828 if (Tmp0 == 1) return 1; // Early out.
37829 unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth+1);
37830 return std::min(Tmp0, Tmp1);
37831 }
37832 }
37833
37834 // Handle target shuffles.
37835 // TODO - use resolveTargetShuffleInputs once we can limit recursive depth.
37836 if (isTargetShuffle(Opcode)) {
37837 SmallVector<int, 64> Mask;
37838 SmallVector<SDValue, 2> Ops;
37839 if (getTargetShuffleMask(Op, true, Ops, Mask)) {
37840 unsigned NumOps = Ops.size();
37841 unsigned NumElts = VT.getVectorNumElements();
37842 if (Mask.size() == NumElts) {
37843 SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
37844 for (unsigned i = 0; i != NumElts; ++i) {
37845 if (!DemandedElts[i])
37846 continue;
37847 int M = Mask[i];
37848 if (M == SM_SentinelUndef) {
37849 // For UNDEF elements, we don't know anything about the common state
37850 // of the shuffle result.
37851 return 1;
37852 } else if (M == SM_SentinelZero) {
37853 // Zero = all sign bits.
37854 continue;
37855 }
37856 assert(0 <= M && (unsigned)M < (NumOps * NumElts) &&
37857 "Shuffle index out of range");
37858
37859 unsigned OpIdx = (unsigned)M / NumElts;
37860 unsigned EltIdx = (unsigned)M % NumElts;
37861 if (Ops[OpIdx].getValueType() != VT) {
37862 // TODO - handle target shuffle ops with different value types.
37863 return 1;
37864 }
37865 DemandedOps[OpIdx].setBit(EltIdx);
37866 }
37867 unsigned Tmp0 = VTBits;
37868 for (unsigned i = 0; i != NumOps && Tmp0 > 1; ++i) {
37869 if (!DemandedOps[i])
37870 continue;
37871 unsigned Tmp1 =
37872 DAG.ComputeNumSignBits(Ops[i], DemandedOps[i], Depth + 1);
37873 Tmp0 = std::min(Tmp0, Tmp1);
37874 }
37875 return Tmp0;
37876 }
37877 }
37878 }
37879
37880 // Fallback case.
37881 return 1;
37882}
37883
37884SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
37885 if (N->getOpcode() == X86ISD::Wrapper || N->getOpcode() == X86ISD::WrapperRIP)
37886 return N->getOperand(0);
37887 return N;
37888}
37889
37890// Helper to look for a normal load that can be narrowed into a vzload with the
37891// specified VT and memory VT. Returns SDValue() on failure.
37892static SDValue narrowLoadToVZLoad(LoadSDNode *LN, MVT MemVT, MVT VT,
37893 SelectionDAG &DAG) {
37894 // Can't if the load is volatile or atomic.
37895 if (!LN->isSimple())
37896 return SDValue();
37897
37898 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
37899 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
37900 return DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, SDLoc(LN), Tys, Ops, MemVT,
37901 LN->getPointerInfo(), LN->getOriginalAlign(),
37902 LN->getMemOperand()->getFlags());
37903}
37904
37905// Attempt to match a combined shuffle mask against supported unary shuffle
37906// instructions.
37907// TODO: Investigate sharing more of this with shuffle lowering.
37908static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
37909 bool AllowFloatDomain, bool AllowIntDomain,
37910 SDValue V1, const SelectionDAG &DAG,
37911 const X86Subtarget &Subtarget, unsigned &Shuffle,
37912 MVT &SrcVT, MVT &DstVT) {
37913 unsigned NumMaskElts = Mask.size();
37914 unsigned MaskEltSize = MaskVT.getScalarSizeInBits();
37915
37916 // Match against a VZEXT_MOVL vXi32 and vXi16 zero-extending instruction.
37917 if (Mask[0] == 0 &&
37918 (MaskEltSize == 32 || (MaskEltSize == 16 && Subtarget.hasFP16()))) {
37919 if ((isUndefOrZero(Mask[1]) && isUndefInRange(Mask, 2, NumMaskElts - 2)) ||
37920 (V1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
37921 isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1))) {
37922 Shuffle = X86ISD::VZEXT_MOVL;
37923 if (MaskEltSize == 16)
37924 SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);
37925 else
37926 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
37927 return true;
37928 }
37929 }
37930
37931 // Match against a ANY/SIGN/ZERO_EXTEND_VECTOR_INREG instruction.
37932 // TODO: Add 512-bit vector support (split AVX512F and AVX512BW).
37933 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE41()) ||
37934 (MaskVT.is256BitVector() && Subtarget.hasInt256()))) {
37935 unsigned MaxScale = 64 / MaskEltSize;
37936 bool UseSign = V1.getScalarValueSizeInBits() == MaskEltSize &&
37937 DAG.ComputeNumSignBits(V1) == MaskEltSize;
37938 for (unsigned Scale = 2; Scale <= MaxScale; Scale *= 2) {
37939 bool MatchAny = true;
37940 bool MatchZero = true;
37941 bool MatchSign = UseSign;
37942 unsigned NumDstElts = NumMaskElts / Scale;
37943 for (unsigned i = 0;
37944 i != NumDstElts && (MatchAny || MatchSign || MatchZero); ++i) {
37945 if (!isUndefOrEqual(Mask[i * Scale], (int)i)) {
37946 MatchAny = MatchSign = MatchZero = false;
37947 break;
37948 }
37949 unsigned Pos = (i * Scale) + 1;
37950 unsigned Len = Scale - 1;
37951 MatchAny &= isUndefInRange(Mask, Pos, Len);
37952 MatchZero &= isUndefOrZeroInRange(Mask, Pos, Len);
37953 MatchSign &= isUndefOrEqualInRange(Mask, (int)i, Pos, Len);
37954 }
37955 if (MatchAny || MatchSign || MatchZero) {
37956 assert((MatchSign || MatchZero) &&
37957 "Failed to match sext/zext but matched aext?");
37958 unsigned SrcSize = std::max(128u, NumDstElts * MaskEltSize);
37959 MVT ScalarTy = MaskVT.isInteger() ? MaskVT.getScalarType()
37960 : MVT::getIntegerVT(MaskEltSize);
37961 SrcVT = MVT::getVectorVT(ScalarTy, SrcSize / MaskEltSize);
37962
37963 Shuffle = unsigned(
37964 MatchAny ? ISD::ANY_EXTEND
37965 : (MatchSign ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND));
37966 if (SrcVT.getVectorNumElements() != NumDstElts)
37967 Shuffle = DAG.getOpcode_EXTEND_VECTOR_INREG(Shuffle);
37968
37969 DstVT = MVT::getIntegerVT(Scale * MaskEltSize);
37970 DstVT = MVT::getVectorVT(DstVT, NumDstElts);
37971 return true;
37972 }
37973 }
37974 }
37975
37976 // Match against a VZEXT_MOVL instruction, SSE1 only supports 32-bits (MOVSS).
37977 if (((MaskEltSize == 32) || (MaskEltSize == 64 && Subtarget.hasSSE2()) ||
37978 (MaskEltSize == 16 && Subtarget.hasFP16())) &&
37979 isUndefOrEqual(Mask[0], 0) &&
37980 isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1)) {
37981 Shuffle = X86ISD::VZEXT_MOVL;
37982 if (MaskEltSize == 16)
37983 SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);
37984 else
37985 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
37986 return true;
37987 }
37988
37989 // Check if we have SSE3 which will let us use MOVDDUP etc. The
37990 // instructions are no slower than UNPCKLPD but has the option to
37991 // fold the input operand into even an unaligned memory load.
37992 if (MaskVT.is128BitVector() && Subtarget.hasSSE3() && AllowFloatDomain) {
37993 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG, V1)) {
37994 Shuffle = X86ISD::MOVDDUP;
37995 SrcVT = DstVT = MVT::v2f64;
37996 return true;
37997 }
37998 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) {
37999 Shuffle = X86ISD::MOVSLDUP;
38000 SrcVT = DstVT = MVT::v4f32;
38001 return true;
38002 }
38003 if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3}, DAG, V1)) {
38004 Shuffle = X86ISD::MOVSHDUP;
38005 SrcVT = DstVT = MVT::v4f32;
38006 return true;
38007 }
38008 }
38009
38010 if (MaskVT.is256BitVector() && AllowFloatDomain) {
38011 assert(Subtarget.hasAVX() && "AVX required for 256-bit vector shuffles");
38012 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, DAG, V1)) {
38013 Shuffle = X86ISD::MOVDDUP;
38014 SrcVT = DstVT = MVT::v4f64;
38015 return true;
38016 }
38017 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG,
38018 V1)) {
38019 Shuffle = X86ISD::MOVSLDUP;
38020 SrcVT = DstVT = MVT::v8f32;
38021 return true;
38022 }
38023 if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3, 5, 5, 7, 7}, DAG,
38024 V1)) {
38025 Shuffle = X86ISD::MOVSHDUP;
38026 SrcVT = DstVT = MVT::v8f32;
38027 return true;
38028 }
38029 }
38030
38031 if (MaskVT.is512BitVector() && AllowFloatDomain) {
38032 assert(Subtarget.hasAVX512() &&
38033 "AVX512 required for 512-bit vector shuffles");
38034 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, DAG,
38035 V1)) {
38036 Shuffle = X86ISD::MOVDDUP;
38037 SrcVT = DstVT = MVT::v8f64;
38038 return true;
38039 }
38040 if (isTargetShuffleEquivalent(
38041 MaskVT, Mask,
38042 {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}, DAG, V1)) {
38043 Shuffle = X86ISD::MOVSLDUP;
38044 SrcVT = DstVT = MVT::v16f32;
38045 return true;
38046 }
38047 if (isTargetShuffleEquivalent(
38048 MaskVT, Mask,
38049 {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15}, DAG, V1)) {
38050 Shuffle = X86ISD::MOVSHDUP;
38051 SrcVT = DstVT = MVT::v16f32;
38052 return true;
38053 }
38054 }
38055
38056 return false;
38057}
38058
38059// Attempt to match a combined shuffle mask against supported unary immediate
38060// permute instructions.
38061// TODO: Investigate sharing more of this with shuffle lowering.
38062static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,
38063 const APInt &Zeroable,
38064 bool AllowFloatDomain, bool AllowIntDomain,
38065 const SelectionDAG &DAG,
38066 const X86Subtarget &Subtarget,
38067 unsigned &Shuffle, MVT &ShuffleVT,
38068 unsigned &PermuteImm) {
38069 unsigned NumMaskElts = Mask.size();
38070 unsigned InputSizeInBits = MaskVT.getSizeInBits();
38071 unsigned MaskScalarSizeInBits = InputSizeInBits / NumMaskElts;
38072 MVT MaskEltVT = MVT::getIntegerVT(MaskScalarSizeInBits);
38073 bool ContainsZeros = isAnyZero(Mask);
38074
38075 // Handle VPERMI/VPERMILPD vXi64/vXi64 patterns.
38076 if (!ContainsZeros && MaskScalarSizeInBits == 64) {
38077 // Check for lane crossing permutes.
38078 if (is128BitLaneCrossingShuffleMask(MaskEltVT, Mask)) {
38079 // PERMPD/PERMQ permutes within a 256-bit vector (AVX2+).
38080 if (Subtarget.hasAVX2() && MaskVT.is256BitVector()) {
38081 Shuffle = X86ISD::VPERMI;
38082 ShuffleVT = (AllowFloatDomain ? MVT::v4f64 : MVT::v4i64);
38083 PermuteImm = getV4X86ShuffleImm(Mask);
38084 return true;
38085 }
38086 if (Subtarget.hasAVX512() && MaskVT.is512BitVector()) {
38087 SmallVector<int, 4> RepeatedMask;
38088 if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask)) {
38089 Shuffle = X86ISD::VPERMI;
38090 ShuffleVT = (AllowFloatDomain ? MVT::v8f64 : MVT::v8i64);
38091 PermuteImm = getV4X86ShuffleImm(RepeatedMask);
38092 return true;
38093 }
38094 }
38095 } else if (AllowFloatDomain && Subtarget.hasAVX()) {
38096 // VPERMILPD can permute with a non-repeating shuffle.
38097 Shuffle = X86ISD::VPERMILPI;
38098 ShuffleVT = MVT::getVectorVT(MVT::f64, Mask.size());
38099 PermuteImm = 0;
38100 for (int i = 0, e = Mask.size(); i != e; ++i) {
38101 int M = Mask[i];
38102 if (M == SM_SentinelUndef)
38103 continue;
38104 assert(((M / 2) == (i / 2)) && "Out of range shuffle mask index");
38105 PermuteImm |= (M & 1) << i;
38106 }
38107 return true;
38108 }
38109 }
38110
38111 // We are checking for shuffle match or shift match. Loop twice so we can
38112 // order which we try and match first depending on target preference.
38113 for (unsigned Order = 0; Order < 2; ++Order) {
38114 if (Subtarget.preferLowerShuffleAsShift() ? (Order == 1) : (Order == 0)) {
38115 // Handle PSHUFD/VPERMILPI vXi32/vXf32 repeated patterns.
38116 // AVX introduced the VPERMILPD/VPERMILPS float permutes, before then we
38117 // had to use 2-input SHUFPD/SHUFPS shuffles (not handled here).
38118 if ((MaskScalarSizeInBits == 64 || MaskScalarSizeInBits == 32) &&
38119 !ContainsZeros && (AllowIntDomain || Subtarget.hasAVX())) {
38120 SmallVector<int, 4> RepeatedMask;
38121 if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) {
38122 // Narrow the repeated mask to create 32-bit element permutes.
38123 SmallVector<int, 4> WordMask = RepeatedMask;
38124 if (MaskScalarSizeInBits == 64)
38125 narrowShuffleMaskElts(2, RepeatedMask, WordMask);
38126
38127 Shuffle = (AllowIntDomain ? X86ISD::PSHUFD : X86ISD::VPERMILPI);
38128 ShuffleVT = (AllowIntDomain ? MVT::i32 : MVT::f32);
38129 ShuffleVT = MVT::getVectorVT(ShuffleVT, InputSizeInBits / 32);
38130 PermuteImm = getV4X86ShuffleImm(WordMask);
38131 return true;
38132 }
38133 }
38134
38135 // Handle PSHUFLW/PSHUFHW vXi16 repeated patterns.
38136 if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits == 16 &&
38137 ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
38138 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
38139 (MaskVT.is512BitVector() && Subtarget.hasBWI()))) {
38140 SmallVector<int, 4> RepeatedMask;
38141 if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) {
38142 ArrayRef<int> LoMask(RepeatedMask.data() + 0, 4);
38143 ArrayRef<int> HiMask(RepeatedMask.data() + 4, 4);
38144
38145 // PSHUFLW: permute lower 4 elements only.
38146 if (isUndefOrInRange(LoMask, 0, 4) &&
38147 isSequentialOrUndefInRange(HiMask, 0, 4, 4)) {
38148 Shuffle = X86ISD::PSHUFLW;
38149 ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);
38150 PermuteImm = getV4X86ShuffleImm(LoMask);
38151 return true;
38152 }
38153
38154 // PSHUFHW: permute upper 4 elements only.
38155 if (isUndefOrInRange(HiMask, 4, 8) &&
38156 isSequentialOrUndefInRange(LoMask, 0, 4, 0)) {
38157 // Offset the HiMask so that we can create the shuffle immediate.
38158 int OffsetHiMask[4];
38159 for (int i = 0; i != 4; ++i)
38160 OffsetHiMask[i] = (HiMask[i] < 0 ? HiMask[i] : HiMask[i] - 4);
38161
38162 Shuffle = X86ISD::PSHUFHW;
38163 ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);
38164 PermuteImm = getV4X86ShuffleImm(OffsetHiMask);
38165 return true;
38166 }
38167 }
38168 }
38169 } else {
38170 // Attempt to match against bit rotates.
38171 if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits < 64 &&
38172 ((MaskVT.is128BitVector() && Subtarget.hasXOP()) ||
38173 Subtarget.hasAVX512())) {
38174 int RotateAmt = matchShuffleAsBitRotate(ShuffleVT, MaskScalarSizeInBits,
38175 Subtarget, Mask);
38176 if (0 < RotateAmt) {
38177 Shuffle = X86ISD::VROTLI;
38178 PermuteImm = (unsigned)RotateAmt;
38179 return true;
38180 }
38181 }
38182 }
38183 // Attempt to match against byte/bit shifts.
38184 if (AllowIntDomain &&
38185 ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
38186 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
38187 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
38188 int ShiftAmt =
38189 matchShuffleAsShift(ShuffleVT, Shuffle, MaskScalarSizeInBits, Mask, 0,
38190 Zeroable, Subtarget);
38191 if (0 < ShiftAmt && (!ShuffleVT.is512BitVector() || Subtarget.hasBWI() ||
38192 32 <= ShuffleVT.getScalarSizeInBits())) {
38193 // Byte shifts can be slower so only match them on second attempt.
38194 if (Order == 0 &&
38195 (Shuffle == X86ISD::VSHLDQ || Shuffle == X86ISD::VSRLDQ))
38196 continue;
38197
38198 PermuteImm = (unsigned)ShiftAmt;
38199 return true;
38200 }
38201
38202 }
38203 }
38204
38205 return false;
38206}
38207
38208// Attempt to match a combined unary shuffle mask against supported binary
38209// shuffle instructions.
38210// TODO: Investigate sharing more of this with shuffle lowering.
38211static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
38212 bool AllowFloatDomain, bool AllowIntDomain,
38213 SDValue &V1, SDValue &V2, const SDLoc &DL,
38214 SelectionDAG &DAG, const X86Subtarget &Subtarget,
38215 unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,
38216 bool IsUnary) {
38217 unsigned NumMaskElts = Mask.size();
38218 unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
38219 unsigned SizeInBits = MaskVT.getSizeInBits();
38220
38221 if (MaskVT.is128BitVector()) {
38222 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, DAG) &&
38223 AllowFloatDomain) {
38224 V2 = V1;
38225 V1 = (SM_SentinelUndef == Mask[0] ? DAG.getUNDEF(MVT::v4f32) : V1);
38226 Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKL : X86ISD::MOVLHPS;
38227 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;
38228 return true;
38229 }
38230 if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1}, DAG) &&
38231 AllowFloatDomain) {
38232 V2 = V1;
38233 Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKH : X86ISD::MOVHLPS;
38234 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;
38235 return true;
38236 }
38237 if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 3}, DAG) &&
38238 Subtarget.hasSSE2() && (AllowFloatDomain || !Subtarget.hasSSE41())) {
38239 std::swap(V1, V2);
38240 Shuffle = X86ISD::MOVSD;
38241 SrcVT = DstVT = MVT::v2f64;
38242 return true;
38243 }
38244 if (isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG) &&
38245 (AllowFloatDomain || !Subtarget.hasSSE41())) {
38246 Shuffle = X86ISD::MOVSS;
38247 SrcVT = DstVT = MVT::v4f32;
38248 return true;
38249 }
38250 if (isTargetShuffleEquivalent(MaskVT, Mask, {8, 1, 2, 3, 4, 5, 6, 7},
38251 DAG) &&
38252 Subtarget.hasFP16()) {
38253 Shuffle = X86ISD::MOVSH;
38254 SrcVT = DstVT = MVT::v8f16;
38255 return true;
38256 }
38257 }
38258
38259 // Attempt to match against either an unary or binary PACKSS/PACKUS shuffle.
38260 if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) ||
38261 ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) ||
38262 ((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) {
38263 if (matchShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG,
38264 Subtarget)) {
38265 DstVT = MaskVT;
38266 return true;
38267 }
38268 }
38269 // TODO: Can we handle this inside matchShuffleWithPACK?
38270 if (MaskVT == MVT::v4i32 && Subtarget.hasSSE2() &&
38271 isTargetShuffleEquivalent(MaskVT, Mask, {0, 2, 4, 6}, DAG) &&
38272 V1.getScalarValueSizeInBits() == 64 &&
38273 V2.getScalarValueSizeInBits() == 64) {
38274 // Use (SSE41) PACKUSWD if the leading zerobits goto the lowest 16-bits.
38275 unsigned MinLZV1 = DAG.computeKnownBits(V1).countMinLeadingZeros();
38276 unsigned MinLZV2 = DAG.computeKnownBits(V2).countMinLeadingZeros();
38277 if (Subtarget.hasSSE41() && MinLZV1 >= 48 && MinLZV2 >= 48) {
38278 SrcVT = MVT::v4i32;
38279 DstVT = MVT::v8i16;
38280 Shuffle = X86ISD::PACKUS;
38281 return true;
38282 }
38283 // Use PACKUSBW if the leading zerobits goto the lowest 8-bits.
38284 if (MinLZV1 >= 56 && MinLZV2 >= 56) {
38285 SrcVT = MVT::v8i16;
38286 DstVT = MVT::v16i8;
38287 Shuffle = X86ISD::PACKUS;
38288 return true;
38289 }
38290 // Use PACKSSWD if the signbits extend to the lowest 16-bits.
38291 if (DAG.ComputeNumSignBits(V1) > 48 && DAG.ComputeNumSignBits(V2) > 48) {
38292 SrcVT = MVT::v4i32;
38293 DstVT = MVT::v8i16;
38294 Shuffle = X86ISD::PACKSS;
38295 return true;
38296 }
38297 }
38298
38299 // Attempt to match against either a unary or binary UNPCKL/UNPCKH shuffle.
38300 if ((MaskVT == MVT::v4f32 && Subtarget.hasSSE1()) ||
38301 (MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
38302 (MaskVT.is256BitVector() && 32 <= EltSizeInBits && Subtarget.hasAVX()) ||
38303 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
38304 (MaskVT.is512BitVector() && Subtarget.hasAVX512() &&
38305 (32 <= EltSizeInBits || Subtarget.hasBWI()))) {
38306 if (matchShuffleWithUNPCK(MaskVT, V1, V2, Shuffle, IsUnary, Mask, DL, DAG,
38307 Subtarget)) {
38308 SrcVT = DstVT = MaskVT;
38309 if (MaskVT.is256BitVector() && !Subtarget.hasAVX2())
38310 SrcVT = DstVT = (32 == EltSizeInBits ? MVT::v8f32 : MVT::v4f64);
38311 return true;
38312 }
38313 }
38314
38315 // Attempt to match against a OR if we're performing a blend shuffle and the
38316 // non-blended source element is zero in each case.
38317 // TODO: Handle cases where V1/V2 sizes doesn't match SizeInBits.
38318 if (SizeInBits == V1.getValueSizeInBits() &&
38319 SizeInBits == V2.getValueSizeInBits() &&
38320 (EltSizeInBits % V1.getScalarValueSizeInBits()) == 0 &&
38321 (EltSizeInBits % V2.getScalarValueSizeInBits()) == 0) {
38322 bool IsBlend = true;
38323 unsigned NumV1Elts = V1.getValueType().getVectorNumElements();
38324 unsigned NumV2Elts = V2.getValueType().getVectorNumElements();
38325 unsigned Scale1 = NumV1Elts / NumMaskElts;
38326 unsigned Scale2 = NumV2Elts / NumMaskElts;
38327 APInt DemandedZeroV1 = APInt::getZero(NumV1Elts);
38328 APInt DemandedZeroV2 = APInt::getZero(NumV2Elts);
38329 for (unsigned i = 0; i != NumMaskElts; ++i) {
38330 int M = Mask[i];
38331 if (M == SM_SentinelUndef)
38332 continue;
38333 if (M == SM_SentinelZero) {
38334 DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
38335 DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
38336 continue;
38337 }
38338 if (M == (int)i) {
38339 DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2);
38340 continue;
38341 }
38342 if (M == (int)(i + NumMaskElts)) {
38343 DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1);
38344 continue;
38345 }
38346 IsBlend = false;
38347 break;
38348 }
38349 if (IsBlend) {
38350 if (DAG.MaskedVectorIsZero(V1, DemandedZeroV1) &&
38351 DAG.MaskedVectorIsZero(V2, DemandedZeroV2)) {
38352 Shuffle = ISD::OR;
38353 SrcVT = DstVT = MaskVT.changeTypeToInteger();
38354 return true;
38355 }
38356 if (NumV1Elts == NumV2Elts && NumV1Elts == NumMaskElts) {
38357 // FIXME: handle mismatched sizes?
38358 // TODO: investigate if `ISD::OR` handling in
38359 // `TargetLowering::SimplifyDemandedVectorElts` can be improved instead.
38360 auto computeKnownBitsElementWise = [&DAG](SDValue V) {
38361 unsigned NumElts = V.getValueType().getVectorNumElements();
38362 KnownBits Known(NumElts);
38363 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
38364 APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);
38365 KnownBits PeepholeKnown = DAG.computeKnownBits(V, Mask);
38366 if (PeepholeKnown.isZero())
38367 Known.Zero.setBit(EltIdx);
38368 if (PeepholeKnown.isAllOnes())
38369 Known.One.setBit(EltIdx);
38370 }
38371 return Known;
38372 };
38373
38374 KnownBits V1Known = computeKnownBitsElementWise(V1);
38375 KnownBits V2Known = computeKnownBitsElementWise(V2);
38376
38377 for (unsigned i = 0; i != NumMaskElts && IsBlend; ++i) {
38378 int M = Mask[i];
38379 if (M == SM_SentinelUndef)
38380 continue;
38381 if (M == SM_SentinelZero) {
38382 IsBlend &= V1Known.Zero[i] && V2Known.Zero[i];
38383 continue;
38384 }
38385 if (M == (int)i) {
38386 IsBlend &= V2Known.Zero[i] || V1Known.One[i];
38387 continue;
38388 }
38389 if (M == (int)(i + NumMaskElts)) {
38390 IsBlend &= V1Known.Zero[i] || V2Known.One[i];
38391 continue;
38392 }
38393 llvm_unreachable("will not get here.");
38394 }
38395 if (IsBlend) {
38396 Shuffle = ISD::OR;
38397 SrcVT = DstVT = MaskVT.changeTypeToInteger();
38398 return true;
38399 }
38400 }
38401 }
38402 }
38403
38404 return false;
38405}
38406
38407static bool matchBinaryPermuteShuffle(
38408 MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable,
38409 bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2,
38410 const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget,
38411 unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) {
38412 unsigned NumMaskElts = Mask.size();
38413 unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
38414
38415 // Attempt to match against VALIGND/VALIGNQ rotate.
38416 if (AllowIntDomain && (EltSizeInBits == 64 || EltSizeInBits == 32) &&
38417 ((MaskVT.is128BitVector() && Subtarget.hasVLX()) ||
38418 (MaskVT.is256BitVector() && Subtarget.hasVLX()) ||
38419 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
38420 if (!isAnyZero(Mask)) {
38421 int Rotation = matchShuffleAsElementRotate(V1, V2, Mask);
38422 if (0 < Rotation) {
38423 Shuffle = X86ISD::VALIGN;
38424 if (EltSizeInBits == 64)
38425 ShuffleVT = MVT::getVectorVT(MVT::i64, MaskVT.getSizeInBits() / 64);
38426 else
38427 ShuffleVT = MVT::getVectorVT(MVT::i32, MaskVT.getSizeInBits() / 32);
38428 PermuteImm = Rotation;
38429 return true;
38430 }
38431 }
38432 }
38433
38434 // Attempt to match against PALIGNR byte rotate.
38435 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSSE3()) ||
38436 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
38437 (MaskVT.is512BitVector() && Subtarget.hasBWI()))) {
38438 int ByteRotation = matchShuffleAsByteRotate(MaskVT, V1, V2, Mask);
38439 if (0 < ByteRotation) {
38440 Shuffle = X86ISD::PALIGNR;
38441 ShuffleVT = MVT::getVectorVT(MVT::i8, MaskVT.getSizeInBits() / 8);
38442 PermuteImm = ByteRotation;
38443 return true;
38444 }
38445 }
38446
38447 // Attempt to combine to X86ISD::BLENDI.
38448 if ((NumMaskElts <= 8 && ((Subtarget.hasSSE41() && MaskVT.is128BitVector()) ||
38449 (Subtarget.hasAVX() && MaskVT.is256BitVector()))) ||
38450 (MaskVT == MVT::v16i16 && Subtarget.hasAVX2())) {
38451 uint64_t BlendMask = 0;
38452 bool ForceV1Zero = false, ForceV2Zero = false;
38453 SmallVector<int, 8> TargetMask(Mask);
38454 if (matchShuffleAsBlend(MaskVT, V1, V2, TargetMask, Zeroable, ForceV1Zero,
38455 ForceV2Zero, BlendMask)) {
38456 if (MaskVT == MVT::v16i16) {
38457 // We can only use v16i16 PBLENDW if the lanes are repeated.
38458 SmallVector<int, 8> RepeatedMask;
38459 if (isRepeatedTargetShuffleMask(128, MaskVT, TargetMask,
38460 RepeatedMask)) {
38461 assert(RepeatedMask.size() == 8 &&
38462 "Repeated mask size doesn't match!");
38463 PermuteImm = 0;
38464 for (int i = 0; i < 8; ++i)
38465 if (RepeatedMask[i] >= 8)
38466 PermuteImm |= 1 << i;
38467 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
38468 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
38469 Shuffle = X86ISD::BLENDI;
38470 ShuffleVT = MaskVT;
38471 return true;
38472 }
38473 } else {
38474 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
38475 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
38476 PermuteImm = (unsigned)BlendMask;
38477 Shuffle = X86ISD::BLENDI;
38478 ShuffleVT = MaskVT;
38479 return true;
38480 }
38481 }
38482 }
38483
38484 // Attempt to combine to INSERTPS, but only if it has elements that need to
38485 // be set to zero.
38486 if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() &&
38487 MaskVT.is128BitVector() && isAnyZero(Mask) &&
38488 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
38489 Shuffle = X86ISD::INSERTPS;
38490 ShuffleVT = MVT::v4f32;
38491 return true;
38492 }
38493
38494 // Attempt to combine to SHUFPD.
38495 if (AllowFloatDomain && EltSizeInBits == 64 &&
38496 ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
38497 (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
38498 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
38499 bool ForceV1Zero = false, ForceV2Zero = false;
38500 if (matchShuffleWithSHUFPD(MaskVT, V1, V2, ForceV1Zero, ForceV2Zero,
38501 PermuteImm, Mask, Zeroable)) {
38502 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
38503 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
38504 Shuffle = X86ISD::SHUFP;
38505 ShuffleVT = MVT::getVectorVT(MVT::f64, MaskVT.getSizeInBits() / 64);
38506 return true;
38507 }
38508 }
38509
38510 // Attempt to combine to SHUFPS.
38511 if (AllowFloatDomain && EltSizeInBits == 32 &&
38512 ((MaskVT.is128BitVector() && Subtarget.hasSSE1()) ||
38513 (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
38514 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
38515 SmallVector<int, 4> RepeatedMask;
38516 if (isRepeatedTargetShuffleMask(128, MaskVT, Mask, RepeatedMask)) {
38517 // Match each half of the repeated mask, to determine if its just
38518 // referencing one of the vectors, is zeroable or entirely undef.
38519 auto MatchHalf = [&](unsigned Offset, int &S0, int &S1) {
38520 int M0 = RepeatedMask[Offset];
38521 int M1 = RepeatedMask[Offset + 1];
38522
38523 if (isUndefInRange(RepeatedMask, Offset, 2)) {
38524 return DAG.getUNDEF(MaskVT);
38525 } else if (isUndefOrZeroInRange(RepeatedMask, Offset, 2)) {
38526 S0 = (SM_SentinelUndef == M0 ? -1 : 0);
38527 S1 = (SM_SentinelUndef == M1 ? -1 : 1);
38528 return getZeroVector(MaskVT, Subtarget, DAG, DL);
38529 } else if (isUndefOrInRange(M0, 0, 4) && isUndefOrInRange(M1, 0, 4)) {
38530 S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3);
38531 S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3);
38532 return V1;
38533 } else if (isUndefOrInRange(M0, 4, 8) && isUndefOrInRange(M1, 4, 8)) {
38534 S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3);
38535 S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3);
38536 return V2;
38537 }
38538
38539 return SDValue();
38540 };
38541
38542 int ShufMask[4] = {-1, -1, -1, -1};
38543 SDValue Lo = MatchHalf(0, ShufMask[0], ShufMask[1]);
38544 SDValue Hi = MatchHalf(2, ShufMask[2], ShufMask[3]);
38545
38546 if (Lo && Hi) {
38547 V1 = Lo;
38548 V2 = Hi;
38549 Shuffle = X86ISD::SHUFP;
38550 ShuffleVT = MVT::getVectorVT(MVT::f32, MaskVT.getSizeInBits() / 32);
38551 PermuteImm = getV4X86ShuffleImm(ShufMask);
38552 return true;
38553 }
38554 }
38555 }
38556
38557 // Attempt to combine to INSERTPS more generally if X86ISD::SHUFP failed.
38558 if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() &&
38559 MaskVT.is128BitVector() &&
38560 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
38561 Shuffle = X86ISD::INSERTPS;
38562 ShuffleVT = MVT::v4f32;
38563 return true;
38564 }
38565
38566 return false;
38567}
38568
38569static SDValue combineX86ShuffleChainWithExtract(
38570 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
38571 bool HasVariableMask, bool AllowVariableCrossLaneMask,
38572 bool AllowVariablePerLaneMask, SelectionDAG &DAG,
38573 const X86Subtarget &Subtarget);
38574
38575/// Combine an arbitrary chain of shuffles into a single instruction if
38576/// possible.
38577///
38578/// This is the leaf of the recursive combine below. When we have found some
38579/// chain of single-use x86 shuffle instructions and accumulated the combined
38580/// shuffle mask represented by them, this will try to pattern match that mask
38581/// into either a single instruction if there is a special purpose instruction
38582/// for this operation, or into a PSHUFB instruction which is a fully general
38583/// instruction but should only be used to replace chains over a certain depth.
38584static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
38585 ArrayRef<int> BaseMask, int Depth,
38586 bool HasVariableMask,
38587 bool AllowVariableCrossLaneMask,
38588 bool AllowVariablePerLaneMask,
38589 SelectionDAG &DAG,
38590 const X86Subtarget &Subtarget) {
38591 assert(!BaseMask.empty() && "Cannot combine an empty shuffle mask!");
38592 assert((Inputs.size() == 1 || Inputs.size() == 2) &&
38593 "Unexpected number of shuffle inputs!");
38594
38595 SDLoc DL(Root);
38596 MVT RootVT = Root.getSimpleValueType();
38597 unsigned RootSizeInBits = RootVT.getSizeInBits();
38598 unsigned NumRootElts = RootVT.getVectorNumElements();
38599
38600 // Canonicalize shuffle input op to the requested type.
38601 auto CanonicalizeShuffleInput = [&](MVT VT, SDValue Op) {
38602 if (VT.getSizeInBits() > Op.getValueSizeInBits())
38603 Op = widenSubVector(Op, false, Subtarget, DAG, DL, VT.getSizeInBits());
38604 else if (VT.getSizeInBits() < Op.getValueSizeInBits())
38605 Op = extractSubVector(Op, 0, DAG, DL, VT.getSizeInBits());
38606 return DAG.getBitcast(VT, Op);
38607 };
38608
38609 // Find the inputs that enter the chain. Note that multiple uses are OK
38610 // here, we're not going to remove the operands we find.
38611 bool UnaryShuffle = (Inputs.size() == 1);
38612 SDValue V1 = peekThroughBitcasts(Inputs[0]);
38613 SDValue V2 = (UnaryShuffle ? DAG.getUNDEF(V1.getValueType())
38614 : peekThroughBitcasts(Inputs[1]));
38615
38616 MVT VT1 = V1.getSimpleValueType();
38617 MVT VT2 = V2.getSimpleValueType();
38618 assert((RootSizeInBits % VT1.getSizeInBits()) == 0 &&
38619 (RootSizeInBits % VT2.getSizeInBits()) == 0 && "Vector size mismatch");
38620
38621 SDValue Res;
38622
38623 unsigned NumBaseMaskElts = BaseMask.size();
38624 if (NumBaseMaskElts == 1) {
38625 assert(BaseMask[0] == 0 && "Invalid shuffle index found!");
38626 return CanonicalizeShuffleInput(RootVT, V1);
38627 }
38628
38629 bool OptForSize = DAG.shouldOptForSize();
38630 unsigned BaseMaskEltSizeInBits = RootSizeInBits / NumBaseMaskElts;
38631 bool FloatDomain = VT1.isFloatingPoint() || VT2.isFloatingPoint() ||
38632 (RootVT.isFloatingPoint() && Depth >= 1) ||
38633 (RootVT.is256BitVector() && !Subtarget.hasAVX2());
38634
38635 // Don't combine if we are a AVX512/EVEX target and the mask element size
38636 // is different from the root element size - this would prevent writemasks
38637 // from being reused.
38638 bool IsMaskedShuffle = false;
38639 if (RootSizeInBits == 512 || (Subtarget.hasVLX() && RootSizeInBits >= 128)) {
38640 if (Root.hasOneUse() && Root->use_begin()->getOpcode() == ISD::VSELECT &&
38641 Root->use_begin()->getOperand(0).getScalarValueSizeInBits() == 1) {
38642 IsMaskedShuffle = true;
38643 }
38644 }
38645
38646 // If we are shuffling a splat (and not introducing zeros) then we can just
38647 // use it directly. This works for smaller elements as well as they already
38648 // repeat across each mask element.
38649 if (UnaryShuffle && !isAnyZero(BaseMask) &&
38650 V1.getValueSizeInBits() >= RootSizeInBits &&
38651 (BaseMaskEltSizeInBits % V1.getScalarValueSizeInBits()) == 0 &&
38652 DAG.isSplatValue(V1, /*AllowUndefs*/ false)) {
38653 return CanonicalizeShuffleInput(RootVT, V1);
38654 }
38655
38656 SmallVector<int, 64> Mask(BaseMask);
38657
38658 // See if the shuffle is a hidden identity shuffle - repeated args in HOPs
38659 // etc. can be simplified.
38660 if (VT1 == VT2 && VT1.getSizeInBits() == RootSizeInBits && VT1.isVector()) {
38661 SmallVector<int> ScaledMask, IdentityMask;
38662 unsigned NumElts = VT1.getVectorNumElements();
38663 if (Mask.size() <= NumElts &&
38664 scaleShuffleElements(Mask, NumElts, ScaledMask)) {
38665 for (unsigned i = 0; i != NumElts; ++i)
38666 IdentityMask.push_back(i);
38667 if (isTargetShuffleEquivalent(RootVT, ScaledMask, IdentityMask, DAG, V1,
38668 V2))
38669 return CanonicalizeShuffleInput(RootVT, V1);
38670 }
38671 }
38672
38673 // Handle 128/256-bit lane shuffles of 512-bit vectors.
38674 if (RootVT.is512BitVector() &&
38675 (NumBaseMaskElts == 2 || NumBaseMaskElts == 4)) {
38676 // If the upper subvectors are zeroable, then an extract+insert is more
38677 // optimal than using X86ISD::SHUF128. The insertion is free, even if it has
38678 // to zero the upper subvectors.
38679 if (isUndefOrZeroInRange(Mask, 1, NumBaseMaskElts - 1)) {
38680 if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
38681 return SDValue(); // Nothing to do!
38682 assert(isInRange(Mask[0], 0, NumBaseMaskElts) &&
38683 "Unexpected lane shuffle");
38684 Res = CanonicalizeShuffleInput(RootVT, V1);
38685 unsigned SubIdx = Mask[0] * (NumRootElts / NumBaseMaskElts);
38686 bool UseZero = isAnyZero(Mask);
38687 Res = extractSubVector(Res, SubIdx, DAG, DL, BaseMaskEltSizeInBits);
38688 return widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits);
38689 }
38690
38691 // Narrow shuffle mask to v4x128.
38692 SmallVector<int, 4> ScaledMask;
38693 assert((BaseMaskEltSizeInBits % 128) == 0 && "Illegal mask size");
38694 narrowShuffleMaskElts(BaseMaskEltSizeInBits / 128, Mask, ScaledMask);
38695
38696 // Try to lower to vshuf64x2/vshuf32x4.
38697 auto MatchSHUF128 = [&](MVT ShuffleVT, const SDLoc &DL,
38698 ArrayRef<int> ScaledMask, SDValue V1, SDValue V2,
38699 SelectionDAG &DAG) {
38700 int PermMask[4] = {-1, -1, -1, -1};
38701 // Ensure elements came from the same Op.
38702 SDValue Ops[2] = {DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT)};
38703 for (int i = 0; i < 4; ++i) {
38704 assert(ScaledMask[i] >= -1 && "Illegal shuffle sentinel value");
38705 if (ScaledMask[i] < 0)
38706 continue;
38707
38708 SDValue Op = ScaledMask[i] >= 4 ? V2 : V1;
38709 unsigned OpIndex = i / 2;
38710 if (Ops[OpIndex].isUndef())
38711 Ops[OpIndex] = Op;
38712 else if (Ops[OpIndex] != Op)
38713 return SDValue();
38714
38715 PermMask[i] = ScaledMask[i] % 4;
38716 }
38717
38718 return DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
38719 CanonicalizeShuffleInput(ShuffleVT, Ops[0]),
38720 CanonicalizeShuffleInput(ShuffleVT, Ops[1]),
38721 getV4X86ShuffleImm8ForMask(PermMask, DL, DAG));
38722 };
38723
38724 // FIXME: Is there a better way to do this? is256BitLaneRepeatedShuffleMask
38725 // doesn't work because our mask is for 128 bits and we don't have an MVT
38726 // to match that.
38727 bool PreferPERMQ = UnaryShuffle && isUndefOrInRange(ScaledMask[0], 0, 2) &&
38728 isUndefOrInRange(ScaledMask[1], 0, 2) &&
38729 isUndefOrInRange(ScaledMask[2], 2, 4) &&
38730 isUndefOrInRange(ScaledMask[3], 2, 4) &&
38731 (ScaledMask[0] < 0 || ScaledMask[2] < 0 ||
38732 ScaledMask[0] == (ScaledMask[2] % 2)) &&
38733 (ScaledMask[1] < 0 || ScaledMask[3] < 0 ||
38734 ScaledMask[1] == (ScaledMask[3] % 2));
38735
38736 if (!isAnyZero(ScaledMask) && !PreferPERMQ) {
38737 if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128)
38738 return SDValue(); // Nothing to do!
38739 MVT ShuffleVT = (FloatDomain ? MVT::v8f64 : MVT::v8i64);
38740 if (SDValue V = MatchSHUF128(ShuffleVT, DL, ScaledMask, V1, V2, DAG))
38741 return DAG.getBitcast(RootVT, V);
38742 }
38743 }
38744
38745 // Handle 128-bit lane shuffles of 256-bit vectors.
38746 if (RootVT.is256BitVector() && NumBaseMaskElts == 2) {
38747 // If the upper half is zeroable, then an extract+insert is more optimal
38748 // than using X86ISD::VPERM2X128. The insertion is free, even if it has to
38749 // zero the upper half.
38750 if (isUndefOrZero(Mask[1])) {
38751 if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
38752 return SDValue(); // Nothing to do!
38753 assert(isInRange(Mask[0], 0, 2) && "Unexpected lane shuffle");
38754 Res = CanonicalizeShuffleInput(RootVT, V1);
38755 Res = extract128BitVector(Res, Mask[0] * (NumRootElts / 2), DAG, DL);
38756 return widenSubVector(Res, Mask[1] == SM_SentinelZero, Subtarget, DAG, DL,
38757 256);
38758 }
38759
38760 // If we're inserting the low subvector, an insert-subvector 'concat'
38761 // pattern is quicker than VPERM2X128.
38762 // TODO: Add AVX2 support instead of VPERMQ/VPERMPD.
38763 if (BaseMask[0] == 0 && (BaseMask[1] == 0 || BaseMask[1] == 2) &&
38764 !Subtarget.hasAVX2()) {
38765 if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR)
38766 return SDValue(); // Nothing to do!
38767 SDValue Lo = CanonicalizeShuffleInput(RootVT, V1);
38768 SDValue Hi = CanonicalizeShuffleInput(RootVT, BaseMask[1] == 0 ? V1 : V2);
38769 Hi = extractSubVector(Hi, 0, DAG, DL, 128);
38770 return insertSubVector(Lo, Hi, NumRootElts / 2, DAG, DL, 128);
38771 }
38772
38773 if (Depth == 0 && Root.getOpcode() == X86ISD::VPERM2X128)
38774 return SDValue(); // Nothing to do!
38775
38776 // If we have AVX2, prefer to use VPERMQ/VPERMPD for unary shuffles unless
38777 // we need to use the zeroing feature.
38778 // Prefer blends for sequential shuffles unless we are optimizing for size.
38779 if (UnaryShuffle &&
38780 !(Subtarget.hasAVX2() && isUndefOrInRange(Mask, 0, 2)) &&
38781 (OptForSize || !isSequentialOrUndefOrZeroInRange(Mask, 0, 2, 0))) {
38782 unsigned PermMask = 0;
38783 PermMask |= ((Mask[0] < 0 ? 0x8 : (Mask[0] & 1)) << 0);
38784 PermMask |= ((Mask[1] < 0 ? 0x8 : (Mask[1] & 1)) << 4);
38785 return DAG.getNode(
38786 X86ISD::VPERM2X128, DL, RootVT, CanonicalizeShuffleInput(RootVT, V1),
38787 DAG.getUNDEF(RootVT), DAG.getTargetConstant(PermMask, DL, MVT::i8));
38788 }
38789
38790 if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128)
38791 return SDValue(); // Nothing to do!
38792
38793 // TODO - handle AVX512VL cases with X86ISD::SHUF128.
38794 if (!UnaryShuffle && !IsMaskedShuffle) {
38795 assert(llvm::all_of(Mask, [](int M) { return 0 <= M && M < 4; }) &&
38796 "Unexpected shuffle sentinel value");
38797 // Prefer blends to X86ISD::VPERM2X128.
38798 if (!((Mask[0] == 0 && Mask[1] == 3) || (Mask[0] == 2 && Mask[1] == 1))) {
38799 unsigned PermMask = 0;
38800 PermMask |= ((Mask[0] & 3) << 0);
38801 PermMask |= ((Mask[1] & 3) << 4);
38802 SDValue LHS = isInRange(Mask[0], 0, 2) ? V1 : V2;
38803 SDValue RHS = isInRange(Mask[1], 0, 2) ? V1 : V2;
38804 return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT,
38805 CanonicalizeShuffleInput(RootVT, LHS),
38806 CanonicalizeShuffleInput(RootVT, RHS),
38807 DAG.getTargetConstant(PermMask, DL, MVT::i8));
38808 }
38809 }
38810 }
38811
38812 // For masks that have been widened to 128-bit elements or more,
38813 // narrow back down to 64-bit elements.
38814 if (BaseMaskEltSizeInBits > 64) {
38815 assert((BaseMaskEltSizeInBits % 64) == 0 && "Illegal mask size");
38816 int MaskScale = BaseMaskEltSizeInBits / 64;
38817 SmallVector<int, 64> ScaledMask;
38818 narrowShuffleMaskElts(MaskScale, Mask, ScaledMask);
38819 Mask = std::move(ScaledMask);
38820 }
38821
38822 // For masked shuffles, we're trying to match the root width for better
38823 // writemask folding, attempt to scale the mask.
38824 // TODO - variable shuffles might need this to be widened again.
38825 if (IsMaskedShuffle && NumRootElts > Mask.size()) {
38826 assert((NumRootElts % Mask.size()) == 0 && "Illegal mask size");
38827 int MaskScale = NumRootElts / Mask.size();
38828 SmallVector<int, 64> ScaledMask;
38829 narrowShuffleMaskElts(MaskScale, Mask, ScaledMask);
38830 Mask = std::move(ScaledMask);
38831 }
38832
38833 unsigned NumMaskElts = Mask.size();
38834 unsigned MaskEltSizeInBits = RootSizeInBits / NumMaskElts;
38835 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
38836
38837 // Determine the effective mask value type.
38838 FloatDomain &= (32 <= MaskEltSizeInBits);
38839 MVT MaskVT = FloatDomain ? MVT::getFloatingPointVT(MaskEltSizeInBits)
38840 : MVT::getIntegerVT(MaskEltSizeInBits);
38841 MaskVT = MVT::getVectorVT(MaskVT, NumMaskElts);
38842
38843 // Only allow legal mask types.
38844 if (!TLI.isTypeLegal(MaskVT))
38845 return SDValue();
38846
38847 // Attempt to match the mask against known shuffle patterns.
38848 MVT ShuffleSrcVT, ShuffleVT;
38849 unsigned Shuffle, PermuteImm;
38850
38851 // Which shuffle domains are permitted?
38852 // Permit domain crossing at higher combine depths.
38853 // TODO: Should we indicate which domain is preferred if both are allowed?
38854 bool AllowFloatDomain = FloatDomain || (Depth >= 3);
38855 bool AllowIntDomain = (!FloatDomain || (Depth >= 3)) && Subtarget.hasSSE2() &&
38856 (!MaskVT.is256BitVector() || Subtarget.hasAVX2());
38857
38858 // Determine zeroable mask elements.
38859 APInt KnownUndef, KnownZero;
38860 resolveZeroablesFromTargetShuffle(Mask, KnownUndef, KnownZero);
38861 APInt Zeroable = KnownUndef | KnownZero;
38862
38863 if (UnaryShuffle) {
38864 // Attempt to match against broadcast-from-vector.
38865 // Limit AVX1 to cases where we're loading+broadcasting a scalar element.
38866 if ((Subtarget.hasAVX2() ||
38867 (Subtarget.hasAVX() && 32 <= MaskEltSizeInBits)) &&
38868 (!IsMaskedShuffle || NumRootElts == NumMaskElts)) {
38869 if (isUndefOrEqual(Mask, 0)) {
38870 if (V1.getValueType() == MaskVT &&
38871 V1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
38872 X86::mayFoldLoad(V1.getOperand(0), Subtarget)) {
38873 if (Depth == 0 && Root.getOpcode() == X86ISD::VBROADCAST)
38874 return SDValue(); // Nothing to do!
38875 Res = V1.getOperand(0);
38876 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
38877 return DAG.getBitcast(RootVT, Res);
38878 }
38879 if (Subtarget.hasAVX2()) {
38880 if (Depth == 0 && Root.getOpcode() == X86ISD::VBROADCAST)
38881 return SDValue(); // Nothing to do!
38882 Res = CanonicalizeShuffleInput(MaskVT, V1);
38883 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
38884 return DAG.getBitcast(RootVT, Res);
38885 }
38886 }
38887 }
38888
38889 if (matchUnaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, V1,
38890 DAG, Subtarget, Shuffle, ShuffleSrcVT, ShuffleVT) &&
38891 (!IsMaskedShuffle ||
38892 (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38893 if (Depth == 0 && Root.getOpcode() == Shuffle)
38894 return SDValue(); // Nothing to do!
38895 Res = CanonicalizeShuffleInput(ShuffleSrcVT, V1);
38896 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res);
38897 return DAG.getBitcast(RootVT, Res);
38898 }
38899
38900 if (matchUnaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
38901 AllowIntDomain, DAG, Subtarget, Shuffle, ShuffleVT,
38902 PermuteImm) &&
38903 (!IsMaskedShuffle ||
38904 (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38905 if (Depth == 0 && Root.getOpcode() == Shuffle)
38906 return SDValue(); // Nothing to do!
38907 Res = CanonicalizeShuffleInput(ShuffleVT, V1);
38908 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res,
38909 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38910 return DAG.getBitcast(RootVT, Res);
38911 }
38912 }
38913
38914 // Attempt to combine to INSERTPS, but only if the inserted element has come
38915 // from a scalar.
38916 // TODO: Handle other insertions here as well?
38917 if (!UnaryShuffle && AllowFloatDomain && RootSizeInBits == 128 &&
38918 Subtarget.hasSSE41() &&
38919 !isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}, DAG)) {
38920 if (MaskEltSizeInBits == 32) {
38921 SDValue SrcV1 = V1, SrcV2 = V2;
38922 if (matchShuffleAsInsertPS(SrcV1, SrcV2, PermuteImm, Zeroable, Mask,
38923 DAG) &&
38924 SrcV2.getOpcode() == ISD::SCALAR_TO_VECTOR) {
38925 if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTPS)
38926 return SDValue(); // Nothing to do!
38927 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,
38928 CanonicalizeShuffleInput(MVT::v4f32, SrcV1),
38929 CanonicalizeShuffleInput(MVT::v4f32, SrcV2),
38930 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38931 return DAG.getBitcast(RootVT, Res);
38932 }
38933 }
38934 if (MaskEltSizeInBits == 64 &&
38935 isTargetShuffleEquivalent(MaskVT, Mask, {0, 2}, DAG) &&
38936 V2.getOpcode() == ISD::SCALAR_TO_VECTOR &&
38937 V2.getScalarValueSizeInBits() <= 32) {
38938 if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTPS)
38939 return SDValue(); // Nothing to do!
38940 PermuteImm = (/*DstIdx*/ 2 << 4) | (/*SrcIdx*/ 0 << 0);
38941 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,
38942 CanonicalizeShuffleInput(MVT::v4f32, V1),
38943 CanonicalizeShuffleInput(MVT::v4f32, V2),
38944 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38945 return DAG.getBitcast(RootVT, Res);
38946 }
38947 }
38948
38949 SDValue NewV1 = V1; // Save operands in case early exit happens.
38950 SDValue NewV2 = V2;
38951 if (matchBinaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, NewV1,
38952 NewV2, DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
38953 ShuffleVT, UnaryShuffle) &&
38954 (!IsMaskedShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38955 if (Depth == 0 && Root.getOpcode() == Shuffle)
38956 return SDValue(); // Nothing to do!
38957 NewV1 = CanonicalizeShuffleInput(ShuffleSrcVT, NewV1);
38958 NewV2 = CanonicalizeShuffleInput(ShuffleSrcVT, NewV2);
38959 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2);
38960 return DAG.getBitcast(RootVT, Res);
38961 }
38962
38963 NewV1 = V1; // Save operands in case early exit happens.
38964 NewV2 = V2;
38965 if (matchBinaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
38966 AllowIntDomain, NewV1, NewV2, DL, DAG,
38967 Subtarget, Shuffle, ShuffleVT, PermuteImm) &&
38968 (!IsMaskedShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
38969 if (Depth == 0 && Root.getOpcode() == Shuffle)
38970 return SDValue(); // Nothing to do!
38971 NewV1 = CanonicalizeShuffleInput(ShuffleVT, NewV1);
38972 NewV2 = CanonicalizeShuffleInput(ShuffleVT, NewV2);
38973 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2,
38974 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
38975 return DAG.getBitcast(RootVT, Res);
38976 }
38977
38978 // Typically from here on, we need an integer version of MaskVT.
38979 MVT IntMaskVT = MVT::getIntegerVT(MaskEltSizeInBits);
38980 IntMaskVT = MVT::getVectorVT(IntMaskVT, NumMaskElts);
38981
38982 // Annoyingly, SSE4A instructions don't map into the above match helpers.
38983 if (Subtarget.hasSSE4A() && AllowIntDomain && RootSizeInBits == 128) {
38984 uint64_t BitLen, BitIdx;
38985 if (matchShuffleAsEXTRQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx,
38986 Zeroable)) {
38987 if (Depth == 0 && Root.getOpcode() == X86ISD::EXTRQI)
38988 return SDValue(); // Nothing to do!
38989 V1 = CanonicalizeShuffleInput(IntMaskVT, V1);
38990 Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1,
38991 DAG.getTargetConstant(BitLen, DL, MVT::i8),
38992 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
38993 return DAG.getBitcast(RootVT, Res);
38994 }
38995
38996 if (matchShuffleAsINSERTQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx)) {
38997 if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTQI)
38998 return SDValue(); // Nothing to do!
38999 V1 = CanonicalizeShuffleInput(IntMaskVT, V1);
39000 V2 = CanonicalizeShuffleInput(IntMaskVT, V2);
39001 Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2,
39002 DAG.getTargetConstant(BitLen, DL, MVT::i8),
39003 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
39004 return DAG.getBitcast(RootVT, Res);
39005 }
39006 }
39007
39008 // Match shuffle against TRUNCATE patterns.
39009 if (AllowIntDomain && MaskEltSizeInBits < 64 && Subtarget.hasAVX512()) {
39010 // Match against a VTRUNC instruction, accounting for src/dst sizes.
39011 if (matchShuffleAsVTRUNC(ShuffleSrcVT, ShuffleVT, IntMaskVT, Mask, Zeroable,
39012 Subtarget)) {
39013 bool IsTRUNCATE = ShuffleVT.getVectorNumElements() ==
39014 ShuffleSrcVT.getVectorNumElements();
39015 unsigned Opc =
39016 IsTRUNCATE ? (unsigned)ISD::TRUNCATE : (unsigned)X86ISD::VTRUNC;
39017 if (Depth == 0 && Root.getOpcode() == Opc)
39018 return SDValue(); // Nothing to do!
39019 V1 = CanonicalizeShuffleInput(ShuffleSrcVT, V1);
39020 Res = DAG.getNode(Opc, DL, ShuffleVT, V1);
39021 if (ShuffleVT.getSizeInBits() < RootSizeInBits)
39022 Res = widenSubVector(Res, true, Subtarget, DAG, DL, RootSizeInBits);
39023 return DAG.getBitcast(RootVT, Res);
39024 }
39025
39026 // Do we need a more general binary truncation pattern?
39027 if (RootSizeInBits < 512 &&
39028 ((RootVT.is256BitVector() && Subtarget.useAVX512Regs()) ||
39029 (RootVT.is128BitVector() && Subtarget.hasVLX())) &&
39030 (MaskEltSizeInBits > 8 || Subtarget.hasBWI()) &&
39031 isSequentialOrUndefInRange(Mask, 0, NumMaskElts, 0, 2)) {
39032 // Bail if this was already a truncation or PACK node.
39033 // We sometimes fail to match PACK if we demand known undef elements.
39034 if (Depth == 0 && (Root.getOpcode() == ISD::TRUNCATE ||
39035 Root.getOpcode() == X86ISD::PACKSS ||
39036 Root.getOpcode() == X86ISD::PACKUS))
39037 return SDValue(); // Nothing to do!
39038 ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);
39039 ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts / 2);
39040 V1 = CanonicalizeShuffleInput(ShuffleSrcVT, V1);
39041 V2 = CanonicalizeShuffleInput(ShuffleSrcVT, V2);
39042 ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);
39043 ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts);
39044 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShuffleSrcVT, V1, V2);
39045 Res = DAG.getNode(ISD::TRUNCATE, DL, IntMaskVT, Res);
39046 return DAG.getBitcast(RootVT, Res);
39047 }
39048 }
39049
39050 // Don't try to re-form single instruction chains under any circumstances now
39051 // that we've done encoding canonicalization for them.
39052 if (Depth < 1)
39053 return SDValue();
39054
39055 // Depth threshold above which we can efficiently use variable mask shuffles.
39056 int VariableCrossLaneShuffleDepth =
39057 Subtarget.hasFastVariableCrossLaneShuffle() ? 1 : 2;
39058 int VariablePerLaneShuffleDepth =
39059 Subtarget.hasFastVariablePerLaneShuffle() ? 1 : 2;
39060 AllowVariableCrossLaneMask &=
39061 (Depth >= VariableCrossLaneShuffleDepth) || HasVariableMask;
39062 AllowVariablePerLaneMask &=
39063 (Depth >= VariablePerLaneShuffleDepth) || HasVariableMask;
39064 // VPERMI2W/VPERMI2B are 3 uops on Skylake and Icelake so we require a
39065 // higher depth before combining them.
39066 bool AllowBWIVPERMV3 =
39067 (Depth >= (VariableCrossLaneShuffleDepth + 2) || HasVariableMask);
39068
39069 bool MaskContainsZeros = isAnyZero(Mask);
39070
39071 if (is128BitLaneCrossingShuffleMask(MaskVT, Mask)) {
39072 // If we have a single input lane-crossing shuffle then lower to VPERMV.
39073 if (UnaryShuffle && AllowVariableCrossLaneMask && !MaskContainsZeros) {
39074 if (Subtarget.hasAVX2() &&
39075 (MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) {
39076 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
39077 Res = CanonicalizeShuffleInput(MaskVT, V1);
39078 Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res);
39079 return DAG.getBitcast(RootVT, Res);
39080 }
39081 // AVX512 variants (non-VLX will pad to 512-bit shuffles).
39082 if ((Subtarget.hasAVX512() &&
39083 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
39084 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
39085 (Subtarget.hasBWI() &&
39086 (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||
39087 (Subtarget.hasVBMI() &&
39088 (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8))) {
39089 V1 = CanonicalizeShuffleInput(MaskVT, V1);
39090 V2 = DAG.getUNDEF(MaskVT);
39091 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
39092 return DAG.getBitcast(RootVT, Res);
39093 }
39094 }
39095
39096 // Lower a unary+zero lane-crossing shuffle as VPERMV3 with a zero
39097 // vector as the second source (non-VLX will pad to 512-bit shuffles).
39098 if (UnaryShuffle && AllowVariableCrossLaneMask &&
39099 ((Subtarget.hasAVX512() &&
39100 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
39101 MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||
39102 MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32 ||
39103 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
39104 (Subtarget.hasBWI() && AllowBWIVPERMV3 &&
39105 (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||
39106 (Subtarget.hasVBMI() && AllowBWIVPERMV3 &&
39107 (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) {
39108 // Adjust shuffle mask - replace SM_SentinelZero with second source index.
39109 for (unsigned i = 0; i != NumMaskElts; ++i)
39110 if (Mask[i] == SM_SentinelZero)
39111 Mask[i] = NumMaskElts + i;
39112 V1 = CanonicalizeShuffleInput(MaskVT, V1);
39113 V2 = getZeroVector(MaskVT, Subtarget, DAG, DL);
39114 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
39115 return DAG.getBitcast(RootVT, Res);
39116 }
39117
39118 // If that failed and either input is extracted then try to combine as a
39119 // shuffle with the larger type.
39120 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
39121 Inputs, Root, BaseMask, Depth, HasVariableMask,
39122 AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG,
39123 Subtarget))
39124 return WideShuffle;
39125
39126 // If we have a dual input lane-crossing shuffle then lower to VPERMV3,
39127 // (non-VLX will pad to 512-bit shuffles).
39128 if (AllowVariableCrossLaneMask && !MaskContainsZeros &&
39129 ((Subtarget.hasAVX512() &&
39130 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
39131 MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||
39132 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32 ||
39133 MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) ||
39134 (Subtarget.hasBWI() && AllowBWIVPERMV3 &&
39135 (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||
39136 (Subtarget.hasVBMI() && AllowBWIVPERMV3 &&
39137 (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) {
39138 V1 = CanonicalizeShuffleInput(MaskVT, V1);
39139 V2 = CanonicalizeShuffleInput(MaskVT, V2);
39140 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
39141 return DAG.getBitcast(RootVT, Res);
39142 }
39143 return SDValue();
39144 }
39145
39146 // See if we can combine a single input shuffle with zeros to a bit-mask,
39147 // which is much simpler than any shuffle.
39148 if (UnaryShuffle && MaskContainsZeros && AllowVariablePerLaneMask &&
39149 isSequentialOrUndefOrZeroInRange(Mask, 0, NumMaskElts, 0) &&
39150 TLI.isTypeLegal(MaskVT)) {
39151 APInt Zero = APInt::getZero(MaskEltSizeInBits);
39152 APInt AllOnes = APInt::getAllOnes(MaskEltSizeInBits);
39153 APInt UndefElts(NumMaskElts, 0);
39154 SmallVector<APInt, 64> EltBits(NumMaskElts, Zero);
39155 for (unsigned i = 0; i != NumMaskElts; ++i) {
39156 int M = Mask[i];
39157 if (M == SM_SentinelUndef) {
39158 UndefElts.setBit(i);
39159 continue;
39160 }
39161 if (M == SM_SentinelZero)
39162 continue;
39163 EltBits[i] = AllOnes;
39164 }
39165 SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL);
39166 Res = CanonicalizeShuffleInput(MaskVT, V1);
39167 unsigned AndOpcode =
39168 MaskVT.isFloatingPoint() ? unsigned(X86ISD::FAND) : unsigned(ISD::AND);
39169 Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask);
39170 return DAG.getBitcast(RootVT, Res);
39171 }
39172
39173 // If we have a single input shuffle with different shuffle patterns in the
39174 // the 128-bit lanes use the variable mask to VPERMILPS.
39175 // TODO Combine other mask types at higher depths.
39176 if (UnaryShuffle && AllowVariablePerLaneMask && !MaskContainsZeros &&
39177 ((MaskVT == MVT::v8f32 && Subtarget.hasAVX()) ||
39178 (MaskVT == MVT::v16f32 && Subtarget.hasAVX512()))) {
39179 SmallVector<SDValue, 16> VPermIdx;
39180 for (int M : Mask) {
39181 SDValue Idx =
39182 M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32);
39183 VPermIdx.push_back(Idx);
39184 }
39185 SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx);
39186 Res = CanonicalizeShuffleInput(MaskVT, V1);
39187 Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask);
39188 return DAG.getBitcast(RootVT, Res);
39189 }
39190
39191 // With XOP, binary shuffles of 128/256-bit floating point vectors can combine
39192 // to VPERMIL2PD/VPERMIL2PS.
39193 if (AllowVariablePerLaneMask && Subtarget.hasXOP() &&
39194 (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v4f32 ||
39195 MaskVT == MVT::v8f32)) {
39196 // VPERMIL2 Operation.
39197 // Bits[3] - Match Bit.
39198 // Bits[2:1] - (Per Lane) PD Shuffle Mask.
39199 // Bits[2:0] - (Per Lane) PS Shuffle Mask.
39200 unsigned NumLanes = MaskVT.getSizeInBits() / 128;
39201 unsigned NumEltsPerLane = NumMaskElts / NumLanes;
39202 SmallVector<int, 8> VPerm2Idx;
39203 unsigned M2ZImm = 0;
39204 for (int M : Mask) {
39205 if (M == SM_SentinelUndef) {
39206 VPerm2Idx.push_back(-1);
39207 continue;
39208 }
39209 if (M == SM_SentinelZero) {
39210 M2ZImm = 2;
39211 VPerm2Idx.push_back(8);
39212 continue;
39213 }
39214 int Index = (M % NumEltsPerLane) + ((M / NumMaskElts) * NumEltsPerLane);
39215 Index = (MaskVT.getScalarSizeInBits() == 64 ? Index << 1 : Index);
39216 VPerm2Idx.push_back(Index);
39217 }
39218 V1 = CanonicalizeShuffleInput(MaskVT, V1);
39219 V2 = CanonicalizeShuffleInput(MaskVT, V2);
39220 SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
39221 Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp,
39222 DAG.getTargetConstant(M2ZImm, DL, MVT::i8));
39223 return DAG.getBitcast(RootVT, Res);
39224 }
39225
39226 // If we have 3 or more shuffle instructions or a chain involving a variable
39227 // mask, we can replace them with a single PSHUFB instruction profitably.
39228 // Intel's manuals suggest only using PSHUFB if doing so replacing 5
39229 // instructions, but in practice PSHUFB tends to be *very* fast so we're
39230 // more aggressive.
39231 if (UnaryShuffle && AllowVariablePerLaneMask &&
39232 ((RootVT.is128BitVector() && Subtarget.hasSSSE3()) ||
39233 (RootVT.is256BitVector() && Subtarget.hasAVX2()) ||
39234 (RootVT.is512BitVector() && Subtarget.hasBWI()))) {
39235 SmallVector<SDValue, 16> PSHUFBMask;
39236 int NumBytes = RootVT.getSizeInBits() / 8;
39237 int Ratio = NumBytes / NumMaskElts;
39238 for (int i = 0; i < NumBytes; ++i) {
39239 int M = Mask[i / Ratio];
39240 if (M == SM_SentinelUndef) {
39241 PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8));
39242 continue;
39243 }
39244 if (M == SM_SentinelZero) {
39245 PSHUFBMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
39246 continue;
39247 }
39248 M = Ratio * M + i % Ratio;
39249 assert((M / 16) == (i / 16) && "Lane crossing detected");
39250 PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8));
39251 }
39252 MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes);
39253 Res = CanonicalizeShuffleInput(ByteVT, V1);
39254 SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask);
39255 Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp);
39256 return DAG.getBitcast(RootVT, Res);
39257 }
39258
39259 // With XOP, if we have a 128-bit binary input shuffle we can always combine
39260 // to VPPERM. We match the depth requirement of PSHUFB - VPPERM is never
39261 // slower than PSHUFB on targets that support both.
39262 if (AllowVariablePerLaneMask && RootVT.is128BitVector() &&
39263 Subtarget.hasXOP()) {
39264 // VPPERM Mask Operation
39265 // Bits[4:0] - Byte Index (0 - 31)
39266 // Bits[7:5] - Permute Operation (0 - Source byte, 4 - ZERO)
39267 SmallVector<SDValue, 16> VPPERMMask;
39268 int NumBytes = 16;
39269 int Ratio = NumBytes / NumMaskElts;
39270 for (int i = 0; i < NumBytes; ++i) {
39271 int M = Mask[i / Ratio];
39272 if (M == SM_SentinelUndef) {
39273 VPPERMMask.push_back(DAG.getUNDEF(MVT::i8));
39274 continue;
39275 }
39276 if (M == SM_SentinelZero) {
39277 VPPERMMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));
39278 continue;
39279 }
39280 M = Ratio * M + i % Ratio;
39281 VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8));
39282 }
39283 MVT ByteVT = MVT::v16i8;
39284 V1 = CanonicalizeShuffleInput(ByteVT, V1);
39285 V2 = CanonicalizeShuffleInput(ByteVT, V2);
39286 SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask);
39287 Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp);
39288 return DAG.getBitcast(RootVT, Res);
39289 }
39290
39291 // If that failed and either input is extracted then try to combine as a
39292 // shuffle with the larger type.
39293 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
39294 Inputs, Root, BaseMask, Depth, HasVariableMask,
39295 AllowVariableCrossLaneMask, AllowVariablePerLaneMask, DAG, Subtarget))
39296 return WideShuffle;
39297
39298 // If we have a dual input shuffle then lower to VPERMV3,
39299 // (non-VLX will pad to 512-bit shuffles)
39300 if (!UnaryShuffle && AllowVariablePerLaneMask && !MaskContainsZeros &&
39301 ((Subtarget.hasAVX512() &&
39302 (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v8f64 ||
39303 MaskVT == MVT::v2i64 || MaskVT == MVT::v4i64 || MaskVT == MVT::v8i64 ||
39304 MaskVT == MVT::v4f32 || MaskVT == MVT::v4i32 || MaskVT == MVT::v8f32 ||
39305 MaskVT == MVT::v8i32 || MaskVT == MVT::v16f32 ||
39306 MaskVT == MVT::v16i32)) ||
39307 (Subtarget.hasBWI() && AllowBWIVPERMV3 &&
39308 (MaskVT == MVT::v8i16 || MaskVT == MVT::v16i16 ||
39309 MaskVT == MVT::v32i16)) ||
39310 (Subtarget.hasVBMI() && AllowBWIVPERMV3 &&
39311 (MaskVT == MVT::v16i8 || MaskVT == MVT::v32i8 ||
39312 MaskVT == MVT::v64i8)))) {
39313 V1 = CanonicalizeShuffleInput(MaskVT, V1);
39314 V2 = CanonicalizeShuffleInput(MaskVT, V2);
39315 Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG);
39316 return DAG.getBitcast(RootVT, Res);
39317 }
39318
39319 // Failed to find any combines.
39320 return SDValue();
39321}
39322
39323// Combine an arbitrary chain of shuffles + extract_subvectors into a single
39324// instruction if possible.
39325//
39326// Wrapper for combineX86ShuffleChain that extends the shuffle mask to a larger
39327// type size to attempt to combine:
39328// shuffle(extract_subvector(x,c1),extract_subvector(y,c2),m1)
39329// -->
39330// extract_subvector(shuffle(x,y,m2),0)
39331static SDValue combineX86ShuffleChainWithExtract(
39332 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
39333 bool HasVariableMask, bool AllowVariableCrossLaneMask,
39334 bool AllowVariablePerLaneMask, SelectionDAG &DAG,
39335 const X86Subtarget &Subtarget) {
39336 unsigned NumMaskElts = BaseMask.size();
39337 unsigned NumInputs = Inputs.size();
39338 if (NumInputs == 0)
39339 return SDValue();
39340
39341 EVT RootVT = Root.getValueType();
39342 unsigned RootSizeInBits = RootVT.getSizeInBits();
39343 unsigned RootEltSizeInBits = RootSizeInBits / NumMaskElts;
39344 assert((RootSizeInBits % NumMaskElts) == 0 && "Unexpected root shuffle mask");
39345
39346 // Peek through extract_subvector to find widest legal vector.
39347 // TODO: Handle ISD::TRUNCATE
39348 unsigned WideSizeInBits = RootSizeInBits;
39349 for (unsigned I = 0; I != NumInputs; ++I) {
39350 SDValue Input = peekThroughBitcasts(Inputs[I]);
39351 while (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR)
39352 Input = peekThroughBitcasts(Input.getOperand(0));
39353 if (DAG.getTargetLoweringInfo().isTypeLegal(Input.getValueType()) &&
39354 WideSizeInBits < Input.getValueSizeInBits())
39355 WideSizeInBits = Input.getValueSizeInBits();
39356 }
39357
39358 // Bail if we fail to find a source larger than the existing root.
39359 unsigned Scale = WideSizeInBits / RootSizeInBits;
39360 if (WideSizeInBits <= RootSizeInBits ||
39361 (WideSizeInBits % RootSizeInBits) != 0)
39362 return SDValue();
39363
39364 // Create new mask for larger type.
39365 SmallVector<int, 64> WideMask(BaseMask);
39366 for (int &M : WideMask) {
39367 if (M < 0)
39368 continue;
39369 M = (M % NumMaskElts) + ((M / NumMaskElts) * Scale * NumMaskElts);
39370 }
39371 WideMask.append((Scale - 1) * NumMaskElts, SM_SentinelUndef);
39372
39373 // Attempt to peek through inputs and adjust mask when we extract from an
39374 // upper subvector.
39375 int AdjustedMasks = 0;
39376 SmallVector<SDValue, 4> WideInputs(Inputs);
39377 for (unsigned I = 0; I != NumInputs; ++I) {
39378 SDValue &Input = WideInputs[I];
39379 Input = peekThroughBitcasts(Input);
39380 while (Input.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
39381 Input.getOperand(0).getValueSizeInBits() <= WideSizeInBits) {
39382 uint64_t Idx = Input.getConstantOperandVal(1);
39383 if (Idx != 0) {
39384 ++AdjustedMasks;
39385 unsigned InputEltSizeInBits = Input.getScalarValueSizeInBits();
39386 Idx = (Idx * InputEltSizeInBits) / RootEltSizeInBits;
39387
39388 int lo = I * WideMask.size();
39389 int hi = (I + 1) * WideMask.size();
39390 for (int &M : WideMask)
39391 if (lo <= M && M < hi)
39392 M += Idx;
39393 }
39394 Input = peekThroughBitcasts(Input.getOperand(0));
39395 }
39396 }
39397
39398 // Remove unused/repeated shuffle source ops.
39399 resolveTargetShuffleInputsAndMask(WideInputs, WideMask);
39400 assert(!WideInputs.empty() && "Shuffle with no inputs detected");
39401
39402 // Bail if we're always extracting from the lowest subvectors,
39403 // combineX86ShuffleChain should match this for the current width, or the
39404 // shuffle still references too many inputs.
39405 if (AdjustedMasks == 0 || WideInputs.size() > 2)
39406 return SDValue();
39407
39408 // Minor canonicalization of the accumulated shuffle mask to make it easier
39409 // to match below. All this does is detect masks with sequential pairs of
39410 // elements, and shrink them to the half-width mask. It does this in a loop
39411 // so it will reduce the size of the mask to the minimal width mask which
39412 // performs an equivalent shuffle.
39413 while (WideMask.size() > 1) {
39414 SmallVector<int, 64> WidenedMask;
39415 if (!canWidenShuffleElements(WideMask, WidenedMask))
39416 break;
39417 WideMask = std::move(WidenedMask);
39418 }
39419
39420 // Canonicalization of binary shuffle masks to improve pattern matching by
39421 // commuting the inputs.
39422 if (WideInputs.size() == 2 && canonicalizeShuffleMaskWithCommute(WideMask)) {
39423 ShuffleVectorSDNode::commuteMask(WideMask);
39424 std::swap(WideInputs[0], WideInputs[1]);
39425 }
39426
39427 // Increase depth for every upper subvector we've peeked through.
39428 Depth += AdjustedMasks;
39429
39430 // Attempt to combine wider chain.
39431 // TODO: Can we use a better Root?
39432 SDValue WideRoot = WideInputs.front().getValueSizeInBits() >
39433 WideInputs.back().getValueSizeInBits()
39434 ? WideInputs.front()
39435 : WideInputs.back();
39436 assert(WideRoot.getValueSizeInBits() == WideSizeInBits &&
39437 "WideRootSize mismatch");
39438
39439 if (SDValue WideShuffle =
39440 combineX86ShuffleChain(WideInputs, WideRoot, WideMask, Depth,
39441 HasVariableMask, AllowVariableCrossLaneMask,
39442 AllowVariablePerLaneMask, DAG, Subtarget)) {
39443 WideShuffle =
39444 extractSubVector(WideShuffle, 0, DAG, SDLoc(Root), RootSizeInBits);
39445 return DAG.getBitcast(RootVT, WideShuffle);
39446 }
39447
39448 return SDValue();
39449}
39450
39451// Canonicalize the combined shuffle mask chain with horizontal ops.
39452// NOTE: This may update the Ops and Mask.
39453static SDValue canonicalizeShuffleMaskWithHorizOp(
39454 MutableArrayRef<SDValue> Ops, MutableArrayRef<int> Mask,
39455 unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG,
39456 const X86Subtarget &Subtarget) {
39457 if (Mask.empty() || Ops.empty())
39458 return SDValue();
39459
39460 SmallVector<SDValue> BC;
39461 for (SDValue Op : Ops)
39462 BC.push_back(peekThroughBitcasts(Op));
39463
39464 // All ops must be the same horizop + type.
39465 SDValue BC0 = BC[0];
39466 EVT VT0 = BC0.getValueType();
39467 unsigned Opcode0 = BC0.getOpcode();
39468 if (VT0.getSizeInBits() != RootSizeInBits || llvm::any_of(BC, [&](SDValue V) {
39469 return V.getOpcode() != Opcode0 || V.getValueType() != VT0;
39470 }))
39471 return SDValue();
39472
39473 bool isHoriz = (Opcode0 == X86ISD::FHADD || Opcode0 == X86ISD::HADD ||
39474 Opcode0 == X86ISD::FHSUB || Opcode0 == X86ISD::HSUB);
39475 bool isPack = (Opcode0 == X86ISD::PACKSS || Opcode0 == X86ISD::PACKUS);
39476 if (!isHoriz && !isPack)
39477 return SDValue();
39478
39479 // Do all ops have a single use?
39480 bool OneUseOps = llvm::all_of(Ops, [](SDValue Op) {
39481 return Op.hasOneUse() &&
39482 peekThroughBitcasts(Op) == peekThroughOneUseBitcasts(Op);
39483 });
39484
39485 int NumElts = VT0.getVectorNumElements();
39486 int NumLanes = VT0.getSizeInBits() / 128;
39487 int NumEltsPerLane = NumElts / NumLanes;
39488 int NumHalfEltsPerLane = NumEltsPerLane / 2;
39489 MVT SrcVT = BC0.getOperand(0).getSimpleValueType();
39490 unsigned EltSizeInBits = RootSizeInBits / Mask.size();
39491
39492 if (NumEltsPerLane >= 4 &&
39493 (isPack || shouldUseHorizontalOp(Ops.size() == 1, DAG, Subtarget))) {
39494 SmallVector<int> LaneMask, ScaledMask;
39495 if (isRepeatedTargetShuffleMask(128, EltSizeInBits, Mask, LaneMask) &&
39496 scaleShuffleElements(LaneMask, 4, ScaledMask)) {
39497 // See if we can remove the shuffle by resorting the HOP chain so that
39498 // the HOP args are pre-shuffled.
39499 // TODO: Generalize to any sized/depth chain.
39500 // TODO: Add support for PACKSS/PACKUS.
39501 if (isHoriz) {
39502 // Attempt to find a HOP(HOP(X,Y),HOP(Z,W)) source operand.
39503 auto GetHOpSrc = [&](int M) {
39504 if (M == SM_SentinelUndef)
39505 return DAG.getUNDEF(VT0);
39506 if (M == SM_SentinelZero)
39507 return getZeroVector(VT0.getSimpleVT(), Subtarget, DAG, DL);
39508 SDValue Src0 = BC[M / 4];
39509 SDValue Src1 = Src0.getOperand((M % 4) >= 2);
39510 if (Src1.getOpcode() == Opcode0 && Src0->isOnlyUserOf(Src1.getNode()))
39511 return Src1.getOperand(M % 2);
39512 return SDValue();
39513 };
39514 SDValue M0 = GetHOpSrc(ScaledMask[0]);
39515 SDValue M1 = GetHOpSrc(ScaledMask[1]);
39516 SDValue M2 = GetHOpSrc(ScaledMask[2]);
39517 SDValue M3 = GetHOpSrc(ScaledMask[3]);
39518 if (M0 && M1 && M2 && M3) {
39519 SDValue LHS = DAG.getNode(Opcode0, DL, SrcVT, M0, M1);
39520 SDValue RHS = DAG.getNode(Opcode0, DL, SrcVT, M2, M3);
39521 return DAG.getNode(Opcode0, DL, VT0, LHS, RHS);
39522 }
39523 }
39524 // shuffle(hop(x,y),hop(z,w)) -> permute(hop(x,z)) etc.
39525 if (Ops.size() >= 2) {
39526 SDValue LHS, RHS;
39527 auto GetHOpSrc = [&](int M, int &OutM) {
39528 // TODO: Support SM_SentinelZero
39529 if (M < 0)
39530 return M == SM_SentinelUndef;
39531 SDValue Src = BC[M / 4].getOperand((M % 4) >= 2);
39532 if (!LHS || LHS == Src) {
39533 LHS = Src;
39534 OutM = (M % 2);
39535 return true;
39536 }
39537 if (!RHS || RHS == Src) {
39538 RHS = Src;
39539 OutM = (M % 2) + 2;
39540 return true;
39541 }
39542 return false;
39543 };
39544 int PostMask[4] = {-1, -1, -1, -1};
39545 if (GetHOpSrc(ScaledMask[0], PostMask[0]) &&
39546 GetHOpSrc(ScaledMask[1], PostMask[1]) &&
39547 GetHOpSrc(ScaledMask[2], PostMask[2]) &&
39548 GetHOpSrc(ScaledMask[3], PostMask[3])) {
39549 LHS = DAG.getBitcast(SrcVT, LHS);
39550 RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS);
39551 SDValue Res = DAG.getNode(Opcode0, DL, VT0, LHS, RHS);
39552 // Use SHUFPS for the permute so this will work on SSE2 targets,
39553 // shuffle combining and domain handling will simplify this later on.
39554 MVT ShuffleVT = MVT::getVectorVT(MVT::f32, RootSizeInBits / 32);
39555 Res = DAG.getBitcast(ShuffleVT, Res);
39556 return DAG.getNode(X86ISD::SHUFP, DL, ShuffleVT, Res, Res,
39557 getV4X86ShuffleImm8ForMask(PostMask, DL, DAG));
39558 }
39559 }
39560 }
39561 }
39562
39563 if (2 < Ops.size())
39564 return SDValue();
39565
39566 SDValue BC1 = BC[BC.size() - 1];
39567 if (Mask.size() == VT0.getVectorNumElements()) {
39568 // Canonicalize binary shuffles of horizontal ops that use the
39569 // same sources to an unary shuffle.
39570 // TODO: Try to perform this fold even if the shuffle remains.
39571 if (Ops.size() == 2) {
39572 auto ContainsOps = [](SDValue HOp, SDValue Op) {
39573 return Op == HOp.getOperand(0) || Op == HOp.getOperand(1);
39574 };
39575 // Commute if all BC0's ops are contained in BC1.
39576 if (ContainsOps(BC1, BC0.getOperand(0)) &&
39577 ContainsOps(BC1, BC0.getOperand(1))) {
39578 ShuffleVectorSDNode::commuteMask(Mask);
39579 std::swap(Ops[0], Ops[1]);
39580 std::swap(BC0, BC1);
39581 }
39582
39583 // If BC1 can be represented by BC0, then convert to unary shuffle.
39584 if (ContainsOps(BC0, BC1.getOperand(0)) &&
39585 ContainsOps(BC0, BC1.getOperand(1))) {
39586 for (int &M : Mask) {
39587 if (M < NumElts) // BC0 element or UNDEF/Zero sentinel.
39588 continue;
39589 int SubLane = ((M % NumEltsPerLane) >= NumHalfEltsPerLane) ? 1 : 0;
39590 M -= NumElts + (SubLane * NumHalfEltsPerLane);
39591 if (BC1.getOperand(SubLane) != BC0.getOperand(0))
39592 M += NumHalfEltsPerLane;
39593 }
39594 }
39595 }
39596
39597 // Canonicalize unary horizontal ops to only refer to lower halves.
39598 for (int i = 0; i != NumElts; ++i) {
39599 int &M = Mask[i];
39600 if (isUndefOrZero(M))
39601 continue;
39602 if (M < NumElts && BC0.getOperand(0) == BC0.getOperand(1) &&
39603 (M % NumEltsPerLane) >= NumHalfEltsPerLane)
39604 M -= NumHalfEltsPerLane;
39605 if (NumElts <= M && BC1.getOperand(0) == BC1.getOperand(1) &&
39606 (M % NumEltsPerLane) >= NumHalfEltsPerLane)
39607 M -= NumHalfEltsPerLane;
39608 }
39609 }
39610
39611 // Combine binary shuffle of 2 similar 'Horizontal' instructions into a
39612 // single instruction. Attempt to match a v2X64 repeating shuffle pattern that
39613 // represents the LHS/RHS inputs for the lower/upper halves.
39614 SmallVector<int, 16> TargetMask128, WideMask128;
39615 if (isRepeatedTargetShuffleMask(128, EltSizeInBits, Mask, TargetMask128) &&
39616 scaleShuffleElements(TargetMask128, 2, WideMask128)) {
39617 assert(isUndefOrZeroOrInRange(WideMask128, 0, 4) && "Illegal shuffle");
39618 bool SingleOp = (Ops.size() == 1);
39619 if (isPack || OneUseOps ||
39620 shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) {
39621 SDValue Lo = isInRange(WideMask128[0], 0, 2) ? BC0 : BC1;
39622 SDValue Hi = isInRange(WideMask128[1], 0, 2) ? BC0 : BC1;
39623 Lo = Lo.getOperand(WideMask128[0] & 1);
39624 Hi = Hi.getOperand(WideMask128[1] & 1);
39625 if (SingleOp) {
39626 SDValue Undef = DAG.getUNDEF(SrcVT);
39627 SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL);
39628 Lo = (WideMask128[0] == SM_SentinelZero ? Zero : Lo);
39629 Hi = (WideMask128[1] == SM_SentinelZero ? Zero : Hi);
39630 Lo = (WideMask128[0] == SM_SentinelUndef ? Undef : Lo);
39631 Hi = (WideMask128[1] == SM_SentinelUndef ? Undef : Hi);
39632 }
39633 return DAG.getNode(Opcode0, DL, VT0, Lo, Hi);
39634 }
39635 }
39636
39637 // If we are post-shuffling a 256-bit hop and not requiring the upper
39638 // elements, then try to narrow to a 128-bit hop directly.
39639 SmallVector<int, 16> WideMask64;
39640 if (Ops.size() == 1 && NumLanes == 2 &&
39641 scaleShuffleElements(Mask, 4, WideMask64) &&
39642 isUndefInRange(WideMask64, 2, 2)) {
39643 int M0 = WideMask64[0];
39644 int M1 = WideMask64[1];
39645 if (isInRange(M0, 0, 4) && isInRange(M1, 0, 4)) {
39646 MVT HalfVT = VT0.getSimpleVT().getHalfNumVectorElementsVT();
39647 unsigned Idx0 = (M0 & 2) ? (SrcVT.getVectorNumElements() / 2) : 0;
39648 unsigned Idx1 = (M1 & 2) ? (SrcVT.getVectorNumElements() / 2) : 0;
39649 SDValue V0 = extract128BitVector(BC[0].getOperand(M0 & 1), Idx0, DAG, DL);
39650 SDValue V1 = extract128BitVector(BC[0].getOperand(M1 & 1), Idx1, DAG, DL);
39651 SDValue Res = DAG.getNode(Opcode0, DL, HalfVT, V0, V1);
39652 return widenSubVector(Res, false, Subtarget, DAG, DL, 256);
39653 }
39654 }
39655
39656 return SDValue();
39657}
39658
39659// Attempt to constant fold all of the constant source ops.
39660// Returns true if the entire shuffle is folded to a constant.
39661// TODO: Extend this to merge multiple constant Ops and update the mask.
39662static SDValue combineX86ShufflesConstants(MVT VT, ArrayRef<SDValue> Ops,
39663 ArrayRef<int> Mask,
39664 bool HasVariableMask,
39665 SelectionDAG &DAG, const SDLoc &DL,
39666 const X86Subtarget &Subtarget) {
39667 unsigned SizeInBits = VT.getSizeInBits();
39668 unsigned NumMaskElts = Mask.size();
39669 unsigned MaskSizeInBits = SizeInBits / NumMaskElts;
39670 unsigned NumOps = Ops.size();
39671
39672 // Extract constant bits from each source op.
39673 SmallVector<APInt, 16> UndefEltsOps(NumOps);
39674 SmallVector<SmallVector<APInt, 16>, 16> RawBitsOps(NumOps);
39675 for (unsigned I = 0; I != NumOps; ++I)
39676 if (!getTargetConstantBitsFromNode(Ops[I], MaskSizeInBits, UndefEltsOps[I],
39677 RawBitsOps[I],
39678 /*AllowWholeUndefs*/ true,
39679 /*AllowPartialUndefs*/ true))
39680 return SDValue();
39681
39682 // If we're optimizing for size, only fold if at least one of the constants is
39683 // only used once or the combined shuffle has included a variable mask
39684 // shuffle, this is to avoid constant pool bloat.
39685 bool IsOptimizingSize = DAG.shouldOptForSize();
39686 if (IsOptimizingSize && !HasVariableMask &&
39687 llvm::none_of(Ops, [](SDValue SrcOp) { return SrcOp->hasOneUse(); }))
39688 return SDValue();
39689
39690 // Shuffle the constant bits according to the mask.
39691 APInt UndefElts(NumMaskElts, 0);
39692 APInt ZeroElts(NumMaskElts, 0);
39693 APInt ConstantElts(NumMaskElts, 0);
39694 SmallVector<APInt, 8> ConstantBitData(NumMaskElts,
39695 APInt::getZero(MaskSizeInBits));
39696 for (unsigned i = 0; i != NumMaskElts; ++i) {
39697 int M = Mask[i];
39698 if (M == SM_SentinelUndef) {
39699 UndefElts.setBit(i);
39700 continue;
39701 } else if (M == SM_SentinelZero) {
39702 ZeroElts.setBit(i);
39703 continue;
39704 }
39705 assert(0 <= M && M < (int)(NumMaskElts * NumOps));
39706
39707 unsigned SrcOpIdx = (unsigned)M / NumMaskElts;
39708 unsigned SrcMaskIdx = (unsigned)M % NumMaskElts;
39709
39710 auto &SrcUndefElts = UndefEltsOps[SrcOpIdx];
39711 if (SrcUndefElts[SrcMaskIdx]) {
39712 UndefElts.setBit(i);
39713 continue;
39714 }
39715
39716 auto &SrcEltBits = RawBitsOps[SrcOpIdx];
39717 APInt &Bits = SrcEltBits[SrcMaskIdx];
39718 if (!Bits) {
39719 ZeroElts.setBit(i);
39720 continue;
39721 }
39722
39723 ConstantElts.setBit(i);
39724 ConstantBitData[i] = Bits;
39725 }
39726 assert((UndefElts | ZeroElts | ConstantElts).isAllOnes());
39727
39728 // Attempt to create a zero vector.
39729 if ((UndefElts | ZeroElts).isAllOnes())
39730 return getZeroVector(VT, Subtarget, DAG, DL);
39731
39732 // Create the constant data.
39733 MVT MaskSVT;
39734 if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64))
39735 MaskSVT = MVT::getFloatingPointVT(MaskSizeInBits);
39736 else
39737 MaskSVT = MVT::getIntegerVT(MaskSizeInBits);
39738
39739 MVT MaskVT = MVT::getVectorVT(MaskSVT, NumMaskElts);
39740 if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT))
39741 return SDValue();
39742
39743 SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL);
39744 return DAG.getBitcast(VT, CstOp);
39745}
39746
39747namespace llvm {
39748 namespace X86 {
39749 enum {
39750 MaxShuffleCombineDepth = 8
39751 };
39752 } // namespace X86
39753} // namespace llvm
39754
39755/// Fully generic combining of x86 shuffle instructions.
39756///
39757/// This should be the last combine run over the x86 shuffle instructions. Once
39758/// they have been fully optimized, this will recursively consider all chains
39759/// of single-use shuffle instructions, build a generic model of the cumulative
39760/// shuffle operation, and check for simpler instructions which implement this
39761/// operation. We use this primarily for two purposes:
39762///
39763/// 1) Collapse generic shuffles to specialized single instructions when
39764/// equivalent. In most cases, this is just an encoding size win, but
39765/// sometimes we will collapse multiple generic shuffles into a single
39766/// special-purpose shuffle.
39767/// 2) Look for sequences of shuffle instructions with 3 or more total
39768/// instructions, and replace them with the slightly more expensive SSSE3
39769/// PSHUFB instruction if available. We do this as the last combining step
39770/// to ensure we avoid using PSHUFB if we can implement the shuffle with
39771/// a suitable short sequence of other instructions. The PSHUFB will either
39772/// use a register or have to read from memory and so is slightly (but only
39773/// slightly) more expensive than the other shuffle instructions.
39774///
39775/// Because this is inherently a quadratic operation (for each shuffle in
39776/// a chain, we recurse up the chain), the depth is limited to 8 instructions.
39777/// This should never be an issue in practice as the shuffle lowering doesn't
39778/// produce sequences of more than 8 instructions.
39779///
39780/// FIXME: We will currently miss some cases where the redundant shuffling
39781/// would simplify under the threshold for PSHUFB formation because of
39782/// combine-ordering. To fix this, we should do the redundant instruction
39783/// combining in this recursive walk.
39784static SDValue combineX86ShufflesRecursively(
39785 ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
39786 ArrayRef<int> RootMask, ArrayRef<const SDNode *> SrcNodes, unsigned Depth,
39787 unsigned MaxDepth, bool HasVariableMask, bool AllowVariableCrossLaneMask,
39788 bool AllowVariablePerLaneMask, SelectionDAG &DAG,
39789 const X86Subtarget &Subtarget) {
39790 assert(!RootMask.empty() &&
39791 (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&
39792 "Illegal shuffle root mask");
39793 MVT RootVT = Root.getSimpleValueType();
39794 assert(RootVT.isVector() && "Shuffles operate on vector types!");
39795 unsigned RootSizeInBits = RootVT.getSizeInBits();
39796 SDLoc DL(Root);
39797
39798 // Bound the depth of our recursive combine because this is ultimately
39799 // quadratic in nature.
39800 if (Depth >= MaxDepth)
39801 return SDValue();
39802
39803 // Directly rip through bitcasts to find the underlying operand.
39804 SDValue Op = SrcOps[SrcOpIndex];
39805 Op = peekThroughOneUseBitcasts(Op);
39806
39807 EVT VT = Op.getValueType();
39808 if (!VT.isVector() || !VT.isSimple())
39809 return SDValue(); // Bail if we hit a non-simple non-vector.
39810
39811 // FIXME: Just bail on f16 for now.
39812 if (VT.getVectorElementType() == MVT::f16)
39813 return SDValue();
39814
39815 assert((RootSizeInBits % VT.getSizeInBits()) == 0 &&
39816 "Can only combine shuffles upto size of the root op.");
39817
39818 // Create a demanded elts mask from the referenced elements of Op.
39819 APInt OpDemandedElts = APInt::getZero(RootMask.size());
39820 for (int M : RootMask) {
39821 int BaseIdx = RootMask.size() * SrcOpIndex;
39822 if (isInRange(M, BaseIdx, BaseIdx + RootMask.size()))
39823 OpDemandedElts.setBit(M - BaseIdx);
39824 }
39825 if (RootSizeInBits != VT.getSizeInBits()) {
39826 // Op is smaller than Root - extract the demanded elts for the subvector.
39827 unsigned Scale = RootSizeInBits / VT.getSizeInBits();
39828 unsigned NumOpMaskElts = RootMask.size() / Scale;
39829 assert((RootMask.size() % Scale) == 0 && "Root mask size mismatch");
39830 assert(OpDemandedElts
39831 .extractBits(RootMask.size() - NumOpMaskElts, NumOpMaskElts)
39832 .isZero() &&
39833 "Out of range elements referenced in root mask");
39834 OpDemandedElts = OpDemandedElts.extractBits(NumOpMaskElts, 0);
39835 }
39836 OpDemandedElts =
39837 APIntOps::ScaleBitMask(OpDemandedElts, VT.getVectorNumElements());
39838
39839 // Extract target shuffle mask and resolve sentinels and inputs.
39840 SmallVector<int, 64> OpMask;
39841 SmallVector<SDValue, 2> OpInputs;
39842 APInt OpUndef, OpZero;
39843 bool IsOpVariableMask = isTargetShuffleVariableMask(Op.getOpcode());
39844 if (getTargetShuffleInputs(Op, OpDemandedElts, OpInputs, OpMask, OpUndef,
39845 OpZero, DAG, Depth, false)) {
39846 // Shuffle inputs must not be larger than the shuffle result.
39847 // TODO: Relax this for single input faux shuffles (e.g. trunc).
39848 if (llvm::any_of(OpInputs, [VT](SDValue OpInput) {
39849 return OpInput.getValueSizeInBits() > VT.getSizeInBits();
39850 }))
39851 return SDValue();
39852 } else if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
39853 (RootSizeInBits % Op.getOperand(0).getValueSizeInBits()) == 0 &&
39854 !isNullConstant(Op.getOperand(1))) {
39855 SDValue SrcVec = Op.getOperand(0);
39856 int ExtractIdx = Op.getConstantOperandVal(1);
39857 unsigned NumElts = VT.getVectorNumElements();
39858 OpInputs.assign({SrcVec});
39859 OpMask.assign(NumElts, SM_SentinelUndef);
39860 std::iota(OpMask.begin(), OpMask.end(), ExtractIdx);
39861 OpZero = OpUndef = APInt::getZero(NumElts);
39862 } else {
39863 return SDValue();
39864 }
39865
39866 // If the shuffle result was smaller than the root, we need to adjust the
39867 // mask indices and pad the mask with undefs.
39868 if (RootSizeInBits > VT.getSizeInBits()) {
39869 unsigned NumSubVecs = RootSizeInBits / VT.getSizeInBits();
39870 unsigned OpMaskSize = OpMask.size();
39871 if (OpInputs.size() > 1) {
39872 unsigned PaddedMaskSize = NumSubVecs * OpMaskSize;
39873 for (int &M : OpMask) {
39874 if (M < 0)
39875 continue;
39876 int EltIdx = M % OpMaskSize;
39877 int OpIdx = M / OpMaskSize;
39878 M = (PaddedMaskSize * OpIdx) + EltIdx;
39879 }
39880 }
39881 OpZero = OpZero.zext(NumSubVecs * OpMaskSize);
39882 OpUndef = OpUndef.zext(NumSubVecs * OpMaskSize);
39883 OpMask.append((NumSubVecs - 1) * OpMaskSize, SM_SentinelUndef);
39884 }
39885
39886 SmallVector<int, 64> Mask;
39887 SmallVector<SDValue, 16> Ops;
39888
39889 // We don't need to merge masks if the root is empty.
39890 bool EmptyRoot = (Depth == 0) && (RootMask.size() == 1);
39891 if (EmptyRoot) {
39892 // Only resolve zeros if it will remove an input, otherwise we might end
39893 // up in an infinite loop.
39894 bool ResolveKnownZeros = true;
39895 if (!OpZero.isZero()) {
39896 APInt UsedInputs = APInt::getZero(OpInputs.size());
39897 for (int i = 0, e = OpMask.size(); i != e; ++i) {
39898 int M = OpMask[i];
39899 if (OpUndef[i] || OpZero[i] || isUndefOrZero(M))
39900 continue;
39901 UsedInputs.setBit(M / OpMask.size());
39902 if (UsedInputs.isAllOnes()) {
39903 ResolveKnownZeros = false;
39904 break;
39905 }
39906 }
39907 }
39908 resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero,
39909 ResolveKnownZeros);
39910
39911 Mask = OpMask;
39912 Ops.append(OpInputs.begin(), OpInputs.end());
39913 } else {
39914 resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero);
39915
39916 // Add the inputs to the Ops list, avoiding duplicates.
39917 Ops.append(SrcOps.begin(), SrcOps.end());
39918
39919 auto AddOp = [&Ops](SDValue Input, int InsertionPoint) -> int {
39920 // Attempt to find an existing match.
39921 SDValue InputBC = peekThroughBitcasts(Input);
39922 for (int i = 0, e = Ops.size(); i < e; ++i)
39923 if (InputBC == peekThroughBitcasts(Ops[i]))
39924 return i;
39925 // Match failed - should we replace an existing Op?
39926 if (InsertionPoint >= 0) {
39927 Ops[InsertionPoint] = Input;
39928 return InsertionPoint;
39929 }
39930 // Add to the end of the Ops list.
39931 Ops.push_back(Input);
39932 return Ops.size() - 1;
39933 };
39934
39935 SmallVector<int, 2> OpInputIdx;
39936 for (SDValue OpInput : OpInputs)
39937 OpInputIdx.push_back(
39938 AddOp(OpInput, OpInputIdx.empty() ? SrcOpIndex : -1));
39939
39940 assert(((RootMask.size() > OpMask.size() &&
39941 RootMask.size() % OpMask.size() == 0) ||
39942 (OpMask.size() > RootMask.size() &&
39943 OpMask.size() % RootMask.size() == 0) ||
39944 OpMask.size() == RootMask.size()) &&
39945 "The smaller number of elements must divide the larger.");
39946
39947 // This function can be performance-critical, so we rely on the power-of-2
39948 // knowledge that we have about the mask sizes to replace div/rem ops with
39949 // bit-masks and shifts.
39950 assert(llvm::has_single_bit<uint32_t>(RootMask.size()) &&
39951 "Non-power-of-2 shuffle mask sizes");
39952 assert(llvm::has_single_bit<uint32_t>(OpMask.size()) &&
39953 "Non-power-of-2 shuffle mask sizes");
39954 unsigned RootMaskSizeLog2 = llvm::countr_zero(RootMask.size());
39955 unsigned OpMaskSizeLog2 = llvm::countr_zero(OpMask.size());
39956
39957 unsigned MaskWidth = std::max<unsigned>(OpMask.size(), RootMask.size());
39958 unsigned RootRatio =
39959 std::max<unsigned>(1, OpMask.size() >> RootMaskSizeLog2);
39960 unsigned OpRatio = std::max<unsigned>(1, RootMask.size() >> OpMaskSizeLog2);
39961 assert((RootRatio == 1 || OpRatio == 1) &&
39962 "Must not have a ratio for both incoming and op masks!");
39963
39964 assert(isPowerOf2_32(MaskWidth) && "Non-power-of-2 shuffle mask sizes");
39965 assert(isPowerOf2_32(RootRatio) && "Non-power-of-2 shuffle mask sizes");
39966 assert(isPowerOf2_32(OpRatio) && "Non-power-of-2 shuffle mask sizes");
39967 unsigned RootRatioLog2 = llvm::countr_zero(RootRatio);
39968 unsigned OpRatioLog2 = llvm::countr_zero(OpRatio);
39969
39970 Mask.resize(MaskWidth, SM_SentinelUndef);
39971
39972 // Merge this shuffle operation's mask into our accumulated mask. Note that
39973 // this shuffle's mask will be the first applied to the input, followed by
39974 // the root mask to get us all the way to the root value arrangement. The
39975 // reason for this order is that we are recursing up the operation chain.
39976 for (unsigned i = 0; i < MaskWidth; ++i) {
39977 unsigned RootIdx = i >> RootRatioLog2;
39978 if (RootMask[RootIdx] < 0) {
39979 // This is a zero or undef lane, we're done.
39980 Mask[i] = RootMask[RootIdx];
39981 continue;
39982 }
39983
39984 unsigned RootMaskedIdx =
39985 RootRatio == 1
39986 ? RootMask[RootIdx]
39987 : (RootMask[RootIdx] << RootRatioLog2) + (i & (RootRatio - 1));
39988
39989 // Just insert the scaled root mask value if it references an input other
39990 // than the SrcOp we're currently inserting.
39991 if ((RootMaskedIdx < (SrcOpIndex * MaskWidth)) ||
39992 (((SrcOpIndex + 1) * MaskWidth) <= RootMaskedIdx)) {
39993 Mask[i] = RootMaskedIdx;
39994 continue;
39995 }
39996
39997 RootMaskedIdx = RootMaskedIdx & (MaskWidth - 1);
39998 unsigned OpIdx = RootMaskedIdx >> OpRatioLog2;
39999 if (OpMask[OpIdx] < 0) {
40000 // The incoming lanes are zero or undef, it doesn't matter which ones we
40001 // are using.
40002 Mask[i] = OpMask[OpIdx];
40003 continue;
40004 }
40005
40006 // Ok, we have non-zero lanes, map them through to one of the Op's inputs.
40007 unsigned OpMaskedIdx = OpRatio == 1 ? OpMask[OpIdx]
40008 : (OpMask[OpIdx] << OpRatioLog2) +
40009 (RootMaskedIdx & (OpRatio - 1));
40010
40011 OpMaskedIdx = OpMaskedIdx & (MaskWidth - 1);
40012 int InputIdx = OpMask[OpIdx] / (int)OpMask.size();
40013 assert(0 <= OpInputIdx[InputIdx] && "Unknown target shuffle input");
40014 OpMaskedIdx += OpInputIdx[InputIdx] * MaskWidth;
40015
40016 Mask[i] = OpMaskedIdx;
40017 }
40018 }
40019
40020 // Peek through vector widenings and set out of bounds mask indices to undef.
40021 // TODO: Can resolveTargetShuffleInputsAndMask do some of this?
40022 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
40023 SDValue &Op = Ops[I];
40024 if (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op.getOperand(0).isUndef() &&
40025 isNullConstant(Op.getOperand(2))) {
40026 Op = Op.getOperand(1);
40027 unsigned Scale = RootSizeInBits / Op.getValueSizeInBits();
40028 int Lo = I * Mask.size();
40029 int Hi = (I + 1) * Mask.size();
40030 int NewHi = Lo + (Mask.size() / Scale);
40031 for (int &M : Mask) {
40032 if (Lo <= M && NewHi <= M && M < Hi)
40033 M = SM_SentinelUndef;
40034 }
40035 }
40036 }
40037
40038 // Peek through any free extract_subvector nodes back to root size.
40039 for (SDValue &Op : Ops)
40040 while (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
40041 (RootSizeInBits % Op.getOperand(0).getValueSizeInBits()) == 0 &&
40042 isNullConstant(Op.getOperand(1)))
40043 Op = Op.getOperand(0);
40044
40045 // Remove unused/repeated shuffle source ops.
40046 resolveTargetShuffleInputsAndMask(Ops, Mask);
40047
40048 // Handle the all undef/zero/ones cases early.
40049 if (all_of(Mask, [](int Idx) { return Idx == SM_SentinelUndef; }))
40050 return DAG.getUNDEF(RootVT);
40051 if (all_of(Mask, [](int Idx) { return Idx < 0; }))
40052 return getZeroVector(RootVT, Subtarget, DAG, DL);
40053 if (Ops.size() == 1 && ISD::isBuildVectorAllOnes(Ops[0].getNode()) &&
40054 !llvm::is_contained(Mask, SM_SentinelZero))
40055 return getOnesVector(RootVT, DAG, DL);
40056
40057 assert(!Ops.empty() && "Shuffle with no inputs detected");
40058 HasVariableMask |= IsOpVariableMask;
40059
40060 // Update the list of shuffle nodes that have been combined so far.
40061 SmallVector<const SDNode *, 16> CombinedNodes(SrcNodes);
40062 CombinedNodes.push_back(Op.getNode());
40063
40064 // See if we can recurse into each shuffle source op (if it's a target
40065 // shuffle). The source op should only be generally combined if it either has
40066 // a single use (i.e. current Op) or all its users have already been combined,
40067 // if not then we can still combine but should prevent generation of variable
40068 // shuffles to avoid constant pool bloat.
40069 // Don't recurse if we already have more source ops than we can combine in
40070 // the remaining recursion depth.
40071 if (Ops.size() < (MaxDepth - Depth)) {
40072 for (int i = 0, e = Ops.size(); i < e; ++i) {
40073 // For empty roots, we need to resolve zeroable elements before combining
40074 // them with other shuffles.
40075 SmallVector<int, 64> ResolvedMask = Mask;
40076 if (EmptyRoot)
40077 resolveTargetShuffleFromZeroables(ResolvedMask, OpUndef, OpZero);
40078 bool AllowCrossLaneVar = false;
40079 bool AllowPerLaneVar = false;
40080 if (Ops[i].getNode()->hasOneUse() ||
40081 SDNode::areOnlyUsersOf(CombinedNodes, Ops[i].getNode())) {
40082 AllowCrossLaneVar = AllowVariableCrossLaneMask;
40083 AllowPerLaneVar = AllowVariablePerLaneMask;
40084 }
40085 if (SDValue Res = combineX86ShufflesRecursively(
40086 Ops, i, Root, ResolvedMask, CombinedNodes, Depth + 1, MaxDepth,
40087 HasVariableMask, AllowCrossLaneVar, AllowPerLaneVar, DAG,
40088 Subtarget))
40089 return Res;
40090 }
40091 }
40092
40093 // Attempt to constant fold all of the constant source ops.
40094 if (SDValue Cst = combineX86ShufflesConstants(
40095 RootVT, Ops, Mask, HasVariableMask, DAG, DL, Subtarget))
40096 return Cst;
40097
40098 // If constant fold failed and we only have constants - then we have
40099 // multiple uses by a single non-variable shuffle - just bail.
40100 if (Depth == 0 && llvm::all_of(Ops, [&](SDValue Op) {
40101 APInt UndefElts;
40102 SmallVector<APInt> RawBits;
40103 unsigned EltSizeInBits = RootSizeInBits / Mask.size();
40104 return getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
40105 RawBits,
40106 /*AllowWholeUndefs*/ true,
40107 /*AllowPartialUndefs*/ true);
40108 })) {
40109 return SDValue();
40110 }
40111
40112 // Canonicalize the combined shuffle mask chain with horizontal ops.
40113 // NOTE: This will update the Ops and Mask.
40114 if (SDValue HOp = canonicalizeShuffleMaskWithHorizOp(
40115 Ops, Mask, RootSizeInBits, DL, DAG, Subtarget))
40116 return DAG.getBitcast(RootVT, HOp);
40117
40118 // Try to refine our inputs given our knowledge of target shuffle mask.
40119 for (auto I : enumerate(Ops)) {
40120 int OpIdx = I.index();
40121 SDValue &Op = I.value();
40122
40123 // What range of shuffle mask element values results in picking from Op?
40124 int Lo = OpIdx * Mask.size();
40125 int Hi = Lo + Mask.size();
40126
40127 // Which elements of Op do we demand, given the mask's granularity?
40128 APInt OpDemandedElts(Mask.size(), 0);
40129 for (int MaskElt : Mask) {
40130 if (isInRange(MaskElt, Lo, Hi)) { // Picks from Op?
40131 int OpEltIdx = MaskElt - Lo;
40132 OpDemandedElts.setBit(OpEltIdx);
40133 }
40134 }
40135
40136 // Is the shuffle result smaller than the root?
40137 if (Op.getValueSizeInBits() < RootSizeInBits) {
40138 // We padded the mask with undefs. But we now need to undo that.
40139 unsigned NumExpectedVectorElts = Mask.size();
40140 unsigned EltSizeInBits = RootSizeInBits / NumExpectedVectorElts;
40141 unsigned NumOpVectorElts = Op.getValueSizeInBits() / EltSizeInBits;
40142 assert(!OpDemandedElts.extractBits(
40143 NumExpectedVectorElts - NumOpVectorElts, NumOpVectorElts) &&
40144 "Demanding the virtual undef widening padding?");
40145 OpDemandedElts = OpDemandedElts.trunc(NumOpVectorElts); // NUW
40146 }
40147
40148 // The Op itself may be of different VT, so we need to scale the mask.
40149 unsigned NumOpElts = Op.getValueType().getVectorNumElements();
40150 APInt OpScaledDemandedElts = APIntOps::ScaleBitMask(OpDemandedElts, NumOpElts);
40151
40152 // Can this operand be simplified any further, given it's demanded elements?
40153 if (SDValue NewOp =
40154 DAG.getTargetLoweringInfo().SimplifyMultipleUseDemandedVectorElts(
40155 Op, OpScaledDemandedElts, DAG))
40156 Op = NewOp;
40157 }
40158 // FIXME: should we rerun resolveTargetShuffleInputsAndMask() now?
40159
40160 // Widen any subvector shuffle inputs we've collected.
40161 // TODO: Remove this to avoid generating temporary nodes, we should only
40162 // widen once combineX86ShuffleChain has found a match.
40163 if (any_of(Ops, [RootSizeInBits](SDValue Op) {
40164 return Op.getValueSizeInBits() < RootSizeInBits;
40165 })) {
40166 for (SDValue &Op : Ops)
40167 if (Op.getValueSizeInBits() < RootSizeInBits)
40168 Op = widenSubVector(Op, false, Subtarget, DAG, SDLoc(Op),
40169 RootSizeInBits);
40170 // Reresolve - we might have repeated subvector sources.
40171 resolveTargetShuffleInputsAndMask(Ops, Mask);
40172 }
40173
40174 // We can only combine unary and binary shuffle mask cases.
40175 if (Ops.size() <= 2) {
40176 // Minor canonicalization of the accumulated shuffle mask to make it easier
40177 // to match below. All this does is detect masks with sequential pairs of
40178 // elements, and shrink them to the half-width mask. It does this in a loop
40179 // so it will reduce the size of the mask to the minimal width mask which
40180 // performs an equivalent shuffle.
40181 while (Mask.size() > 1) {
40182 SmallVector<int, 64> WidenedMask;
40183 if (!canWidenShuffleElements(Mask, WidenedMask))
40184 break;
40185 Mask = std::move(WidenedMask);
40186 }
40187
40188 // Canonicalization of binary shuffle masks to improve pattern matching by
40189 // commuting the inputs.
40190 if (Ops.size() == 2 && canonicalizeShuffleMaskWithCommute(Mask)) {
40191 ShuffleVectorSDNode::commuteMask(Mask);
40192 std::swap(Ops[0], Ops[1]);
40193 }
40194
40195 // Try to combine into a single shuffle instruction.
40196 if (SDValue Shuffle = combineX86ShuffleChain(
40197 Ops, Root, Mask, Depth, HasVariableMask, AllowVariableCrossLaneMask,
40198 AllowVariablePerLaneMask, DAG, Subtarget))
40199 return Shuffle;
40200
40201 // If all the operands come from the same larger vector, fallthrough and try
40202 // to use combineX86ShuffleChainWithExtract.
40203 SDValue LHS = peekThroughBitcasts(Ops.front());
40204 SDValue RHS = peekThroughBitcasts(Ops.back());
40205 if (Ops.size() != 2 || !Subtarget.hasAVX2() || RootSizeInBits != 128 ||
40206 (RootSizeInBits / Mask.size()) != 64 ||
40207 LHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
40208 RHS.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
40209 LHS.getOperand(0) != RHS.getOperand(0))
40210 return SDValue();
40211 }
40212
40213 // If that failed and any input is extracted then try to combine as a
40214 // shuffle with the larger type.
40215 return combineX86ShuffleChainWithExtract(
40216 Ops, Root, Mask, Depth, HasVariableMask, AllowVariableCrossLaneMask,
40217 AllowVariablePerLaneMask, DAG, Subtarget);
40218}
40219
40220/// Helper entry wrapper to combineX86ShufflesRecursively.
40221static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
40222 const X86Subtarget &Subtarget) {
40223 return combineX86ShufflesRecursively(
40224 {Op}, 0, Op, {0}, {}, /*Depth*/ 0, X86::MaxShuffleCombineDepth,
40225 /*HasVarMask*/ false,
40226 /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, DAG,
40227 Subtarget);
40228}
40229
40230/// Get the PSHUF-style mask from PSHUF node.
40231///
40232/// This is a very minor wrapper around getTargetShuffleMask to easy forming v4
40233/// PSHUF-style masks that can be reused with such instructions.
40234static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
40235 MVT VT = N.getSimpleValueType();
40236 SmallVector<int, 4> Mask;
40237 SmallVector<SDValue, 2> Ops;
40238 bool HaveMask = getTargetShuffleMask(N, false, Ops, Mask);
40239 (void)HaveMask;
40240 assert(HaveMask);
40241
40242 // If we have more than 128-bits, only the low 128-bits of shuffle mask
40243 // matter. Check that the upper masks are repeats and remove them.
40244 if (VT.getSizeInBits() > 128) {
40245 int LaneElts = 128 / VT.getScalarSizeInBits();
40246#ifndef NDEBUG
40247 for (int i = 1, NumLanes = VT.getSizeInBits() / 128; i < NumLanes; ++i)
40248 for (int j = 0; j < LaneElts; ++j)
40249 assert(Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) &&
40250 "Mask doesn't repeat in high 128-bit lanes!");
40251#endif
40252 Mask.resize(LaneElts);
40253 }
40254
40255 switch (N.getOpcode()) {
40256 case X86ISD::PSHUFD:
40257 return Mask;
40258 case X86ISD::PSHUFLW:
40259 Mask.resize(4);
40260 return Mask;
40261 case X86ISD::PSHUFHW:
40262 Mask.erase(Mask.begin(), Mask.begin() + 4);
40263 for (int &M : Mask)
40264 M -= 4;
40265 return Mask;
40266 default:
40267 llvm_unreachable("No valid shuffle instruction found!");
40268 }
40269}
40270
40271/// Search for a combinable shuffle across a chain ending in pshufd.
40272///
40273/// We walk up the chain and look for a combinable shuffle, skipping over
40274/// shuffles that we could hoist this shuffle's transformation past without
40275/// altering anything.
40276static SDValue combineRedundantDWordShuffle(SDValue N,
40277 MutableArrayRef<int> Mask,
40278 const SDLoc &DL,
40279 SelectionDAG &DAG) {
40280 assert(N.getOpcode() == X86ISD::PSHUFD &&
40281 "Called with something other than an x86 128-bit half shuffle!");
40282
40283 // Walk up a single-use chain looking for a combinable shuffle. Keep a stack
40284 // of the shuffles in the chain so that we can form a fresh chain to replace
40285 // this one.
40286 SmallVector<SDValue, 8> Chain;
40287 SDValue V = N.getOperand(0);
40288 for (; V.hasOneUse(); V = V.getOperand(0)) {
40289 switch (V.getOpcode()) {
40290 default:
40291 return SDValue(); // Nothing combined!
40292
40293 case ISD::BITCAST:
40294 // Skip bitcasts as we always know the type for the target specific
40295 // instructions.
40296 continue;
40297
40298 case X86ISD::PSHUFD:
40299 // Found another dword shuffle.
40300 break;
40301
40302 case X86ISD::PSHUFLW:
40303 // Check that the low words (being shuffled) are the identity in the
40304 // dword shuffle, and the high words are self-contained.
40305 if (Mask[0] != 0 || Mask[1] != 1 ||
40306 !(Mask[2] >= 2 && Mask[2] < 4 && Mask[3] >= 2 && Mask[3] < 4))
40307 return SDValue();
40308
40309 Chain.push_back(V);
40310 continue;
40311
40312 case X86ISD::PSHUFHW:
40313 // Check that the high words (being shuffled) are the identity in the
40314 // dword shuffle, and the low words are self-contained.
40315 if (Mask[2] != 2 || Mask[3] != 3 ||
40316 !(Mask[0] >= 0 && Mask[0] < 2 && Mask[1] >= 0 && Mask[1] < 2))
40317 return SDValue();
40318
40319 Chain.push_back(V);
40320 continue;
40321
40322 case X86ISD::UNPCKL:
40323 case X86ISD::UNPCKH:
40324 // For either i8 -> i16 or i16 -> i32 unpacks, we can combine a dword
40325 // shuffle into a preceding word shuffle.
40326 if (V.getSimpleValueType().getVectorElementType() != MVT::i8 &&
40327 V.getSimpleValueType().getVectorElementType() != MVT::i16)
40328 return SDValue();
40329
40330 // Search for a half-shuffle which we can combine with.
40331 unsigned CombineOp =
40332 V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
40333 if (V.getOperand(0) != V.getOperand(1) ||
40334 !V->isOnlyUserOf(V.getOperand(0).getNode()))
40335 return SDValue();
40336 Chain.push_back(V);
40337 V = V.getOperand(0);
40338 do {
40339 switch (V.getOpcode()) {
40340 default:
40341 return SDValue(); // Nothing to combine.
40342
40343 case X86ISD::PSHUFLW:
40344 case X86ISD::PSHUFHW:
40345 if (V.getOpcode() == CombineOp)
40346 break;
40347
40348 Chain.push_back(V);
40349
40350 [[fallthrough]];
40351 case ISD::BITCAST:
40352 V = V.getOperand(0);
40353 continue;
40354 }
40355 break;
40356 } while (V.hasOneUse());
40357 break;
40358 }
40359 // Break out of the loop if we break out of the switch.
40360 break;
40361 }
40362
40363 if (!V.hasOneUse())
40364 // We fell out of the loop without finding a viable combining instruction.
40365 return SDValue();
40366
40367 // Merge this node's mask and our incoming mask.
40368 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
40369 for (int &M : Mask)
40370 M = VMask[M];
40371 V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
40372 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
40373
40374 // Rebuild the chain around this new shuffle.
40375 while (!Chain.empty()) {
40376 SDValue W = Chain.pop_back_val();
40377
40378 if (V.getValueType() != W.getOperand(0).getValueType())
40379 V = DAG.getBitcast(W.getOperand(0).getValueType(), V);
40380
40381 switch (W.getOpcode()) {
40382 default:
40383 llvm_unreachable("Only PSHUF and UNPCK instructions get here!");
40384
40385 case X86ISD::UNPCKL:
40386 case X86ISD::UNPCKH:
40387 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V);
40388 break;
40389
40390 case X86ISD::PSHUFD:
40391 case X86ISD::PSHUFLW:
40392 case X86ISD::PSHUFHW:
40393 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1));
40394 break;
40395 }
40396 }
40397 if (V.getValueType() != N.getValueType())
40398 V = DAG.getBitcast(N.getValueType(), V);
40399
40400 // Return the new chain to replace N.
40401 return V;
40402}
40403
40404// Attempt to commute shufps LHS loads:
40405// permilps(shufps(load(),x)) --> permilps(shufps(x,load()))
40406static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,
40407 SelectionDAG &DAG) {
40408 // TODO: Add vXf64 support.
40409 if (VT != MVT::v4f32 && VT != MVT::v8f32 && VT != MVT::v16f32)
40410 return SDValue();
40411
40412 // SHUFP(LHS, RHS) -> SHUFP(RHS, LHS) iff LHS is foldable + RHS is not.
40413 auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) {
40414 if (V.getOpcode() != X86ISD::SHUFP || !Parent->isOnlyUserOf(V.getNode()))
40415 return SDValue();
40416 SDValue N0 = V.getOperand(0);
40417 SDValue N1 = V.getOperand(1);
40418 unsigned Imm = V.getConstantOperandVal(2);
40419 const X86Subtarget &Subtarget = DAG.getSubtarget<X86Subtarget>();
40420 if (!X86::mayFoldLoad(peekThroughOneUseBitcasts(N0), Subtarget) ||
40421 X86::mayFoldLoad(peekThroughOneUseBitcasts(N1), Subtarget))
40422 return SDValue();
40423 Imm = ((Imm & 0x0F) << 4) | ((Imm & 0xF0) >> 4);
40424 return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0,
40425 DAG.getTargetConstant(Imm, DL, MVT::i8));
40426 };
40427
40428 switch (N.getOpcode()) {
40429 case X86ISD::VPERMILPI:
40430 if (SDValue NewSHUFP = commuteSHUFP(N, N.getOperand(0))) {
40431 unsigned Imm = N.getConstantOperandVal(1);
40432 return DAG.getNode(X86ISD::VPERMILPI, DL, VT, NewSHUFP,
40433 DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
40434 }
40435 break;
40436 case X86ISD::SHUFP: {
40437 SDValue N0 = N.getOperand(0);
40438 SDValue N1 = N.getOperand(1);
40439 unsigned Imm = N.getConstantOperandVal(2);
40440 if (N0 == N1) {
40441 if (SDValue NewSHUFP = commuteSHUFP(N, N0))
40442 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, NewSHUFP,
40443 DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));
40444 } else if (SDValue NewSHUFP = commuteSHUFP(N, N0)) {
40445 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, N1,
40446 DAG.getTargetConstant(Imm ^ 0x0A, DL, MVT::i8));
40447 } else if (SDValue NewSHUFP = commuteSHUFP(N, N1)) {
40448 return DAG.getNode(X86ISD::SHUFP, DL, VT, N0, NewSHUFP,
40449 DAG.getTargetConstant(Imm ^ 0xA0, DL, MVT::i8));
40450 }
40451 break;
40452 }
40453 }
40454
40455 return SDValue();
40456}
40457
40458// Attempt to fold BLEND(PERMUTE(X),PERMUTE(Y)) -> PERMUTE(BLEND(X,Y))
40459// iff we don't demand the same element index for both X and Y.
40460static SDValue
40461combineBlendOfPermutes(MVT VT, SDValue N0, SDValue N1, ArrayRef<int> BlendMask,
40462 const APInt &DemandedElts, SelectionDAG &DAG,
40463 const X86Subtarget &Subtarget, const SDLoc &DL) {
40464 assert(isBlendOrUndef(BlendMask) && "Blend shuffle expected");
40465 if (!N0.hasOneUse() || !N1.hasOneUse())
40466 return SDValue();
40467
40468 unsigned NumElts = VT.getVectorNumElements();
40469 SDValue BC0 = peekThroughOneUseBitcasts(N0);
40470 SDValue BC1 = peekThroughOneUseBitcasts(N1);
40471
40472 // See if both operands are shuffles, and that we can scale the shuffle masks
40473 // to the same width as the blend mask.
40474 // TODO: Support SM_SentinelZero?
40475 SmallVector<SDValue, 2> Ops0, Ops1;
40476 SmallVector<int, 32> Mask0, Mask1, ScaledMask0, ScaledMask1;
40477 if (!getTargetShuffleMask(BC0, /*AllowSentinelZero=*/false, Ops0, Mask0) ||
40478 !getTargetShuffleMask(BC1, /*AllowSentinelZero=*/false, Ops1, Mask1) ||
40479 !scaleShuffleElements(Mask0, NumElts, ScaledMask0) ||
40480 !scaleShuffleElements(Mask1, NumElts, ScaledMask1))
40481 return SDValue();
40482
40483 // Determine the demanded elts from both permutes.
40484 APInt Demanded0, DemandedLHS0, DemandedRHS0;
40485 APInt Demanded1, DemandedLHS1, DemandedRHS1;
40486 if (!getShuffleDemandedElts(NumElts, BlendMask, DemandedElts, Demanded0,
40487 Demanded1,
40488 /*AllowUndefElts=*/true) ||
40489 !getShuffleDemandedElts(NumElts, ScaledMask0, Demanded0, DemandedLHS0,
40490 DemandedRHS0, /*AllowUndefElts=*/true) ||
40491 !getShuffleDemandedElts(NumElts, ScaledMask1, Demanded1, DemandedLHS1,
40492 DemandedRHS1, /*AllowUndefElts=*/true))
40493 return SDValue();
40494
40495 // Confirm that we only use a single operand from both permutes and that we
40496 // don't demand the same index from both.
40497 if (!DemandedRHS0.isZero() || !DemandedRHS1.isZero() ||
40498 DemandedLHS0.intersects(DemandedLHS1))
40499 return SDValue();
40500
40501 // Use the permute demanded elts masks as the new blend mask.
40502 // Create the new permute mask as a blend of the 2 original permute masks.
40503 SmallVector<int, 32> NewBlendMask(NumElts, SM_SentinelUndef);
40504 SmallVector<int, 32> NewPermuteMask(NumElts, SM_SentinelUndef);
40505 for (unsigned I = 0; I != NumElts; ++I) {
40506 if (Demanded0[I]) {
40507 int M = ScaledMask0[I];
40508 if (0 <= M) {
40509 assert(isUndefOrEqual(NewBlendMask[M], M) &&
40510 "BlendMask demands LHS AND RHS");
40511 NewBlendMask[M] = M;
40512 NewPermuteMask[I] = M;
40513 }
40514 } else if (Demanded1[I]) {
40515 int M = ScaledMask1[I];
40516 if (0 <= M) {
40517 assert(isUndefOrEqual(NewBlendMask[M], M + NumElts) &&
40518 "BlendMask demands LHS AND RHS");
40519 NewBlendMask[M] = M + NumElts;
40520 NewPermuteMask[I] = M;
40521 }
40522 }
40523 }
40524 assert(isBlendOrUndef(NewBlendMask) && "Bad blend");
40525 assert(isUndefOrInRange(NewPermuteMask, 0, NumElts) && "Bad permute");
40526
40527 // v16i16 shuffles can explode in complexity very easily, only accept them if
40528 // the blend mask is the same in the 128-bit subvectors (or can widen to
40529 // v8i32) and the permute can be widened as well.
40530 if (VT == MVT::v16i16) {
40531 if (!is128BitLaneRepeatedShuffleMask(VT, NewBlendMask) &&
40532 !canWidenShuffleElements(NewBlendMask))
40533 return SDValue();
40534 if (!canWidenShuffleElements(NewPermuteMask))
40535 return SDValue();
40536 }
40537
40538 // Don't introduce lane-crossing permutes without AVX2, unless it can be
40539 // widened to a lane permute (vperm2f128).
40540 if (VT.is256BitVector() && !Subtarget.hasAVX2() &&
40541 isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(),
40542 NewPermuteMask) &&
40543 !canScaleShuffleElements(NewPermuteMask, 2))
40544 return SDValue();
40545
40546 SDValue NewBlend =
40547 DAG.getVectorShuffle(VT, DL, DAG.getBitcast(VT, Ops0[0]),
40548 DAG.getBitcast(VT, Ops1[0]), NewBlendMask);
40549 return DAG.getVectorShuffle(VT, DL, NewBlend, DAG.getUNDEF(VT),
40550 NewPermuteMask);
40551}
40552
40553// TODO - move this to TLI like isBinOp?
40554static bool isUnaryOp(unsigned Opcode) {
40555 switch (Opcode) {
40556 case ISD::CTLZ:
40557 case ISD::CTTZ:
40558 case ISD::CTPOP:
40559 return true;
40560 }
40561 return false;
40562}
40563
40564// Canonicalize SHUFFLE(UNARYOP(X)) -> UNARYOP(SHUFFLE(X)).
40565// Canonicalize SHUFFLE(BINOP(X,Y)) -> BINOP(SHUFFLE(X),SHUFFLE(Y)).
40566static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG,
40567 const SDLoc &DL) {
40568 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40569 EVT ShuffleVT = N.getValueType();
40570 unsigned Opc = N.getOpcode();
40571
40572 auto IsMergeableWithShuffle = [Opc, &DAG](SDValue Op, bool FoldShuf = true,
40573 bool FoldLoad = false) {
40574 // AllZeros/AllOnes constants are freely shuffled and will peek through
40575 // bitcasts. Other constant build vectors do not peek through bitcasts. Only
40576 // merge with target shuffles if it has one use so shuffle combining is
40577 // likely to kick in. Shuffles of splats are expected to be removed.
40578 return ISD::isBuildVectorAllOnes(Op.getNode()) ||
40579 ISD::isBuildVectorAllZeros(Op.getNode()) ||
40580 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
40581 ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()) ||
40582 getTargetConstantFromNode(dyn_cast<LoadSDNode>(Op)) ||
40583 (Op.getOpcode() == Opc && Op->hasOneUse()) ||
40584 (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op->hasOneUse()) ||
40585 (FoldShuf && isTargetShuffle(Op.getOpcode()) && Op->hasOneUse()) ||
40586 (FoldLoad && isShuffleFoldableLoad(Op)) ||
40587 DAG.isSplatValue(Op, /*AllowUndefs*/ false);
40588 };
40589 auto IsSafeToMoveShuffle = [ShuffleVT](SDValue Op, unsigned BinOp) {
40590 // Ensure we only shuffle whole vector src elements, unless its a logical
40591 // binops where we can more aggressively move shuffles from dst to src.
40592 return isLogicOp(BinOp) ||
40593 (Op.getScalarValueSizeInBits() <= ShuffleVT.getScalarSizeInBits());
40594 };
40595
40596 switch (Opc) {
40597 // Unary and Unary+Permute Shuffles.
40598 case X86ISD::PSHUFB: {
40599 // Don't merge PSHUFB if it contains zero'd elements.
40600 SmallVector<int> Mask;
40601 SmallVector<SDValue> Ops;
40602 if (!getTargetShuffleMask(N, false, Ops, Mask))
40603 break;
40604 [[fallthrough]];
40605 }
40606 case X86ISD::VBROADCAST:
40607 case X86ISD::MOVDDUP:
40608 case X86ISD::PSHUFD:
40609 case X86ISD::PSHUFHW:
40610 case X86ISD::PSHUFLW:
40611 case X86ISD::VPERMI:
40612 case X86ISD::VPERMILPI: {
40613 if (N.getOperand(0).getValueType() == ShuffleVT &&
40614 N->isOnlyUserOf(N.getOperand(0).getNode())) {
40615 SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
40616 unsigned SrcOpcode = N0.getOpcode();
40617 if (TLI.isBinOp(SrcOpcode) && IsSafeToMoveShuffle(N0, SrcOpcode)) {
40618 SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
40619 SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
40620 if (IsMergeableWithShuffle(Op00, Opc != X86ISD::VPERMI,
40621 Opc != X86ISD::PSHUFB) ||
40622 IsMergeableWithShuffle(Op01, Opc != X86ISD::VPERMI,
40623 Opc != X86ISD::PSHUFB)) {
40624 SDValue LHS, RHS;
40625 Op00 = DAG.getBitcast(ShuffleVT, Op00);
40626 Op01 = DAG.getBitcast(ShuffleVT, Op01);
40627 if (N.getNumOperands() == 2) {
40628 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, N.getOperand(1));
40629 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, N.getOperand(1));
40630 } else {
40631 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00);
40632 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01);
40633 }
40634 EVT OpVT = N0.getValueType();
40635 return DAG.getBitcast(ShuffleVT,
40636 DAG.getNode(SrcOpcode, DL, OpVT,
40637 DAG.getBitcast(OpVT, LHS),
40638 DAG.getBitcast(OpVT, RHS)));
40639 }
40640 }
40641 }
40642 break;
40643 }
40644 // Binary and Binary+Permute Shuffles.
40645 case X86ISD::INSERTPS: {
40646 // Don't merge INSERTPS if it contains zero'd elements.
40647 unsigned InsertPSMask = N.getConstantOperandVal(2);
40648 unsigned ZeroMask = InsertPSMask & 0xF;
40649 if (ZeroMask != 0)
40650 break;
40651 [[fallthrough]];
40652 }
40653 case X86ISD::MOVSD:
40654 case X86ISD::MOVSS:
40655 case X86ISD::BLENDI:
40656 case X86ISD::SHUFP:
40657 case X86ISD::UNPCKH:
40658 case X86ISD::UNPCKL: {
40659 if (N->isOnlyUserOf(N.getOperand(0).getNode()) &&
40660 N->isOnlyUserOf(N.getOperand(1).getNode())) {
40661 SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
40662 SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
40663 unsigned SrcOpcode = N0.getOpcode();
40664 if (TLI.isBinOp(SrcOpcode) && N1.getOpcode() == SrcOpcode &&
40665 N0.getValueType() == N1.getValueType() &&
40666 IsSafeToMoveShuffle(N0, SrcOpcode) &&
40667 IsSafeToMoveShuffle(N1, SrcOpcode)) {
40668 SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
40669 SDValue Op10 = peekThroughOneUseBitcasts(N1.getOperand(0));
40670 SDValue Op01 = peekThroughOneUseBitcasts(N0.getOperand(1));
40671 SDValue Op11 = peekThroughOneUseBitcasts(N1.getOperand(1));
40672 // Ensure the total number of shuffles doesn't increase by folding this
40673 // shuffle through to the source ops.
40674 if (((IsMergeableWithShuffle(Op00) && IsMergeableWithShuffle(Op10)) ||
40675 (IsMergeableWithShuffle(Op01) && IsMergeableWithShuffle(Op11))) ||
40676 ((IsMergeableWithShuffle(Op00) || IsMergeableWithShuffle(Op10)) &&
40677 (IsMergeableWithShuffle(Op01) || IsMergeableWithShuffle(Op11)))) {
40678 SDValue LHS, RHS;
40679 Op00 = DAG.getBitcast(ShuffleVT, Op00);
40680 Op10 = DAG.getBitcast(ShuffleVT, Op10);
40681 Op01 = DAG.getBitcast(ShuffleVT, Op01);
40682 Op11 = DAG.getBitcast(ShuffleVT, Op11);
40683 if (N.getNumOperands() == 3) {
40684 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2));
40685 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11, N.getOperand(2));
40686 } else {
40687 LHS = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10);
40688 RHS = DAG.getNode(Opc, DL, ShuffleVT, Op01, Op11);
40689 }
40690 EVT OpVT = N0.getValueType();
40691 return DAG.getBitcast(ShuffleVT,
40692 DAG.getNode(SrcOpcode, DL, OpVT,
40693 DAG.getBitcast(OpVT, LHS),
40694 DAG.getBitcast(OpVT, RHS)));
40695 }
40696 }
40697 if (isUnaryOp(SrcOpcode) && N1.getOpcode() == SrcOpcode &&
40698 N0.getValueType() == N1.getValueType() &&
40699 IsSafeToMoveShuffle(N0, SrcOpcode) &&
40700 IsSafeToMoveShuffle(N1, SrcOpcode)) {
40701 SDValue Op00 = peekThroughOneUseBitcasts(N0.getOperand(0));
40702 SDValue Op10 = peekThroughOneUseBitcasts(N1.getOperand(0));
40703 SDValue Res;
40704 Op00 = DAG.getBitcast(ShuffleVT, Op00);
40705 Op10 = DAG.getBitcast(ShuffleVT, Op10);
40706 if (N.getNumOperands() == 3) {
40707 Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10, N.getOperand(2));
40708 } else {
40709 Res = DAG.getNode(Opc, DL, ShuffleVT, Op00, Op10);
40710 }
40711 EVT OpVT = N0.getValueType();
40712 return DAG.getBitcast(
40713 ShuffleVT,
40714 DAG.getNode(SrcOpcode, DL, OpVT, DAG.getBitcast(OpVT, Res)));
40715 }
40716 }
40717 break;
40718 }
40719 }
40720 return SDValue();
40721}
40722
40723/// Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).
40724static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V,
40725 SelectionDAG &DAG,
40726 const SDLoc &DL) {
40727 assert(V.getOpcode() == X86ISD::VPERM2X128 && "Unknown lane shuffle");
40728
40729 MVT VT = V.getSimpleValueType();
40730 SDValue Src0 = peekThroughBitcasts(V.getOperand(0));
40731 SDValue Src1 = peekThroughBitcasts(V.getOperand(1));
40732 unsigned SrcOpc0 = Src0.getOpcode();
40733 unsigned SrcOpc1 = Src1.getOpcode();
40734 EVT SrcVT0 = Src0.getValueType();
40735 EVT SrcVT1 = Src1.getValueType();
40736
40737 if (!Src1.isUndef() && (SrcVT0 != SrcVT1 || SrcOpc0 != SrcOpc1))
40738 return SDValue();
40739
40740 switch (SrcOpc0) {
40741 case X86ISD::MOVDDUP: {
40742 SDValue LHS = Src0.getOperand(0);
40743 SDValue RHS = Src1.isUndef() ? Src1 : Src1.getOperand(0);
40744 SDValue Res =
40745 DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS, V.getOperand(2));
40746 Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res);
40747 return DAG.getBitcast(VT, Res);
40748 }
40749 case X86ISD::VPERMILPI:
40750 // TODO: Handle v4f64 permutes with different low/high lane masks.
40751 if (SrcVT0 == MVT::v4f64) {
40752 uint64_t Mask = Src0.getConstantOperandVal(1);
40753 if ((Mask & 0x3) != ((Mask >> 2) & 0x3))
40754 break;
40755 }
40756 [[fallthrough]];
40757 case X86ISD::VSHLI:
40758 case X86ISD::VSRLI:
40759 case X86ISD::VSRAI:
40760 case X86ISD::PSHUFD:
40761 if (Src1.isUndef() || Src0.getOperand(1) == Src1.getOperand(1)) {
40762 SDValue LHS = Src0.getOperand(0);
40763 SDValue RHS = Src1.isUndef() ? Src1 : Src1.getOperand(0);
40764 SDValue Res = DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT0, LHS, RHS,
40765 V.getOperand(2));
40766 Res = DAG.getNode(SrcOpc0, DL, SrcVT0, Res, Src0.getOperand(1));
40767 return DAG.getBitcast(VT, Res);
40768 }
40769 break;
40770 }
40771
40772 return SDValue();
40773}
40774
40775/// Try to combine x86 target specific shuffles.
40776static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
40777 SelectionDAG &DAG,
40778 TargetLowering::DAGCombinerInfo &DCI,
40779 const X86Subtarget &Subtarget) {
40780 MVT VT = N.getSimpleValueType();
40781 SmallVector<int, 4> Mask;
40782 unsigned Opcode = N.getOpcode();
40783 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40784
40785 if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG))
40786 return R;
40787
40788 // Handle specific target shuffles.
40789 switch (Opcode) {
40790 case X86ISD::MOVDDUP: {
40791 SDValue Src = N.getOperand(0);
40792 // Turn a 128-bit MOVDDUP of a full vector load into movddup+vzload.
40793 if (VT == MVT::v2f64 && Src.hasOneUse() &&
40794 ISD::isNormalLoad(Src.getNode())) {
40795 LoadSDNode *LN = cast<LoadSDNode>(Src);
40796 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::f64, MVT::v2f64, DAG)) {
40797 SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad);
40798 DCI.CombineTo(N.getNode(), Movddup);
40799 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
40800 DCI.recursivelyDeleteUnusedNodes(LN);
40801 return N; // Return N so it doesn't get rechecked!
40802 }
40803 }
40804
40805 return SDValue();
40806 }
40807 case X86ISD::VBROADCAST: {
40808 SDValue Src = N.getOperand(0);
40809 SDValue BC = peekThroughBitcasts(Src);
40810 EVT SrcVT = Src.getValueType();
40811 EVT BCVT = BC.getValueType();
40812
40813 // If broadcasting from another shuffle, attempt to simplify it.
40814 // TODO - we really need a general SimplifyDemandedVectorElts mechanism.
40815 if (isTargetShuffle(BC.getOpcode()) &&
40816 VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits() == 0) {
40817 unsigned Scale = VT.getScalarSizeInBits() / BCVT.getScalarSizeInBits();
40818 SmallVector<int, 16> DemandedMask(BCVT.getVectorNumElements(),
40819 SM_SentinelUndef);
40820 for (unsigned i = 0; i != Scale; ++i)
40821 DemandedMask[i] = i;
40822 if (SDValue Res = combineX86ShufflesRecursively(
40823 {BC}, 0, BC, DemandedMask, {}, /*Depth*/ 0,
40824 X86::MaxShuffleCombineDepth,
40825 /*HasVarMask*/ false, /*AllowCrossLaneVarMask*/ true,
40826 /*AllowPerLaneVarMask*/ true, DAG, Subtarget))
40827 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
40828 DAG.getBitcast(SrcVT, Res));
40829 }
40830
40831 // broadcast(bitcast(src)) -> bitcast(broadcast(src))
40832 // 32-bit targets have to bitcast i64 to f64, so better to bitcast upward.
40833 if (Src.getOpcode() == ISD::BITCAST &&
40834 SrcVT.getScalarSizeInBits() == BCVT.getScalarSizeInBits() &&
40835 TLI.isTypeLegal(BCVT) &&
40836 FixedVectorType::isValidElementType(
40837 BCVT.getScalarType().getTypeForEVT(*DAG.getContext()))) {
40838 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), BCVT.getScalarType(),
40839 VT.getVectorNumElements());
40840 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
40841 }
40842
40843 // vbroadcast(bitcast(vbroadcast(src))) -> bitcast(vbroadcast(src))
40844 // If we're re-broadcasting a smaller type then broadcast with that type and
40845 // bitcast.
40846 // TODO: Do this for any splat?
40847 if (Src.getOpcode() == ISD::BITCAST &&
40848 (BC.getOpcode() == X86ISD::VBROADCAST ||
40849 BC.getOpcode() == X86ISD::VBROADCAST_LOAD) &&
40850 (VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits()) == 0 &&
40851 (VT.getSizeInBits() % BCVT.getSizeInBits()) == 0) {
40852 MVT NewVT =
40853 MVT::getVectorVT(BCVT.getSimpleVT().getScalarType(),
40854 VT.getSizeInBits() / BCVT.getScalarSizeInBits());
40855 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
40856 }
40857
40858 // Reduce broadcast source vector to lowest 128-bits.
40859 if (SrcVT.getSizeInBits() > 128)
40860 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
40861 extract128BitVector(Src, 0, DAG, DL));
40862
40863 // broadcast(scalar_to_vector(x)) -> broadcast(x).
40864 if (Src.getOpcode() == ISD::SCALAR_TO_VECTOR &&
40865 Src.getValueType().getScalarType() == Src.getOperand(0).getValueType())
40866 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
40867
40868 // broadcast(extract_vector_elt(x, 0)) -> broadcast(x).
40869 if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
40870 isNullConstant(Src.getOperand(1)) &&
40871 Src.getValueType() ==
40872 Src.getOperand(0).getValueType().getScalarType() &&
40873 TLI.isTypeLegal(Src.getOperand(0).getValueType()))
40874 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
40875
40876 // Share broadcast with the longest vector and extract low subvector (free).
40877 // Ensure the same SDValue from the SDNode use is being used.
40878 for (SDNode *User : Src->uses())
40879 if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST &&
40880 Src == User->getOperand(0) &&
40881 User->getValueSizeInBits(0).getFixedValue() >
40882 VT.getFixedSizeInBits()) {
40883 return extractSubVector(SDValue(User, 0), 0, DAG, DL,
40884 VT.getSizeInBits());
40885 }
40886
40887 // vbroadcast(scalarload X) -> vbroadcast_load X
40888 // For float loads, extract other uses of the scalar from the broadcast.
40889 if (!SrcVT.isVector() && (Src.hasOneUse() || VT.isFloatingPoint()) &&
40890 ISD::isNormalLoad(Src.getNode())) {
40891 LoadSDNode *LN = cast<LoadSDNode>(Src);
40892 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40893 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
40894 SDValue BcastLd =
40895 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
40896 LN->getMemoryVT(), LN->getMemOperand());
40897 // If the load value is used only by N, replace it via CombineTo N.
40898 bool NoReplaceExtract = Src.hasOneUse();
40899 DCI.CombineTo(N.getNode(), BcastLd);
40900 if (NoReplaceExtract) {
40901 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40902 DCI.recursivelyDeleteUnusedNodes(LN);
40903 } else {
40904 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd,
40905 DAG.getIntPtrConstant(0, DL));
40906 DCI.CombineTo(LN, Scl, BcastLd.getValue(1));
40907 }
40908 return N; // Return N so it doesn't get rechecked!
40909 }
40910
40911 // Due to isTypeDesirableForOp, we won't always shrink a load truncated to
40912 // i16. So shrink it ourselves if we can make a broadcast_load.
40913 if (SrcVT == MVT::i16 && Src.getOpcode() == ISD::TRUNCATE &&
40914 Src.hasOneUse() && Src.getOperand(0).hasOneUse()) {
40915 assert(Subtarget.hasAVX2() && "Expected AVX2");
40916 SDValue TruncIn = Src.getOperand(0);
40917
40918 // If this is a truncate of a non extending load we can just narrow it to
40919 // use a broadcast_load.
40920 if (ISD::isNormalLoad(TruncIn.getNode())) {
40921 LoadSDNode *LN = cast<LoadSDNode>(TruncIn);
40922 // Unless its volatile or atomic.
40923 if (LN->isSimple()) {
40924 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40925 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
40926 SDValue BcastLd = DAG.getMemIntrinsicNode(
40927 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
40928 LN->getPointerInfo(), LN->getOriginalAlign(),
40929 LN->getMemOperand()->getFlags());
40930 DCI.CombineTo(N.getNode(), BcastLd);
40931 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40932 DCI.recursivelyDeleteUnusedNodes(Src.getNode());
40933 return N; // Return N so it doesn't get rechecked!
40934 }
40935 }
40936
40937 // If this is a truncate of an i16 extload, we can directly replace it.
40938 if (ISD::isUNINDEXEDLoad(Src.getOperand(0).getNode()) &&
40939 ISD::isEXTLoad(Src.getOperand(0).getNode())) {
40940 LoadSDNode *LN = cast<LoadSDNode>(Src.getOperand(0));
40941 if (LN->getMemoryVT().getSizeInBits() == 16) {
40942 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40943 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
40944 SDValue BcastLd =
40945 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
40946 LN->getMemoryVT(), LN->getMemOperand());
40947 DCI.CombineTo(N.getNode(), BcastLd);
40948 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40949 DCI.recursivelyDeleteUnusedNodes(Src.getNode());
40950 return N; // Return N so it doesn't get rechecked!
40951 }
40952 }
40953
40954 // If this is a truncate of load that has been shifted right, we can
40955 // offset the pointer and use a narrower load.
40956 if (TruncIn.getOpcode() == ISD::SRL &&
40957 TruncIn.getOperand(0).hasOneUse() &&
40958 isa<ConstantSDNode>(TruncIn.getOperand(1)) &&
40959 ISD::isNormalLoad(TruncIn.getOperand(0).getNode())) {
40960 LoadSDNode *LN = cast<LoadSDNode>(TruncIn.getOperand(0));
40961 unsigned ShiftAmt = TruncIn.getConstantOperandVal(1);
40962 // Make sure the shift amount and the load size are divisible by 16.
40963 // Don't do this if the load is volatile or atomic.
40964 if (ShiftAmt % 16 == 0 && TruncIn.getValueSizeInBits() % 16 == 0 &&
40965 LN->isSimple()) {
40966 unsigned Offset = ShiftAmt / 8;
40967 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40968 SDValue Ptr = DAG.getMemBasePlusOffset(
40969 LN->getBasePtr(), TypeSize::getFixed(Offset), DL);
40970 SDValue Ops[] = { LN->getChain(), Ptr };
40971 SDValue BcastLd = DAG.getMemIntrinsicNode(
40972 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,
40973 LN->getPointerInfo().getWithOffset(Offset),
40974 LN->getOriginalAlign(),
40975 LN->getMemOperand()->getFlags());
40976 DCI.CombineTo(N.getNode(), BcastLd);
40977 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40978 DCI.recursivelyDeleteUnusedNodes(Src.getNode());
40979 return N; // Return N so it doesn't get rechecked!
40980 }
40981 }
40982 }
40983
40984 // vbroadcast(vzload X) -> vbroadcast_load X
40985 if (Src.getOpcode() == X86ISD::VZEXT_LOAD && Src.hasOneUse()) {
40986 MemSDNode *LN = cast<MemIntrinsicSDNode>(Src);
40987 if (LN->getMemoryVT().getSizeInBits() == VT.getScalarSizeInBits()) {
40988 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
40989 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
40990 SDValue BcastLd =
40991 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
40992 LN->getMemoryVT(), LN->getMemOperand());
40993 DCI.CombineTo(N.getNode(), BcastLd);
40994 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
40995 DCI.recursivelyDeleteUnusedNodes(LN);
40996 return N; // Return N so it doesn't get rechecked!
40997 }
40998 }
40999
41000 // vbroadcast(vector load X) -> vbroadcast_load
41001 if ((SrcVT == MVT::v2f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v2i64 ||
41002 SrcVT == MVT::v4i32) &&
41003 Src.hasOneUse() && ISD::isNormalLoad(Src.getNode())) {
41004 LoadSDNode *LN = cast<LoadSDNode>(Src);
41005 // Unless the load is volatile or atomic.
41006 if (LN->isSimple()) {
41007 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
41008 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
41009 SDValue BcastLd = DAG.getMemIntrinsicNode(
41010 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SrcVT.getScalarType(),
41011 LN->getPointerInfo(), LN->getOriginalAlign(),
41012 LN->getMemOperand()->getFlags());
41013 DCI.CombineTo(N.getNode(), BcastLd);
41014 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
41015 DCI.recursivelyDeleteUnusedNodes(LN);
41016 return N; // Return N so it doesn't get rechecked!
41017 }
41018 }
41019
41020 return SDValue();
41021 }
41022 case X86ISD::VZEXT_MOVL: {
41023 SDValue N0 = N.getOperand(0);
41024
41025 // If this a vzmovl of a full vector load, replace it with a vzload, unless
41026 // the load is volatile.
41027 if (N0.hasOneUse() && ISD::isNormalLoad(N0.getNode())) {
41028 auto *LN = cast<LoadSDNode>(N0);
41029 if (SDValue VZLoad =
41030 narrowLoadToVZLoad(LN, VT.getVectorElementType(), VT, DAG)) {
41031 DCI.CombineTo(N.getNode(), VZLoad);
41032 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
41033 DCI.recursivelyDeleteUnusedNodes(LN);
41034 return N;
41035 }
41036 }
41037
41038 // If this a VZEXT_MOVL of a VBROADCAST_LOAD, we don't need the broadcast
41039 // and can just use a VZEXT_LOAD.
41040 // FIXME: Is there some way to do this with SimplifyDemandedVectorElts?
41041 if (N0.hasOneUse() && N0.getOpcode() == X86ISD::VBROADCAST_LOAD) {
41042 auto *LN = cast<MemSDNode>(N0);
41043 if (VT.getScalarSizeInBits() == LN->getMemoryVT().getSizeInBits()) {
41044 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
41045 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
41046 SDValue VZLoad =
41047 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops,
41048 LN->getMemoryVT(), LN->getMemOperand());
41049 DCI.CombineTo(N.getNode(), VZLoad);
41050 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
41051 DCI.recursivelyDeleteUnusedNodes(LN);
41052 return N;
41053 }
41054 }
41055
41056 // Turn (v2i64 (vzext_movl (scalar_to_vector (i64 X)))) into
41057 // (v2i64 (bitcast (v4i32 (vzext_movl (scalar_to_vector (i32 (trunc X)))))))
41058 // if the upper bits of the i64 are zero.
41059 if (N0.hasOneUse() && N0.getOpcode() == ISD::SCALAR_TO_VECTOR &&
41060 N0.getOperand(0).hasOneUse() &&
41061 N0.getOperand(0).getValueType() == MVT::i64) {
41062 SDValue In = N0.getOperand(0);
41063 APInt Mask = APInt::getHighBitsSet(64, 32);
41064 if (DAG.MaskedValueIsZero(In, Mask)) {
41065 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In);
41066 MVT VecVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);
41067 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Trunc);
41068 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, SclVec);
41069 return DAG.getBitcast(VT, Movl);
41070 }
41071 }
41072
41073 // Load a scalar integer constant directly to XMM instead of transferring an
41074 // immediate value from GPR.
41075 // vzext_movl (scalar_to_vector C) --> load [C,0...]
41076 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR) {
41077 if (auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
41078 // Create a vector constant - scalar constant followed by zeros.
41079 EVT ScalarVT = N0.getOperand(0).getValueType();
41080 Type *ScalarTy = ScalarVT.getTypeForEVT(*DAG.getContext());
41081 unsigned NumElts = VT.getVectorNumElements();
41082 Constant *Zero = ConstantInt::getNullValue(ScalarTy);
41083 SmallVector<Constant *, 32> ConstantVec(NumElts, Zero);
41084 ConstantVec[0] = const_cast<ConstantInt *>(C->getConstantIntValue());
41085
41086 // Load the vector constant from constant pool.
41087 MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
41088 SDValue CP = DAG.getConstantPool(ConstantVector::get(ConstantVec), PVT);
41089 MachinePointerInfo MPI =
41090 MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
41091 Align Alignment = cast<ConstantPoolSDNode>(CP)->getAlign();
41092 return DAG.getLoad(VT, DL, DAG.getEntryNode(), CP, MPI, Alignment,
41093 MachineMemOperand::MOLoad);
41094 }
41095 }
41096
41097 // Pull subvector inserts into undef through VZEXT_MOVL by making it an
41098 // insert into a zero vector. This helps get VZEXT_MOVL closer to
41099 // scalar_to_vectors where 256/512 are canonicalized to an insert and a
41100 // 128-bit scalar_to_vector. This reduces the number of isel patterns.
41101 if (!DCI.isBeforeLegalizeOps() && N0.hasOneUse()) {
41102 SDValue V = peekThroughOneUseBitcasts(N0);
41103
41104 if (V.getOpcode() == ISD::INSERT_SUBVECTOR && V.getOperand(0).isUndef() &&
41105 isNullConstant(V.getOperand(2))) {
41106 SDValue In = V.getOperand(1);
41107 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(),
41108 In.getValueSizeInBits() /
41109 VT.getScalarSizeInBits());
41110 In = DAG.getBitcast(SubVT, In);
41111 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, SubVT, In);
41112 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
41113 getZeroVector(VT, Subtarget, DAG, DL), Movl,
41114 V.getOperand(2));
41115 }
41116 }
41117
41118 return SDValue();
41119 }
41120 case X86ISD::BLENDI: {
41121 SDValue N0 = N.getOperand(0);
41122 SDValue N1 = N.getOperand(1);
41123 unsigned EltBits = VT.getScalarSizeInBits();
41124
41125 if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) {
41126 // blend(bitcast(x),bitcast(y)) -> bitcast(blend(x,y)) to narrower types.
41127 // TODO: Handle MVT::v16i16 repeated blend mask.
41128 if (N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType()) {
41129 MVT SrcVT = N0.getOperand(0).getSimpleValueType();
41130 unsigned SrcBits = SrcVT.getScalarSizeInBits();
41131 if ((EltBits % SrcBits) == 0 && SrcBits >= 32) {
41132 unsigned Size = VT.getVectorNumElements();
41133 unsigned NewSize = SrcVT.getVectorNumElements();
41134 APInt BlendMask = N.getConstantOperandAPInt(2).zextOrTrunc(Size);
41135 APInt NewBlendMask = APIntOps::ScaleBitMask(BlendMask, NewSize);
41136 return DAG.getBitcast(
41137 VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0),
41138 N1.getOperand(0),
41139 DAG.getTargetConstant(NewBlendMask.getZExtValue(),
41140 DL, MVT::i8)));
41141 }
41142 }
41143 // Share PSHUFB masks:
41144 // blend(pshufb(x,m1),pshufb(y,m2))
41145 // --> m3 = blend(m1,m2)
41146 // blend(pshufb(x,m3),pshufb(y,m3))
41147 if (N0.hasOneUse() && N1.hasOneUse()) {
41148 SmallVector<int> Mask, ByteMask;
41149 SmallVector<SDValue> Ops;
41150 SDValue LHS = peekThroughOneUseBitcasts(N0);
41151 SDValue RHS = peekThroughOneUseBitcasts(N1);
41152 if (LHS.getOpcode() == X86ISD::PSHUFB &&
41153 RHS.getOpcode() == X86ISD::PSHUFB &&
41154 LHS.getOperand(1) != RHS.getOperand(1) &&
41155 LHS.getOperand(1).hasOneUse() && RHS.getOperand(1).hasOneUse() &&
41156 getTargetShuffleMask(N, /*AllowSentinelZero=*/false, Ops, Mask)) {
41157 assert(Ops.size() == 2 && LHS == peekThroughOneUseBitcasts(Ops[0]) &&
41158 RHS == peekThroughOneUseBitcasts(Ops[1]) &&
41159 "BLENDI decode mismatch");
41160 MVT ShufVT = LHS.getSimpleValueType();
41161 SDValue MaskLHS = LHS.getOperand(1);
41162 SDValue MaskRHS = RHS.getOperand(1);
41163 llvm::narrowShuffleMaskElts(EltBits / 8, Mask, ByteMask);
41164 if (SDValue NewMask = combineX86ShufflesConstants(
41165 ShufVT, {MaskLHS, MaskRHS}, ByteMask,
41166 /*HasVariableMask=*/true, DAG, DL, Subtarget)) {
41167 SDValue NewLHS = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT,
41168 LHS.getOperand(0), NewMask);
41169 SDValue NewRHS = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT,
41170 RHS.getOperand(0), NewMask);
41171 return DAG.getNode(X86ISD::BLENDI, DL, VT,
41172 DAG.getBitcast(VT, NewLHS),
41173 DAG.getBitcast(VT, NewRHS), N.getOperand(2));
41174 }
41175 }
41176 }
41177 }
41178 return SDValue();
41179 }
41180 case X86ISD::SHUFP: {
41181 // Fold shufps(shuffle(x),shuffle(y)) -> shufps(x,y).
41182 // This is a more relaxed shuffle combiner that can ignore oneuse limits.
41183 // TODO: Support types other than v4f32.
41184 if (VT == MVT::v4f32) {
41185 bool Updated = false;
41186 SmallVector<int> Mask;
41187 SmallVector<SDValue> Ops;
41188 if (getTargetShuffleMask(N, false, Ops, Mask) && Ops.size() == 2) {
41189 for (int i = 0; i != 2; ++i) {
41190 SmallVector<SDValue> SubOps;
41191 SmallVector<int> SubMask, SubScaledMask;
41192 SDValue Sub = peekThroughBitcasts(Ops[i]);
41193 // TODO: Scaling might be easier if we specify the demanded elts.
41194 if (getTargetShuffleInputs(Sub, SubOps, SubMask, DAG, 0, false) &&
41195 scaleShuffleElements(SubMask, 4, SubScaledMask) &&
41196 SubOps.size() == 1 && isUndefOrInRange(SubScaledMask, 0, 4)) {
41197 int Ofs = i * 2;
41198 Mask[Ofs + 0] = SubScaledMask[Mask[Ofs + 0] % 4] + (i * 4);
41199 Mask[Ofs + 1] = SubScaledMask[Mask[Ofs + 1] % 4] + (i * 4);
41200 Ops[i] = DAG.getBitcast(VT, SubOps[0]);
41201 Updated = true;
41202 }
41203 }
41204 }
41205 if (Updated) {
41206 for (int &M : Mask)
41207 M %= 4;
41208 Ops.push_back(getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
41209 return DAG.getNode(X86ISD::SHUFP, DL, VT, Ops);
41210 }
41211 }
41212 return SDValue();
41213 }
41214 case X86ISD::VPERMI: {
41215 // vpermi(bitcast(x)) -> bitcast(vpermi(x)) for same number of elements.
41216 // TODO: Remove when we have preferred domains in combineX86ShuffleChain.
41217 SDValue N0 = N.getOperand(0);
41218 SDValue N1 = N.getOperand(1);
41219 unsigned EltSizeInBits = VT.getScalarSizeInBits();
41220 if (N0.getOpcode() == ISD::BITCAST &&
41221 N0.getOperand(0).getScalarValueSizeInBits() == EltSizeInBits) {
41222 SDValue Src = N0.getOperand(0);
41223 EVT SrcVT = Src.getValueType();
41224 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1);
41225 return DAG.getBitcast(VT, Res);
41226 }
41227 return SDValue();
41228 }
41229 case X86ISD::SHUF128: {
41230 // If we're permuting the upper 256-bits subvectors of a concatenation, then
41231 // see if we can peek through and access the subvector directly.
41232 if (VT.is512BitVector()) {
41233 // 512-bit mask uses 4 x i2 indices - if the msb is always set then only the
41234 // upper subvector is used.
41235 SDValue LHS = N->getOperand(0);
41236 SDValue RHS = N->getOperand(1);
41237 uint64_t Mask = N->getConstantOperandVal(2);
41238 SmallVector<SDValue> LHSOps, RHSOps;
41239 SDValue NewLHS, NewRHS;
41240 if ((Mask & 0x0A) == 0x0A &&
41241 collectConcatOps(LHS.getNode(), LHSOps, DAG) && LHSOps.size() == 2) {
41242 NewLHS = widenSubVector(LHSOps[1], false, Subtarget, DAG, DL, 512);
41243 Mask &= ~0x0A;
41244 }
41245 if ((Mask & 0xA0) == 0xA0 &&
41246 collectConcatOps(RHS.getNode(), RHSOps, DAG) && RHSOps.size() == 2) {
41247 NewRHS = widenSubVector(RHSOps[1], false, Subtarget, DAG, DL, 512);
41248 Mask &= ~0xA0;
41249 }
41250 if (NewLHS || NewRHS)
41251 return DAG.getNode(X86ISD::SHUF128, DL, VT, NewLHS ? NewLHS : LHS,
41252 NewRHS ? NewRHS : RHS,
41253 DAG.getTargetConstant(Mask, DL, MVT::i8));
41254 }
41255 return SDValue();
41256 }
41257 case X86ISD::VPERM2X128: {
41258 // Fold vperm2x128(bitcast(x),bitcast(y),c) -> bitcast(vperm2x128(x,y,c)).
41259 SDValue LHS = N->getOperand(0);
41260 SDValue RHS = N->getOperand(1);
41261 if (LHS.getOpcode() == ISD::BITCAST &&
41262 (RHS.getOpcode() == ISD::BITCAST || RHS.isUndef())) {
41263 EVT SrcVT = LHS.getOperand(0).getValueType();
41264 if (RHS.isUndef() || SrcVT == RHS.getOperand(0).getValueType()) {
41265 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT,
41266 DAG.getBitcast(SrcVT, LHS),
41267 DAG.getBitcast(SrcVT, RHS),
41268 N->getOperand(2)));
41269 }
41270 }
41271
41272 // Fold vperm2x128(op(),op()) -> op(vperm2x128(),vperm2x128()).
41273 if (SDValue Res = canonicalizeLaneShuffleWithRepeatedOps(N, DAG, DL))
41274 return Res;
41275
41276 // Fold vperm2x128 subvector shuffle with an inner concat pattern.
41277 // vperm2x128(concat(X,Y),concat(Z,W)) --> concat X,Y etc.
41278 auto FindSubVector128 = [&](unsigned Idx) {
41279 if (Idx > 3)
41280 return SDValue();
41281 SDValue Src = peekThroughBitcasts(N.getOperand(Idx < 2 ? 0 : 1));
41282 SmallVector<SDValue> SubOps;
41283 if (collectConcatOps(Src.getNode(), SubOps, DAG) && SubOps.size() == 2)
41284 return SubOps[Idx & 1];
41285 unsigned NumElts = Src.getValueType().getVectorNumElements();
41286 if ((Idx & 1) == 1 && Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
41287 Src.getOperand(1).getValueSizeInBits() == 128 &&
41288 Src.getConstantOperandAPInt(2) == (NumElts / 2)) {
41289 return Src.getOperand(1);
41290 }
41291 return SDValue();
41292 };
41293 unsigned Imm = N.getConstantOperandVal(2);
41294 if (SDValue SubLo = FindSubVector128(Imm & 0x0F)) {
41295 if (SDValue SubHi = FindSubVector128((Imm & 0xF0) >> 4)) {
41296 MVT SubVT = VT.getHalfNumVectorElementsVT();
41297 SubLo = DAG.getBitcast(SubVT, SubLo);
41298 SubHi = DAG.getBitcast(SubVT, SubHi);
41299 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi);
41300 }
41301 }
41302 return SDValue();
41303 }
41304 case X86ISD::PSHUFD:
41305 case X86ISD::PSHUFLW:
41306 case X86ISD::PSHUFHW: {
41307 SDValue N0 = N.getOperand(0);
41308 SDValue N1 = N.getOperand(1);
41309 if (N0->hasOneUse()) {
41310 SDValue V = peekThroughOneUseBitcasts(N0);
41311 switch (V.getOpcode()) {
41312 case X86ISD::VSHL:
41313 case X86ISD::VSRL:
41314 case X86ISD::VSRA:
41315 case X86ISD::VSHLI:
41316 case X86ISD::VSRLI:
41317 case X86ISD::VSRAI:
41318 case X86ISD::VROTLI:
41319 case X86ISD::VROTRI: {
41320 MVT InnerVT = V.getSimpleValueType();
41321 if (InnerVT.getScalarSizeInBits() <= VT.getScalarSizeInBits()) {
41322 SDValue Res = DAG.getNode(Opcode, DL, VT,
41323 DAG.getBitcast(VT, V.getOperand(0)), N1);
41324 Res = DAG.getBitcast(InnerVT, Res);
41325 Res = DAG.getNode(V.getOpcode(), DL, InnerVT, Res, V.getOperand(1));
41326 return DAG.getBitcast(VT, Res);
41327 }
41328 break;
41329 }
41330 }
41331 }
41332
41333 Mask = getPSHUFShuffleMask(N);
41334 assert(Mask.size() == 4);
41335 break;
41336 }
41337 case X86ISD::MOVSD:
41338 case X86ISD::MOVSH:
41339 case X86ISD::MOVSS: {
41340 SDValue N0 = N.getOperand(0);
41341 SDValue N1 = N.getOperand(1);
41342
41343 // Canonicalize scalar FPOps:
41344 // MOVS*(N0, OP(N0, N1)) --> MOVS*(N0, SCALAR_TO_VECTOR(OP(N0[0], N1[0])))
41345 // If commutable, allow OP(N1[0], N0[0]).
41346 unsigned Opcode1 = N1.getOpcode();
41347 if (Opcode1 == ISD::FADD || Opcode1 == ISD::FMUL || Opcode1 == ISD::FSUB ||
41348 Opcode1 == ISD::FDIV) {
41349 SDValue N10 = N1.getOperand(0);
41350 SDValue N11 = N1.getOperand(1);
41351 if (N10 == N0 ||
41352 (N11 == N0 && (Opcode1 == ISD::FADD || Opcode1 == ISD::FMUL))) {
41353 if (N10 != N0)
41354 std::swap(N10, N11);
41355 MVT SVT = VT.getVectorElementType();
41356 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
41357 N10 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N10, ZeroIdx);
41358 N11 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N11, ZeroIdx);
41359 SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11);
41360 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
41361 return DAG.getNode(Opcode, DL, VT, N0, SclVec);
41362 }
41363 }
41364
41365 return SDValue();
41366 }
41367 case X86ISD::INSERTPS: {
41368 assert(VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32");
41369 SDValue Op0 = N.getOperand(0);
41370 SDValue Op1 = N.getOperand(1);
41371 unsigned InsertPSMask = N.getConstantOperandVal(2);
41372 unsigned SrcIdx = (InsertPSMask >> 6) & 0x3;
41373 unsigned DstIdx = (InsertPSMask >> 4) & 0x3;
41374 unsigned ZeroMask = InsertPSMask & 0xF;
41375
41376 // If we zero out all elements from Op0 then we don't need to reference it.
41377 if (((ZeroMask | (1u << DstIdx)) == 0xF) && !Op0.isUndef())
41378 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,
41379 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
41380
41381 // If we zero out the element from Op1 then we don't need to reference it.
41382 if ((ZeroMask & (1u << DstIdx)) && !Op1.isUndef())
41383 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
41384 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
41385
41386 // Attempt to merge insertps Op1 with an inner target shuffle node.
41387 SmallVector<int, 8> TargetMask1;
41388 SmallVector<SDValue, 2> Ops1;
41389 APInt KnownUndef1, KnownZero1;
41390 if (getTargetShuffleAndZeroables(Op1, TargetMask1, Ops1, KnownUndef1,
41391 KnownZero1)) {
41392 if (KnownUndef1[SrcIdx] || KnownZero1[SrcIdx]) {
41393 // Zero/UNDEF insertion - zero out element and remove dependency.
41394 InsertPSMask |= (1u << DstIdx);
41395 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
41396 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
41397 }
41398 // Update insertps mask srcidx and reference the source input directly.
41399 int M = TargetMask1[SrcIdx];
41400 assert(0 <= M && M < 8 && "Shuffle index out of range");
41401 InsertPSMask = (InsertPSMask & 0x3f) | ((M & 0x3) << 6);
41402 Op1 = Ops1[M < 4 ? 0 : 1];
41403 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
41404 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
41405 }
41406
41407 // Attempt to merge insertps Op0 with an inner target shuffle node.
41408 SmallVector<int, 8> TargetMask0;
41409 SmallVector<SDValue, 2> Ops0;
41410 APInt KnownUndef0, KnownZero0;
41411 if (getTargetShuffleAndZeroables(Op0, TargetMask0, Ops0, KnownUndef0,
41412 KnownZero0)) {
41413 bool Updated = false;
41414 bool UseInput00 = false;
41415 bool UseInput01 = false;
41416 for (int i = 0; i != 4; ++i) {
41417 if ((InsertPSMask & (1u << i)) || (i == (int)DstIdx)) {
41418 // No change if element is already zero or the inserted element.
41419 continue;
41420 }
41421
41422 if (KnownUndef0[i] || KnownZero0[i]) {
41423 // If the target mask is undef/zero then we must zero the element.
41424 InsertPSMask |= (1u << i);
41425 Updated = true;
41426 continue;
41427 }
41428
41429 // The input vector element must be inline.
41430 int M = TargetMask0[i];
41431 if (M != i && M != (i + 4))
41432 return SDValue();
41433
41434 // Determine which inputs of the target shuffle we're using.
41435 UseInput00 |= (0 <= M && M < 4);
41436 UseInput01 |= (4 <= M);
41437 }
41438
41439 // If we're not using both inputs of the target shuffle then use the
41440 // referenced input directly.
41441 if (UseInput00 && !UseInput01) {
41442 Updated = true;
41443 Op0 = Ops0[0];
41444 } else if (!UseInput00 && UseInput01) {
41445 Updated = true;
41446 Op0 = Ops0[1];
41447 }
41448
41449 if (Updated)
41450 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
41451 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
41452 }
41453
41454 // If we're inserting an element from a vbroadcast load, fold the
41455 // load into the X86insertps instruction. We need to convert the scalar
41456 // load to a vector and clear the source lane of the INSERTPS control.
41457 if (Op1.getOpcode() == X86ISD::VBROADCAST_LOAD && Op1.hasOneUse()) {
41458 auto *MemIntr = cast<MemIntrinsicSDNode>(Op1);
41459 if (MemIntr->getMemoryVT().getScalarSizeInBits() == 32) {
41460 SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
41461 MemIntr->getBasePtr(),
41462 MemIntr->getMemOperand());
41463 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
41464 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
41465 Load),
41466 DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
41467 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
41468 return Insert;
41469 }
41470 }
41471
41472 return SDValue();
41473 }
41474 case X86ISD::VPERMV3: {
41475 // Combine VPERMV3 to widened VPERMV if the two source operands are split
41476 // from the same vector.
41477 SDValue V1 = peekThroughBitcasts(N.getOperand(0));
41478 SDValue V2 = peekThroughBitcasts(N.getOperand(2));
41479 MVT SVT = V1.getSimpleValueType();
41480 if (V1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
41481 V1.getConstantOperandVal(1) == 0 &&
41482 V2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
41483 V2.getConstantOperandVal(1) == SVT.getVectorNumElements() &&
41484 V1.getOperand(0) == V2.getOperand(0)) {
41485 EVT NVT = V1.getOperand(0).getValueType();
41486 if (NVT.is256BitVector() ||
41487 (NVT.is512BitVector() && Subtarget.hasEVEX512())) {
41488 MVT WideVT = MVT::getVectorVT(
41489 VT.getScalarType(), NVT.getSizeInBits() / VT.getScalarSizeInBits());
41490 SDValue Mask = widenSubVector(N.getOperand(1), false, Subtarget, DAG,
41491 DL, WideVT.getSizeInBits());
41492 SDValue Perm = DAG.getNode(X86ISD::VPERMV, DL, WideVT, Mask,
41493 DAG.getBitcast(WideVT, V1.getOperand(0)));
41494 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Perm,
41495 DAG.getIntPtrConstant(0, DL));
41496 }
41497 }
41498 return SDValue();
41499 }
41500 default:
41501 return SDValue();
41502 }
41503
41504 // Nuke no-op shuffles that show up after combining.
41505 if (isNoopShuffleMask(Mask))
41506 return N.getOperand(0);
41507
41508 // Look for simplifications involving one or two shuffle instructions.
41509 SDValue V = N.getOperand(0);
41510 switch (N.getOpcode()) {
41511 default:
41512 break;
41513 case X86ISD::PSHUFLW:
41514 case X86ISD::PSHUFHW:
41515 assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!");
41516
41517 // See if this reduces to a PSHUFD which is no more expensive and can
41518 // combine with more operations. Note that it has to at least flip the
41519 // dwords as otherwise it would have been removed as a no-op.
41520 if (ArrayRef<int>(Mask).equals({2, 3, 0, 1})) {
41521 int DMask[] = {0, 1, 2, 3};
41522 int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
41523 DMask[DOffset + 0] = DOffset + 1;
41524 DMask[DOffset + 1] = DOffset + 0;
41525 MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);
41526 V = DAG.getBitcast(DVT, V);
41527 V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V,
41528 getV4X86ShuffleImm8ForMask(DMask, DL, DAG));
41529 return DAG.getBitcast(VT, V);
41530 }
41531
41532 // Look for shuffle patterns which can be implemented as a single unpack.
41533 // FIXME: This doesn't handle the location of the PSHUFD generically, and
41534 // only works when we have a PSHUFD followed by two half-shuffles.
41535 if (Mask[0] == Mask[1] && Mask[2] == Mask[3] &&
41536 (V.getOpcode() == X86ISD::PSHUFLW ||
41537 V.getOpcode() == X86ISD::PSHUFHW) &&
41538 V.getOpcode() != N.getOpcode() &&
41539 V.hasOneUse() && V.getOperand(0).hasOneUse()) {
41540 SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
41541 if (D.getOpcode() == X86ISD::PSHUFD) {
41542 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
41543 SmallVector<int, 4> DMask = getPSHUFShuffleMask(D);
41544 int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
41545 int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
41546 int WordMask[8];
41547 for (int i = 0; i < 4; ++i) {
41548 WordMask[i + NOffset] = Mask[i] + NOffset;
41549 WordMask[i + VOffset] = VMask[i] + VOffset;
41550 }
41551 // Map the word mask through the DWord mask.
41552 int MappedMask[8];
41553 for (int i = 0; i < 8; ++i)
41554 MappedMask[i] = 2 * DMask[WordMask[i] / 2] + WordMask[i] % 2;
41555 if (ArrayRef<int>(MappedMask).equals({0, 0, 1, 1, 2, 2, 3, 3}) ||
41556 ArrayRef<int>(MappedMask).equals({4, 4, 5, 5, 6, 6, 7, 7})) {
41557 // We can replace all three shuffles with an unpack.
41558 V = DAG.getBitcast(VT, D.getOperand(0));
41559 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
41560 : X86ISD::UNPCKH,
41561 DL, VT, V, V);
41562 }
41563 }
41564 }
41565
41566 break;
41567
41568 case X86ISD::PSHUFD:
41569 if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DL, DAG))
41570 return NewN;
41571
41572 break;
41573 }
41574
41575 return SDValue();
41576}
41577
41578/// Checks if the shuffle mask takes subsequent elements
41579/// alternately from two vectors.
41580/// For example <0, 5, 2, 7> or <8, 1, 10, 3, 12, 5, 14, 7> are both correct.
41581static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
41582
41583 int ParitySrc[2] = {-1, -1};
41584 unsigned Size = Mask.size();
41585 for (unsigned i = 0; i != Size; ++i) {
41586 int M = Mask[i];
41587 if (M < 0)
41588 continue;
41589
41590 // Make sure we are using the matching element from the input.
41591 if ((M % Size) != i)
41592 return false;
41593
41594 // Make sure we use the same input for all elements of the same parity.
41595 int Src = M / Size;
41596 if (ParitySrc[i % 2] >= 0 && ParitySrc[i % 2] != Src)
41597 return false;
41598 ParitySrc[i % 2] = Src;
41599 }
41600
41601 // Make sure each input is used.
41602 if (ParitySrc[0] < 0 || ParitySrc[1] < 0 || ParitySrc[0] == ParitySrc[1])
41603 return false;
41604
41605 Op0Even = ParitySrc[0] == 0;
41606 return true;
41607}
41608
41609/// Returns true iff the shuffle node \p N can be replaced with ADDSUB(SUBADD)
41610/// operation. If true is returned then the operands of ADDSUB(SUBADD) operation
41611/// are written to the parameters \p Opnd0 and \p Opnd1.
41612///
41613/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes
41614/// so it is easier to generically match. We also insert dummy vector shuffle
41615/// nodes for the operands which explicitly discard the lanes which are unused
41616/// by this operation to try to flow through the rest of the combiner
41617/// the fact that they're unused.
41618static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
41619 SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
41620 bool &IsSubAdd) {
41621
41622 EVT VT = N->getValueType(0);
41623 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41624 if (!Subtarget.hasSSE3() || !TLI.isTypeLegal(VT) ||
41625 !VT.getSimpleVT().isFloatingPoint())
41626 return false;
41627
41628 // We only handle target-independent shuffles.
41629 // FIXME: It would be easy and harmless to use the target shuffle mask
41630 // extraction tool to support more.
41631 if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
41632 return false;
41633
41634 SDValue V1 = N->getOperand(0);
41635 SDValue V2 = N->getOperand(1);
41636
41637 // Make sure we have an FADD and an FSUB.
41638 if ((V1.getOpcode() != ISD::FADD && V1.getOpcode() != ISD::FSUB) ||
41639 (V2.getOpcode() != ISD::FADD && V2.getOpcode() != ISD::FSUB) ||
41640 V1.getOpcode() == V2.getOpcode())
41641 return false;
41642
41643 // If there are other uses of these operations we can't fold them.
41644 if (!V1->hasOneUse() || !V2->hasOneUse())
41645 return false;
41646
41647 // Ensure that both operations have the same operands. Note that we can
41648 // commute the FADD operands.
41649 SDValue LHS, RHS;
41650 if (V1.getOpcode() == ISD::FSUB) {
41651 LHS = V1->getOperand(0); RHS = V1->getOperand(1);
41652 if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
41653 (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
41654 return false;
41655 } else {
41656 assert(V2.getOpcode() == ISD::FSUB && "Unexpected opcode");
41657 LHS = V2->getOperand(0); RHS = V2->getOperand(1);
41658 if ((V1->getOperand(0) != LHS || V1->getOperand(1) != RHS) &&
41659 (V1->getOperand(0) != RHS || V1->getOperand(1) != LHS))
41660 return false;
41661 }
41662
41663 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
41664 bool Op0Even;
41665 if (!isAddSubOrSubAddMask(Mask, Op0Even))
41666 return false;
41667
41668 // It's a subadd if the vector in the even parity is an FADD.
41669 IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
41670 : V2->getOpcode() == ISD::FADD;
41671
41672 Opnd0 = LHS;
41673 Opnd1 = RHS;
41674 return true;
41675}
41676
41677/// Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
41678static SDValue combineShuffleToFMAddSub(SDNode *N, const SDLoc &DL,
41679 const X86Subtarget &Subtarget,
41680 SelectionDAG &DAG) {
41681 // We only handle target-independent shuffles.
41682 // FIXME: It would be easy and harmless to use the target shuffle mask
41683 // extraction tool to support more.
41684 if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
41685 return SDValue();
41686
41687 MVT VT = N->getSimpleValueType(0);
41688 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41689 if (!Subtarget.hasAnyFMA() || !TLI.isTypeLegal(VT))
41690 return SDValue();
41691
41692 // We're trying to match (shuffle fma(a, b, c), X86Fmsub(a, b, c).
41693 SDValue Op0 = N->getOperand(0);
41694 SDValue Op1 = N->getOperand(1);
41695 SDValue FMAdd = Op0, FMSub = Op1;
41696 if (FMSub.getOpcode() != X86ISD::FMSUB)
41697 std::swap(FMAdd, FMSub);
41698
41699 if (FMAdd.getOpcode() != ISD::FMA || FMSub.getOpcode() != X86ISD::FMSUB ||
41700 FMAdd.getOperand(0) != FMSub.getOperand(0) || !FMAdd.hasOneUse() ||
41701 FMAdd.getOperand(1) != FMSub.getOperand(1) || !FMSub.hasOneUse() ||
41702 FMAdd.getOperand(2) != FMSub.getOperand(2))
41703 return SDValue();
41704
41705 // Check for correct shuffle mask.
41706 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
41707 bool Op0Even;
41708 if (!isAddSubOrSubAddMask(Mask, Op0Even))
41709 return SDValue();
41710
41711 // FMAddSub takes zeroth operand from FMSub node.
41712 bool IsSubAdd = Op0Even ? Op0 == FMAdd : Op1 == FMAdd;
41713 unsigned Opcode = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
41714 return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1),
41715 FMAdd.getOperand(2));
41716}
41717
41718/// Try to combine a shuffle into a target-specific add-sub or
41719/// mul-add-sub node.
41720static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N, const SDLoc &DL,
41721 const X86Subtarget &Subtarget,
41722 SelectionDAG &DAG) {
41723 if (SDValue V = combineShuffleToFMAddSub(N, DL, Subtarget, DAG))
41724 return V;
41725
41726 SDValue Opnd0, Opnd1;
41727 bool IsSubAdd;
41728 if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd))
41729 return SDValue();
41730
41731 MVT VT = N->getSimpleValueType(0);
41732
41733 // Try to generate X86ISD::FMADDSUB node here.
41734 SDValue Opnd2;
41735 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2)) {
41736 unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
41737 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
41738 }
41739
41740 if (IsSubAdd)
41741 return SDValue();
41742
41743 // Do not generate X86ISD::ADDSUB node for 512-bit types even though
41744 // the ADDSUB idiom has been successfully recognized. There are no known
41745 // X86 targets with 512-bit ADDSUB instructions!
41746 if (VT.is512BitVector())
41747 return SDValue();
41748
41749 // Do not generate X86ISD::ADDSUB node for FP16's vector types even though
41750 // the ADDSUB idiom has been successfully recognized. There are no known
41751 // X86 targets with FP16 ADDSUB instructions!
41752 if (VT.getVectorElementType() == MVT::f16)
41753 return SDValue();
41754
41755 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
41756}
41757
41758// We are looking for a shuffle where both sources are concatenated with undef
41759// and have a width that is half of the output's width. AVX2 has VPERMD/Q, so
41760// if we can express this as a single-source shuffle, that's preferable.
41761static SDValue combineShuffleOfConcatUndef(SDNode *N, const SDLoc &DL,
41762 SelectionDAG &DAG,
41763 const X86Subtarget &Subtarget) {
41764 if (!Subtarget.hasAVX2() || !isa<ShuffleVectorSDNode>(N))
41765 return SDValue();
41766
41767 EVT VT = N->getValueType(0);
41768
41769 // We only care about shuffles of 128/256-bit vectors of 32/64-bit values.
41770 if (!VT.is128BitVector() && !VT.is256BitVector())
41771 return SDValue();
41772
41773 if (VT.getVectorElementType() != MVT::i32 &&
41774 VT.getVectorElementType() != MVT::i64 &&
41775 VT.getVectorElementType() != MVT::f32 &&
41776 VT.getVectorElementType() != MVT::f64)
41777 return SDValue();
41778
41779 SDValue N0 = N->getOperand(0);
41780 SDValue N1 = N->getOperand(1);
41781
41782 // Check that both sources are concats with undef.
41783 if (N0.getOpcode() != ISD::CONCAT_VECTORS ||
41784 N1.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 ||
41785 N1.getNumOperands() != 2 || !N0.getOperand(1).isUndef() ||
41786 !N1.getOperand(1).isUndef())
41787 return SDValue();
41788
41789 // Construct the new shuffle mask. Elements from the first source retain their
41790 // index, but elements from the second source no longer need to skip an undef.
41791 SmallVector<int, 8> Mask;
41792 int NumElts = VT.getVectorNumElements();
41793
41794 auto *SVOp = cast<ShuffleVectorSDNode>(N);
41795 for (int Elt : SVOp->getMask())
41796 Mask.push_back(Elt < NumElts ? Elt : (Elt - NumElts / 2));
41797
41798 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
41799 N1.getOperand(0));
41800 return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask);
41801}
41802
41803/// If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the
41804/// low half of each source vector and does not set any high half elements in
41805/// the destination vector, narrow the shuffle to half its original size.
41806static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) {
41807 EVT VT = Shuf->getValueType(0);
41808 if (!DAG.getTargetLoweringInfo().isTypeLegal(Shuf->getValueType(0)))
41809 return SDValue();
41810 if (!VT.is256BitVector() && !VT.is512BitVector())
41811 return SDValue();
41812
41813 // See if we can ignore all of the high elements of the shuffle.
41814 ArrayRef<int> Mask = Shuf->getMask();
41815 if (!isUndefUpperHalf(Mask))
41816 return SDValue();
41817
41818 // Check if the shuffle mask accesses only the low half of each input vector
41819 // (half-index output is 0 or 2).
41820 int HalfIdx1, HalfIdx2;
41821 SmallVector<int, 8> HalfMask(Mask.size() / 2);
41822 if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2) ||
41823 (HalfIdx1 % 2 == 1) || (HalfIdx2 % 2 == 1))
41824 return SDValue();
41825
41826 // Create a half-width shuffle to replace the unnecessarily wide shuffle.
41827 // The trick is knowing that all of the insert/extract are actually free
41828 // subregister (zmm<->ymm or ymm<->xmm) ops. That leaves us with a shuffle
41829 // of narrow inputs into a narrow output, and that is always cheaper than
41830 // the wide shuffle that we started with.
41831 return getShuffleHalfVectors(SDLoc(Shuf), Shuf->getOperand(0),
41832 Shuf->getOperand(1), HalfMask, HalfIdx1,
41833 HalfIdx2, false, DAG, /*UseConcat*/ true);
41834}
41835
41836static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
41837 TargetLowering::DAGCombinerInfo &DCI,
41838 const X86Subtarget &Subtarget) {
41839 if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(N))
41840 if (SDValue V = narrowShuffle(Shuf, DAG))
41841 return V;
41842
41843 // If we have legalized the vector types, look for blends of FADD and FSUB
41844 // nodes that we can fuse into an ADDSUB, FMADDSUB, or FMSUBADD node.
41845 SDLoc dl(N);
41846 EVT VT = N->getValueType(0);
41847 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41848 if (TLI.isTypeLegal(VT) && !isSoftF16(VT, Subtarget))
41849 if (SDValue AddSub =
41850 combineShuffleToAddSubOrFMAddSub(N, dl, Subtarget, DAG))
41851 return AddSub;
41852
41853 // Attempt to combine into a vector load/broadcast.
41854 if (SDValue LD = combineToConsecutiveLoads(
41855 VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true))
41856 return LD;
41857
41858 // For AVX2, we sometimes want to combine
41859 // (vector_shuffle <mask> (concat_vectors t1, undef)
41860 // (concat_vectors t2, undef))
41861 // Into:
41862 // (vector_shuffle <mask> (concat_vectors t1, t2), undef)
41863 // Since the latter can be efficiently lowered with VPERMD/VPERMQ
41864 if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, dl, DAG, Subtarget))
41865 return ShufConcat;
41866
41867 if (isTargetShuffle(N->getOpcode())) {
41868 SDValue Op(N, 0);
41869 if (SDValue Shuffle = combineTargetShuffle(Op, dl, DAG, DCI, Subtarget))
41870 return Shuffle;
41871
41872 // Try recursively combining arbitrary sequences of x86 shuffle
41873 // instructions into higher-order shuffles. We do this after combining
41874 // specific PSHUF instruction sequences into their minimal form so that we
41875 // can evaluate how many specialized shuffle instructions are involved in
41876 // a particular chain.
41877 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
41878 return Res;
41879
41880 // Simplify source operands based on shuffle mask.
41881 // TODO - merge this into combineX86ShufflesRecursively.
41882 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
41883 if (TLI.SimplifyDemandedVectorElts(Op, DemandedElts, DCI))
41884 return SDValue(N, 0);
41885
41886 // Canonicalize SHUFFLE(UNARYOP(X)) -> UNARYOP(SHUFFLE(X)).
41887 // Canonicalize SHUFFLE(BINOP(X,Y)) -> BINOP(SHUFFLE(X),SHUFFLE(Y)).
41888 // Perform this after other shuffle combines to allow inner shuffles to be
41889 // combined away first.
41890 if (SDValue BinOp = canonicalizeShuffleWithOp(Op, DAG, dl))
41891 return BinOp;
41892 }
41893
41894 return SDValue();
41895}
41896
41897// Simplify variable target shuffle masks based on the demanded elements.
41898// TODO: Handle DemandedBits in mask indices as well?
41899bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle(
41900 SDValue Op, const APInt &DemandedElts, unsigned MaskIndex,
41901 TargetLowering::TargetLoweringOpt &TLO, unsigned Depth) const {
41902 // If we're demanding all elements don't bother trying to simplify the mask.
41903 unsigned NumElts = DemandedElts.getBitWidth();
41904 if (DemandedElts.isAllOnes())
41905 return false;
41906
41907 SDValue Mask = Op.getOperand(MaskIndex);
41908 if (!Mask.hasOneUse())
41909 return false;
41910
41911 // Attempt to generically simplify the variable shuffle mask.
41912 APInt MaskUndef, MaskZero;
41913 if (SimplifyDemandedVectorElts(Mask, DemandedElts, MaskUndef, MaskZero, TLO,
41914 Depth + 1))
41915 return true;
41916
41917 // Attempt to extract+simplify a (constant pool load) shuffle mask.
41918 // TODO: Support other types from getTargetShuffleMaskIndices?
41919 SDValue BC = peekThroughOneUseBitcasts(Mask);
41920 EVT BCVT = BC.getValueType();
41921 auto *Load = dyn_cast<LoadSDNode>(BC);
41922 if (!Load || !Load->getBasePtr().hasOneUse())
41923 return false;
41924
41925 const Constant *C = getTargetConstantFromNode(Load);
41926 if (!C)
41927 return false;
41928
41929 Type *CTy = C->getType();
41930 if (!CTy->isVectorTy() ||
41931 CTy->getPrimitiveSizeInBits() != Mask.getValueSizeInBits())
41932 return false;
41933
41934 // Handle scaling for i64 elements on 32-bit targets.
41935 unsigned NumCstElts = cast<FixedVectorType>(CTy)->getNumElements();
41936 if (NumCstElts != NumElts && NumCstElts != (NumElts * 2))
41937 return false;
41938 unsigned Scale = NumCstElts / NumElts;
41939
41940 // Simplify mask if we have an undemanded element that is not undef.
41941 bool Simplified = false;
41942 SmallVector<Constant *, 32> ConstVecOps;
41943 for (unsigned i = 0; i != NumCstElts; ++i) {
41944 Constant *Elt = C->getAggregateElement(i);
41945 if (!DemandedElts[i / Scale] && !isa<UndefValue>(Elt)) {
41946 ConstVecOps.push_back(UndefValue::get(Elt->getType()));
41947 Simplified = true;
41948 continue;
41949 }
41950 ConstVecOps.push_back(Elt);
41951 }
41952 if (!Simplified)
41953 return false;
41954
41955 // Generate new constant pool entry + legalize immediately for the load.
41956 SDLoc DL(Op);
41957 SDValue CV = TLO.DAG.getConstantPool(ConstantVector::get(ConstVecOps), BCVT);
41958 SDValue LegalCV = LowerConstantPool(CV, TLO.DAG);
41959 SDValue NewMask = TLO.DAG.getLoad(
41960 BCVT, DL, TLO.DAG.getEntryNode(), LegalCV,
41961 MachinePointerInfo::getConstantPool(TLO.DAG.getMachineFunction()),
41962 Load->getAlign());
41963 return TLO.CombineTo(Mask, TLO.DAG.getBitcast(Mask.getValueType(), NewMask));
41964}
41965
41966bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
41967 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
41968 TargetLoweringOpt &TLO, unsigned Depth) const {
41969 int NumElts = DemandedElts.getBitWidth();
41970 unsigned Opc = Op.getOpcode();
41971 EVT VT = Op.getValueType();
41972
41973 // Handle special case opcodes.
41974 switch (Opc) {
41975 case X86ISD::PMULDQ:
41976 case X86ISD::PMULUDQ: {
41977 APInt LHSUndef, LHSZero;
41978 APInt RHSUndef, RHSZero;
41979 SDValue LHS = Op.getOperand(0);
41980 SDValue RHS = Op.getOperand(1);
41981 if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO,
41982 Depth + 1))
41983 return true;
41984 if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO,
41985 Depth + 1))
41986 return true;
41987 // Multiply by zero.
41988 KnownZero = LHSZero | RHSZero;
41989 break;
41990 }
41991 case X86ISD::VPMADDUBSW:
41992 case X86ISD::VPMADDWD: {
41993 APInt LHSUndef, LHSZero;
41994 APInt RHSUndef, RHSZero;
41995 SDValue LHS = Op.getOperand(0);
41996 SDValue RHS = Op.getOperand(1);
41997 APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, 2 * NumElts);
41998
41999 if (SimplifyDemandedVectorElts(LHS, DemandedSrcElts, LHSUndef, LHSZero, TLO,
42000 Depth + 1))
42001 return true;
42002 if (SimplifyDemandedVectorElts(RHS, DemandedSrcElts, RHSUndef, RHSZero, TLO,
42003 Depth + 1))
42004 return true;
42005
42006 // TODO: Multiply by zero.
42007
42008 // If RHS/LHS elements are known zero then we don't need the LHS/RHS equivalent.
42009 APInt DemandedLHSElts = DemandedSrcElts & ~RHSZero;
42010 if (SimplifyDemandedVectorElts(LHS, DemandedLHSElts, LHSUndef, LHSZero, TLO,
42011 Depth + 1))
42012 return true;
42013 APInt DemandedRHSElts = DemandedSrcElts & ~LHSZero;
42014 if (SimplifyDemandedVectorElts(RHS, DemandedRHSElts, RHSUndef, RHSZero, TLO,
42015 Depth + 1))
42016 return true;
42017 break;
42018 }
42019 case X86ISD::PSADBW: {
42020 SDValue LHS = Op.getOperand(0);
42021 SDValue RHS = Op.getOperand(1);
42022 assert(VT.getScalarType() == MVT::i64 &&
42023 LHS.getValueType() == RHS.getValueType() &&
42024 LHS.getValueType().getScalarType() == MVT::i8 &&
42025 "Unexpected PSADBW types");
42026
42027 // Aggressively peek through ops to get at the demanded elts.
42028 if (!DemandedElts.isAllOnes()) {
42029 unsigned NumSrcElts = LHS.getValueType().getVectorNumElements();
42030 APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
42031 SDValue NewLHS = SimplifyMultipleUseDemandedVectorElts(
42032 LHS, DemandedSrcElts, TLO.DAG, Depth + 1);
42033 SDValue NewRHS = SimplifyMultipleUseDemandedVectorElts(
42034 RHS, DemandedSrcElts, TLO.DAG, Depth + 1);
42035 if (NewLHS || NewRHS) {
42036 NewLHS = NewLHS ? NewLHS : LHS;
42037 NewRHS = NewRHS ? NewRHS : RHS;
42038 return TLO.CombineTo(
42039 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS));
42040 }
42041 }
42042 break;
42043 }
42044 case X86ISD::VSHL:
42045 case X86ISD::VSRL:
42046 case X86ISD::VSRA: {
42047 // We only need the bottom 64-bits of the (128-bit) shift amount.
42048 SDValue Amt = Op.getOperand(1);
42049 MVT AmtVT = Amt.getSimpleValueType();
42050 assert(AmtVT.is128BitVector() && "Unexpected value type");
42051
42052 // If we reuse the shift amount just for sse shift amounts then we know that
42053 // only the bottom 64-bits are only ever used.
42054 bool AssumeSingleUse = llvm::all_of(Amt->uses(), [&Amt](SDNode *Use) {
42055 unsigned UseOpc = Use->getOpcode();
42056 return (UseOpc == X86ISD::VSHL || UseOpc == X86ISD::VSRL ||
42057 UseOpc == X86ISD::VSRA) &&
42058 Use->getOperand(0) != Amt;
42059 });
42060
42061 APInt AmtUndef, AmtZero;
42062 unsigned NumAmtElts = AmtVT.getVectorNumElements();
42063 APInt AmtElts = APInt::getLowBitsSet(NumAmtElts, NumAmtElts / 2);
42064 if (SimplifyDemandedVectorElts(Amt, AmtElts, AmtUndef, AmtZero, TLO,
42065 Depth + 1, AssumeSingleUse))
42066 return true;
42067 [[fallthrough]];
42068 }
42069 case X86ISD::VSHLI:
42070 case X86ISD::VSRLI:
42071 case X86ISD::VSRAI: {
42072 SDValue Src = Op.getOperand(0);
42073 APInt SrcUndef;
42074 if (SimplifyDemandedVectorElts(Src, DemandedElts, SrcUndef, KnownZero, TLO,
42075 Depth + 1))
42076 return true;
42077
42078 // Fold shift(0,x) -> 0
42079 if (DemandedElts.isSubsetOf(KnownZero))
42080 return TLO.CombineTo(
42081 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
42082
42083 // Aggressively peek through ops to get at the demanded elts.
42084 if (!DemandedElts.isAllOnes())
42085 if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
42086 Src, DemandedElts, TLO.DAG, Depth + 1))
42087 return TLO.CombineTo(
42088 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1)));
42089 break;
42090 }
42091 case X86ISD::VPSHA:
42092 case X86ISD::VPSHL:
42093 case X86ISD::VSHLV:
42094 case X86ISD::VSRLV:
42095 case X86ISD::VSRAV: {
42096 APInt LHSUndef, LHSZero;
42097 APInt RHSUndef, RHSZero;
42098 SDValue LHS = Op.getOperand(0);
42099 SDValue RHS = Op.getOperand(1);
42100 if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO,
42101 Depth + 1))
42102 return true;
42103
42104 // Fold shift(0,x) -> 0
42105 if (DemandedElts.isSubsetOf(LHSZero))
42106 return TLO.CombineTo(
42107 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
42108
42109 if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO,
42110 Depth + 1))
42111 return true;
42112
42113 KnownZero = LHSZero;
42114 break;
42115 }
42116 case X86ISD::PCMPEQ:
42117 case X86ISD::PCMPGT: {
42118 APInt LHSUndef, LHSZero;
42119 APInt RHSUndef, RHSZero;
42120 SDValue LHS = Op.getOperand(0);
42121 SDValue RHS = Op.getOperand(1);
42122 if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO,
42123 Depth + 1))
42124 return true;
42125 if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO,
42126 Depth + 1))
42127 return true;
42128 break;
42129 }
42130 case X86ISD::KSHIFTL: {
42131 SDValue Src = Op.getOperand(0);
42132 auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
42133 assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount");
42134 unsigned ShiftAmt = Amt->getZExtValue();
42135
42136 if (ShiftAmt == 0)
42137 return TLO.CombineTo(Op, Src);
42138
42139 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
42140 // single shift. We can do this if the bottom bits (which are shifted
42141 // out) are never demanded.
42142 if (Src.getOpcode() == X86ISD::KSHIFTR) {
42143 if (!DemandedElts.intersects(APInt::getLowBitsSet(NumElts, ShiftAmt))) {
42144 unsigned C1 = Src.getConstantOperandVal(1);
42145 unsigned NewOpc = X86ISD::KSHIFTL;
42146 int Diff = ShiftAmt - C1;
42147 if (Diff < 0) {
42148 Diff = -Diff;
42149 NewOpc = X86ISD::KSHIFTR;
42150 }
42151
42152 SDLoc dl(Op);
42153 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
42154 return TLO.CombineTo(
42155 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
42156 }
42157 }
42158
42159 APInt DemandedSrc = DemandedElts.lshr(ShiftAmt);
42160 if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
42161 Depth + 1))
42162 return true;
42163
42164 KnownUndef <<= ShiftAmt;
42165 KnownZero <<= ShiftAmt;
42166 KnownZero.setLowBits(ShiftAmt);
42167 break;
42168 }
42169 case X86ISD::KSHIFTR: {
42170 SDValue Src = Op.getOperand(0);
42171 auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
42172 assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount");
42173 unsigned ShiftAmt = Amt->getZExtValue();
42174
42175 if (ShiftAmt == 0)
42176 return TLO.CombineTo(Op, Src);
42177
42178 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
42179 // single shift. We can do this if the top bits (which are shifted
42180 // out) are never demanded.
42181 if (Src.getOpcode() == X86ISD::KSHIFTL) {
42182 if (!DemandedElts.intersects(APInt::getHighBitsSet(NumElts, ShiftAmt))) {
42183 unsigned C1 = Src.getConstantOperandVal(1);
42184 unsigned NewOpc = X86ISD::KSHIFTR;
42185 int Diff = ShiftAmt - C1;
42186 if (Diff < 0) {
42187 Diff = -Diff;
42188 NewOpc = X86ISD::KSHIFTL;
42189 }
42190
42191 SDLoc dl(Op);
42192 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
42193 return TLO.CombineTo(
42194 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
42195 }
42196 }
42197
42198 APInt DemandedSrc = DemandedElts.shl(ShiftAmt);
42199 if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
42200 Depth + 1))
42201 return true;
42202
42203 KnownUndef.lshrInPlace(ShiftAmt);
42204 KnownZero.lshrInPlace(ShiftAmt);
42205 KnownZero.setHighBits(ShiftAmt);
42206 break;
42207 }
42208 case X86ISD::ANDNP: {
42209 // ANDNP = (~LHS & RHS);
42210 SDValue LHS = Op.getOperand(0);
42211 SDValue RHS = Op.getOperand(1);
42212
42213 auto GetDemandedMasks = [&](SDValue Op, bool Invert = false) {
42214 APInt UndefElts;
42215 SmallVector<APInt> EltBits;
42216 int NumElts = VT.getVectorNumElements();
42217 int EltSizeInBits = VT.getScalarSizeInBits();
42218 APInt OpBits = APInt::getAllOnes(EltSizeInBits);
42219 APInt OpElts = DemandedElts;
42220 if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
42221 EltBits)) {
42222 OpBits.clearAllBits();
42223 OpElts.clearAllBits();
42224 for (int I = 0; I != NumElts; ++I) {
42225 if (!DemandedElts[I])
42226 continue;
42227 if (UndefElts[I]) {
42228 // We can't assume an undef src element gives an undef dst - the
42229 // other src might be zero.
42230 OpBits.setAllBits();
42231 OpElts.setBit(I);
42232 } else if ((Invert && !EltBits[I].isAllOnes()) ||
42233 (!Invert && !EltBits[I].isZero())) {
42234 OpBits |= Invert ? ~EltBits[I] : EltBits[I];
42235 OpElts.setBit(I);
42236 }
42237 }
42238 }
42239 return std::make_pair(OpBits, OpElts);
42240 };
42241 APInt BitsLHS, EltsLHS;
42242 APInt BitsRHS, EltsRHS;
42243 std::tie(BitsLHS, EltsLHS) = GetDemandedMasks(RHS);
42244 std::tie(BitsRHS, EltsRHS) = GetDemandedMasks(LHS, true);
42245
42246 APInt LHSUndef, LHSZero;
42247 APInt RHSUndef, RHSZero;
42248 if (SimplifyDemandedVectorElts(LHS, EltsLHS, LHSUndef, LHSZero, TLO,
42249 Depth + 1))
42250 return true;
42251 if (SimplifyDemandedVectorElts(RHS, EltsRHS, RHSUndef, RHSZero, TLO,
42252 Depth + 1))
42253 return true;
42254
42255 if (!DemandedElts.isAllOnes()) {
42256 SDValue NewLHS = SimplifyMultipleUseDemandedBits(LHS, BitsLHS, EltsLHS,
42257 TLO.DAG, Depth + 1);
42258 SDValue NewRHS = SimplifyMultipleUseDemandedBits(RHS, BitsRHS, EltsRHS,
42259 TLO.DAG, Depth + 1);
42260 if (NewLHS || NewRHS) {
42261 NewLHS = NewLHS ? NewLHS : LHS;
42262 NewRHS = NewRHS ? NewRHS : RHS;
42263 return TLO.CombineTo(
42264 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS));
42265 }
42266 }
42267 break;
42268 }
42269 case X86ISD::CVTSI2P:
42270 case X86ISD::CVTUI2P:
42271 case X86ISD::CVTPH2PS:
42272 case X86ISD::CVTPS2PH: {
42273 SDValue Src = Op.getOperand(0);
42274 EVT SrcVT = Src.getValueType();
42275 APInt SrcUndef, SrcZero;
42276 APInt SrcElts = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
42277 if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
42278 Depth + 1))
42279 return true;
42280 break;
42281 }
42282 case X86ISD::PACKSS:
42283 case X86ISD::PACKUS: {
42284 SDValue N0 = Op.getOperand(0);
42285 SDValue N1 = Op.getOperand(1);
42286
42287 APInt DemandedLHS, DemandedRHS;
42288 getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
42289
42290 APInt LHSUndef, LHSZero;
42291 if (SimplifyDemandedVectorElts(N0, DemandedLHS, LHSUndef, LHSZero, TLO,
42292 Depth + 1))
42293 return true;
42294 APInt RHSUndef, RHSZero;
42295 if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO,
42296 Depth + 1))
42297 return true;
42298
42299 // TODO - pass on known zero/undef.
42300
42301 // Aggressively peek through ops to get at the demanded elts.
42302 // TODO - we should do this for all target/faux shuffles ops.
42303 if (!DemandedElts.isAllOnes()) {
42304 SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
42305 TLO.DAG, Depth + 1);
42306 SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
42307 TLO.DAG, Depth + 1);
42308 if (NewN0 || NewN1) {
42309 NewN0 = NewN0 ? NewN0 : N0;
42310 NewN1 = NewN1 ? NewN1 : N1;
42311 return TLO.CombineTo(Op,
42312 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1));
42313 }
42314 }
42315 break;
42316 }
42317 case X86ISD::HADD:
42318 case X86ISD::HSUB:
42319 case X86ISD::FHADD:
42320 case X86ISD::FHSUB: {
42321 SDValue N0 = Op.getOperand(0);
42322 SDValue N1 = Op.getOperand(1);
42323
42324 APInt DemandedLHS, DemandedRHS;
42325 getHorizDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
42326
42327 APInt LHSUndef, LHSZero;
42328 if (SimplifyDemandedVectorElts(N0, DemandedLHS, LHSUndef, LHSZero, TLO,
42329 Depth + 1))
42330 return true;
42331 APInt RHSUndef, RHSZero;
42332 if (SimplifyDemandedVectorElts(N1, DemandedRHS, RHSUndef, RHSZero, TLO,
42333 Depth + 1))
42334 return true;
42335
42336 // TODO - pass on known zero/undef.
42337
42338 // Aggressively peek through ops to get at the demanded elts.
42339 // TODO: Handle repeated operands.
42340 if (N0 != N1 && !DemandedElts.isAllOnes()) {
42341 SDValue NewN0 = SimplifyMultipleUseDemandedVectorElts(N0, DemandedLHS,
42342 TLO.DAG, Depth + 1);
42343 SDValue NewN1 = SimplifyMultipleUseDemandedVectorElts(N1, DemandedRHS,
42344 TLO.DAG, Depth + 1);
42345 if (NewN0 || NewN1) {
42346 NewN0 = NewN0 ? NewN0 : N0;
42347 NewN1 = NewN1 ? NewN1 : N1;
42348 return TLO.CombineTo(Op,
42349 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1));
42350 }
42351 }
42352 break;
42353 }
42354 case X86ISD::VTRUNC:
42355 case X86ISD::VTRUNCS:
42356 case X86ISD::VTRUNCUS: {
42357 SDValue Src = Op.getOperand(0);
42358 MVT SrcVT = Src.getSimpleValueType();
42359 APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
42360 APInt SrcUndef, SrcZero;
42361 if (SimplifyDemandedVectorElts(Src, DemandedSrc, SrcUndef, SrcZero, TLO,
42362 Depth + 1))
42363 return true;
42364 KnownZero = SrcZero.zextOrTrunc(NumElts);
42365 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
42366 break;
42367 }
42368 case X86ISD::BLENDI: {
42369 SmallVector<int, 16> BlendMask;
42370 DecodeBLENDMask(NumElts, Op.getConstantOperandVal(2), BlendMask);
42371 if (SDValue R = combineBlendOfPermutes(
42372 VT.getSimpleVT(), Op.getOperand(0), Op.getOperand(1), BlendMask,
42373 DemandedElts, TLO.DAG, Subtarget, SDLoc(Op)))
42374 return TLO.CombineTo(Op, R);
42375 break;
42376 }
42377 case X86ISD::BLENDV: {
42378 APInt SelUndef, SelZero;
42379 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, SelUndef,
42380 SelZero, TLO, Depth + 1))
42381 return true;
42382
42383 // TODO: Use SelZero to adjust LHS/RHS DemandedElts.
42384 APInt LHSUndef, LHSZero;
42385 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, LHSUndef,
42386 LHSZero, TLO, Depth + 1))
42387 return true;
42388
42389 APInt RHSUndef, RHSZero;
42390 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedElts, RHSUndef,
42391 RHSZero, TLO, Depth + 1))
42392 return true;
42393
42394 KnownZero = LHSZero & RHSZero;
42395 KnownUndef = LHSUndef & RHSUndef;
42396 break;
42397 }
42398 case X86ISD::VZEXT_MOVL: {
42399 // If upper demanded elements are already zero then we have nothing to do.
42400 SDValue Src = Op.getOperand(0);
42401 APInt DemandedUpperElts = DemandedElts;
42402 DemandedUpperElts.clearLowBits(1);
42403 if (TLO.DAG.MaskedVectorIsZero(Src, DemandedUpperElts, Depth + 1))
42404 return TLO.CombineTo(Op, Src);
42405 break;
42406 }
42407 case X86ISD::VZEXT_LOAD: {
42408 // If upper demanded elements are not demanded then simplify to a
42409 // scalar_to_vector(load()).
42410 MVT SVT = VT.getSimpleVT().getVectorElementType();
42411 if (DemandedElts == 1 && Op.getValue(1).use_empty() && isTypeLegal(SVT)) {
42412 SDLoc DL(Op);
42413 auto *Mem = cast<MemSDNode>(Op);
42414 SDValue Elt = TLO.DAG.getLoad(SVT, DL, Mem->getChain(), Mem->getBasePtr(),
42415 Mem->getMemOperand());
42416 SDValue Vec = TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Elt);
42417 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Vec));
42418 }
42419 break;
42420 }
42421 case X86ISD::VBROADCAST: {
42422 SDValue Src = Op.getOperand(0);
42423 MVT SrcVT = Src.getSimpleValueType();
42424 if (!SrcVT.isVector())
42425 break;
42426 // Don't bother broadcasting if we just need the 0'th element.
42427 if (DemandedElts == 1) {
42428 if (Src.getValueType() != VT)
42429 Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG,
42430 SDLoc(Op));
42431 return TLO.CombineTo(Op, Src);
42432 }
42433 APInt SrcUndef, SrcZero;
42434 APInt SrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(), 0);
42435 if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
42436 Depth + 1))
42437 return true;
42438 // Aggressively peek through src to get at the demanded elt.
42439 // TODO - we should do this for all target/faux shuffles ops.
42440 if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
42441 Src, SrcElts, TLO.DAG, Depth + 1))
42442 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
42443 break;
42444 }
42445 case X86ISD::VPERMV:
42446 if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 0, TLO,
42447 Depth))
42448 return true;
42449 break;
42450 case X86ISD::PSHUFB:
42451 case X86ISD::VPERMV3:
42452 case X86ISD::VPERMILPV:
42453 if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 1, TLO,
42454 Depth))
42455 return true;
42456 break;
42457 case X86ISD::VPPERM:
42458 case X86ISD::VPERMIL2:
42459 if (SimplifyDemandedVectorEltsForTargetShuffle(Op, DemandedElts, 2, TLO,
42460 Depth))
42461 return true;
42462 break;
42463 }
42464
42465 // For 256/512-bit ops that are 128/256-bit ops glued together, if we do not
42466 // demand any of the high elements, then narrow the op to 128/256-bits: e.g.
42467 // (op ymm0, ymm1) --> insert undef, (op xmm0, xmm1), 0
42468 if ((VT.is256BitVector() || VT.is512BitVector()) &&
42469 DemandedElts.lshr(NumElts / 2) == 0) {
42470 unsigned SizeInBits = VT.getSizeInBits();
42471 unsigned ExtSizeInBits = SizeInBits / 2;
42472
42473 // See if 512-bit ops only use the bottom 128-bits.
42474 if (VT.is512BitVector() && DemandedElts.lshr(NumElts / 4) == 0)
42475 ExtSizeInBits = SizeInBits / 4;
42476
42477 switch (Opc) {
42478 // Scalar broadcast.
42479 case X86ISD::VBROADCAST: {
42480 SDLoc DL(Op);
42481 SDValue Src = Op.getOperand(0);
42482 if (Src.getValueSizeInBits() > ExtSizeInBits)
42483 Src = extractSubVector(Src, 0, TLO.DAG, DL, ExtSizeInBits);
42484 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
42485 ExtSizeInBits / VT.getScalarSizeInBits());
42486 SDValue Bcst = TLO.DAG.getNode(X86ISD::VBROADCAST, DL, BcstVT, Src);
42487 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0,
42488 TLO.DAG, DL, ExtSizeInBits));
42489 }
42490 case X86ISD::VBROADCAST_LOAD: {
42491 SDLoc DL(Op);
42492 auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
42493 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
42494 ExtSizeInBits / VT.getScalarSizeInBits());
42495 SDVTList Tys = TLO.DAG.getVTList(BcstVT, MVT::Other);
42496 SDValue Ops[] = {MemIntr->getOperand(0), MemIntr->getOperand(1)};
42497 SDValue Bcst = TLO.DAG.getMemIntrinsicNode(
42498 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MemIntr->getMemoryVT(),
42499 MemIntr->getMemOperand());
42500 TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
42501 Bcst.getValue(1));
42502 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0,
42503 TLO.DAG, DL, ExtSizeInBits));
42504 }
42505 // Subvector broadcast.
42506 case X86ISD::SUBV_BROADCAST_LOAD: {
42507 auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
42508 EVT MemVT = MemIntr->getMemoryVT();
42509 if (ExtSizeInBits == MemVT.getStoreSizeInBits()) {
42510 SDLoc DL(Op);
42511 SDValue Ld =
42512 TLO.DAG.getLoad(MemVT, DL, MemIntr->getChain(),
42513 MemIntr->getBasePtr(), MemIntr->getMemOperand());
42514 TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1),
42515 Ld.getValue(1));
42516 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Ld, 0,
42517 TLO.DAG, DL, ExtSizeInBits));
42518 } else if ((ExtSizeInBits % MemVT.getStoreSizeInBits()) == 0) {
42519 SDLoc DL(Op);
42520 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(),
42521 ExtSizeInBits / VT.getScalarSizeInBits());
42522 if (SDValue BcstLd =
42523 getBROADCAST_LOAD(Opc, DL, BcstVT, MemVT, MemIntr, 0, TLO.DAG))
42524 return TLO.CombineTo(Op,
42525 insertSubVector(TLO.DAG.getUNDEF(VT), BcstLd, 0,
42526 TLO.DAG, DL, ExtSizeInBits));
42527 }
42528 break;
42529 }
42530 // Byte shifts by immediate.
42531 case X86ISD::VSHLDQ:
42532 case X86ISD::VSRLDQ:
42533 // Shift by uniform.
42534 case X86ISD::VSHL:
42535 case X86ISD::VSRL:
42536 case X86ISD::VSRA:
42537 // Shift by immediate.
42538 case X86ISD::VSHLI:
42539 case X86ISD::VSRLI:
42540 case X86ISD::VSRAI: {
42541 SDLoc DL(Op);
42542 SDValue Ext0 =
42543 extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
42544 SDValue ExtOp =
42545 TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0, Op.getOperand(1));
42546 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
42547 SDValue Insert =
42548 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
42549 return TLO.CombineTo(Op, Insert);
42550 }
42551 case X86ISD::VPERMI: {
42552 // Simplify PERMPD/PERMQ to extract_subvector.
42553 // TODO: This should be done in shuffle combining.
42554 if (VT == MVT::v4f64 || VT == MVT::v4i64) {
42555 SmallVector<int, 4> Mask;
42556 DecodeVPERMMask(NumElts, Op.getConstantOperandVal(1), Mask);
42557 if (isUndefOrEqual(Mask[0], 2) && isUndefOrEqual(Mask[1], 3)) {
42558 SDLoc DL(Op);
42559 SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128);
42560 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
42561 SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128);
42562 return TLO.CombineTo(Op, Insert);
42563 }
42564 }
42565 break;
42566 }
42567 case X86ISD::VPERM2X128: {
42568 // Simplify VPERM2F128/VPERM2I128 to extract_subvector.
42569 SDLoc DL(Op);
42570 unsigned LoMask = Op.getConstantOperandVal(2) & 0xF;
42571 if (LoMask & 0x8)
42572 return TLO.CombineTo(
42573 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, DL));
42574 unsigned EltIdx = (LoMask & 0x1) * (NumElts / 2);
42575 unsigned SrcIdx = (LoMask & 0x2) >> 1;
42576 SDValue ExtOp =
42577 extractSubVector(Op.getOperand(SrcIdx), EltIdx, TLO.DAG, DL, 128);
42578 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
42579 SDValue Insert =
42580 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
42581 return TLO.CombineTo(Op, Insert);
42582 }
42583 // Conversions.
42584 // TODO: Add more CVT opcodes when we have test coverage.
42585 case X86ISD::CVTTP2SI:
42586 case X86ISD::CVTTP2UI:
42587 case X86ISD::CVTPH2PS: {
42588 SDLoc DL(Op);
42589 unsigned Scale = SizeInBits / ExtSizeInBits;
42590 SDValue SrcOp = Op.getOperand(0);
42591 MVT SrcVT = SrcOp.getSimpleValueType();
42592 unsigned SrcExtSize =
42593 std::max<unsigned>(SrcVT.getSizeInBits() / Scale, 128);
42594 MVT ExtVT = MVT::getVectorVT(VT.getSimpleVT().getScalarType(),
42595 ExtSizeInBits / VT.getScalarSizeInBits());
42596 SDValue ExtOp = TLO.DAG.getNode(
42597 Opc, DL, ExtVT, extractSubVector(SrcOp, 0, TLO.DAG, DL, SrcExtSize));
42598 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
42599 SDValue Insert =
42600 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
42601 return TLO.CombineTo(Op, Insert);
42602 }
42603 // Zero upper elements.
42604 case X86ISD::VZEXT_MOVL:
42605 // Variable blend.
42606 case X86ISD::BLENDV:
42607 // Target unary shuffles by immediate:
42608 case X86ISD::PSHUFD:
42609 case X86ISD::PSHUFLW:
42610 case X86ISD::PSHUFHW:
42611 case X86ISD::VPERMILPI:
42612 // (Non-Lane Crossing) Target Shuffles.
42613 case X86ISD::VPERMILPV:
42614 case X86ISD::VPERMIL2:
42615 case X86ISD::PSHUFB:
42616 case X86ISD::UNPCKL:
42617 case X86ISD::UNPCKH:
42618 case X86ISD::BLENDI:
42619 // Integer ops.
42620 case X86ISD::PACKSS:
42621 case X86ISD::PACKUS:
42622 case X86ISD::PCMPEQ:
42623 case X86ISD::PCMPGT:
42624 case X86ISD::PMULUDQ:
42625 case X86ISD::PMULDQ:
42626 case X86ISD::VSHLV:
42627 case X86ISD::VSRLV:
42628 case X86ISD::VSRAV:
42629 // Float ops.
42630 case X86ISD::FMAX:
42631 case X86ISD::FMIN:
42632 case X86ISD::FMAXC:
42633 case X86ISD::FMINC:
42634 // Horizontal Ops.
42635 case X86ISD::HADD:
42636 case X86ISD::HSUB:
42637 case X86ISD::FHADD:
42638 case X86ISD::FHSUB: {
42639 SDLoc DL(Op);
42640 SmallVector<SDValue, 4> Ops;
42641 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
42642 SDValue SrcOp = Op.getOperand(i);
42643 EVT SrcVT = SrcOp.getValueType();
42644 assert((!SrcVT.isVector() || SrcVT.getSizeInBits() == SizeInBits) &&
42645 "Unsupported vector size");
42646 Ops.push_back(SrcVT.isVector() ? extractSubVector(SrcOp, 0, TLO.DAG, DL,
42647 ExtSizeInBits)
42648 : SrcOp);
42649 }
42650 MVT ExtVT = VT.getSimpleVT();
42651 ExtVT = MVT::getVectorVT(ExtVT.getScalarType(),
42652 ExtSizeInBits / ExtVT.getScalarSizeInBits());
42653 SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ops);
42654 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
42655 SDValue Insert =
42656 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
42657 return TLO.CombineTo(Op, Insert);
42658 }
42659 }
42660 }
42661
42662 // For splats, unless we *only* demand the 0'th element,
42663 // stop attempts at simplification here, we aren't going to improve things,
42664 // this is better than any potential shuffle.
42665 if (!DemandedElts.isOne() && TLO.DAG.isSplatValue(Op, /*AllowUndefs*/false))
42666 return false;
42667
42668 // Get target/faux shuffle mask.
42669 APInt OpUndef, OpZero;
42670 SmallVector<int, 64> OpMask;
42671 SmallVector<SDValue, 2> OpInputs;
42672 if (!getTargetShuffleInputs(Op, DemandedElts, OpInputs, OpMask, OpUndef,
42673 OpZero, TLO.DAG, Depth, false))
42674 return false;
42675
42676 // Shuffle inputs must be the same size as the result.
42677 if (OpMask.size() != (unsigned)NumElts ||
42678 llvm::any_of(OpInputs, [VT](SDValue V) {
42679 return VT.getSizeInBits() != V.getValueSizeInBits() ||
42680 !V.getValueType().isVector();
42681 }))
42682 return false;
42683
42684 KnownZero = OpZero;
42685 KnownUndef = OpUndef;
42686
42687 // Check if shuffle mask can be simplified to undef/zero/identity.
42688 int NumSrcs = OpInputs.size();
42689 for (int i = 0; i != NumElts; ++i)
42690 if (!DemandedElts[i])
42691 OpMask[i] = SM_SentinelUndef;
42692
42693 if (isUndefInRange(OpMask, 0, NumElts)) {
42694 KnownUndef.setAllBits();
42695 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
42696 }
42697 if (isUndefOrZeroInRange(OpMask, 0, NumElts)) {
42698 KnownZero.setAllBits();
42699 return TLO.CombineTo(
42700 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
42701 }
42702 for (int Src = 0; Src != NumSrcs; ++Src)
42703 if (isSequentialOrUndefInRange(OpMask, 0, NumElts, Src * NumElts))
42704 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, OpInputs[Src]));
42705
42706 // Attempt to simplify inputs.
42707 for (int Src = 0; Src != NumSrcs; ++Src) {
42708 // TODO: Support inputs of different types.
42709 if (OpInputs[Src].getValueType() != VT)
42710 continue;
42711
42712 int Lo = Src * NumElts;
42713 APInt SrcElts = APInt::getZero(NumElts);
42714 for (int i = 0; i != NumElts; ++i)
42715 if (DemandedElts[i]) {
42716 int M = OpMask[i] - Lo;
42717 if (0 <= M && M < NumElts)
42718 SrcElts.setBit(M);
42719 }
42720
42721 // TODO - Propagate input undef/zero elts.
42722 APInt SrcUndef, SrcZero;
42723 if (SimplifyDemandedVectorElts(OpInputs[Src], SrcElts, SrcUndef, SrcZero,
42724 TLO, Depth + 1))
42725 return true;
42726 }
42727
42728 // If we don't demand all elements, then attempt to combine to a simpler
42729 // shuffle.
42730 // We need to convert the depth to something combineX86ShufflesRecursively
42731 // can handle - so pretend its Depth == 0 again, and reduce the max depth
42732 // to match. This prevents combineX86ShuffleChain from returning a
42733 // combined shuffle that's the same as the original root, causing an
42734 // infinite loop.
42735 if (!DemandedElts.isAllOnes()) {
42736 assert(Depth < X86::MaxShuffleCombineDepth && "Depth out of range");
42737
42738 SmallVector<int, 64> DemandedMask(NumElts, SM_SentinelUndef);
42739 for (int i = 0; i != NumElts; ++i)
42740 if (DemandedElts[i])
42741 DemandedMask[i] = i;
42742
42743 SDValue NewShuffle = combineX86ShufflesRecursively(
42744 {Op}, 0, Op, DemandedMask, {}, 0, X86::MaxShuffleCombineDepth - Depth,
42745 /*HasVarMask*/ false,
42746 /*AllowCrossLaneVarMask*/ true, /*AllowPerLaneVarMask*/ true, TLO.DAG,
42747 Subtarget);
42748 if (NewShuffle)
42749 return TLO.CombineTo(Op, NewShuffle);
42750 }
42751
42752 return false;
42753}
42754
42755bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
42756 SDValue Op, const APInt &OriginalDemandedBits,
42757 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
42758 unsigned Depth) const {
42759 EVT VT = Op.getValueType();
42760 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
42761 unsigned Opc = Op.getOpcode();
42762 switch(Opc) {
42763 case X86ISD::VTRUNC: {
42764 KnownBits KnownOp;
42765 SDValue Src = Op.getOperand(0);
42766 MVT SrcVT = Src.getSimpleValueType();
42767
42768 // Simplify the input, using demanded bit information.
42769 APInt TruncMask = OriginalDemandedBits.zext(SrcVT.getScalarSizeInBits());
42770 APInt DemandedElts = OriginalDemandedElts.trunc(SrcVT.getVectorNumElements());
42771 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, KnownOp, TLO, Depth + 1))
42772 return true;
42773 break;
42774 }
42775 case X86ISD::PMULDQ:
42776 case X86ISD::PMULUDQ: {
42777 // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
42778 KnownBits KnownLHS, KnownRHS;
42779 SDValue LHS = Op.getOperand(0);
42780 SDValue RHS = Op.getOperand(1);
42781
42782 // Don't mask bits on 32-bit AVX512 targets which might lose a broadcast.
42783 // FIXME: Can we bound this better?
42784 APInt DemandedMask = APInt::getLowBitsSet(64, 32);
42785 APInt DemandedMaskLHS = APInt::getAllOnes(64);
42786 APInt DemandedMaskRHS = APInt::getAllOnes(64);
42787
42788 bool Is32BitAVX512 = !Subtarget.is64Bit() && Subtarget.hasAVX512();
42789 if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(LHS))
42790 DemandedMaskLHS = DemandedMask;
42791 if (!Is32BitAVX512 || !TLO.DAG.isSplatValue(RHS))
42792 DemandedMaskRHS = DemandedMask;
42793
42794 if (SimplifyDemandedBits(LHS, DemandedMaskLHS, OriginalDemandedElts,
42795 KnownLHS, TLO, Depth + 1))
42796 return true;
42797 if (SimplifyDemandedBits(RHS, DemandedMaskRHS, OriginalDemandedElts,
42798 KnownRHS, TLO, Depth + 1))
42799 return true;
42800
42801 // PMULUDQ(X,1) -> AND(X,(1<<32)-1) 'getZeroExtendInReg'.
42802 KnownRHS = KnownRHS.trunc(32);
42803 if (Opc == X86ISD::PMULUDQ && KnownRHS.isConstant() &&
42804 KnownRHS.getConstant().isOne()) {
42805 SDLoc DL(Op);
42806 SDValue Mask = TLO.DAG.getConstant(DemandedMask, DL, VT);
42807 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, DL, VT, LHS, Mask));
42808 }
42809
42810 // Aggressively peek through ops to get at the demanded low bits.
42811 SDValue DemandedLHS = SimplifyMultipleUseDemandedBits(
42812 LHS, DemandedMaskLHS, OriginalDemandedElts, TLO.DAG, Depth + 1);
42813 SDValue DemandedRHS = SimplifyMultipleUseDemandedBits(
42814 RHS, DemandedMaskRHS, OriginalDemandedElts, TLO.DAG, Depth + 1);
42815 if (DemandedLHS || DemandedRHS) {
42816 DemandedLHS = DemandedLHS ? DemandedLHS : LHS;
42817 DemandedRHS = DemandedRHS ? DemandedRHS : RHS;
42818 return TLO.CombineTo(
42819 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, DemandedLHS, DemandedRHS));
42820 }
42821 break;
42822 }
42823 case X86ISD::ANDNP: {
42824 KnownBits Known2;
42825 SDValue Op0 = Op.getOperand(0);
42826 SDValue Op1 = Op.getOperand(1);
42827
42828 if (SimplifyDemandedBits(Op1, OriginalDemandedBits, OriginalDemandedElts,
42829 Known, TLO, Depth + 1))
42830 return true;
42831
42832 if (SimplifyDemandedBits(Op0, ~Known.Zero & OriginalDemandedBits,
42833 OriginalDemandedElts, Known2, TLO, Depth + 1))
42834 return true;
42835
42836 // If the RHS is a constant, see if we can simplify it.
42837 if (ShrinkDemandedConstant(Op, ~Known2.One & OriginalDemandedBits,
42838 OriginalDemandedElts, TLO))
42839 return true;
42840
42841 // ANDNP = (~Op0 & Op1);
42842 Known.One &= Known2.Zero;
42843 Known.Zero |= Known2.One;
42844 break;
42845 }
42846 case X86ISD::VSHLI: {
42847 SDValue Op0 = Op.getOperand(0);
42848
42849 unsigned ShAmt = Op.getConstantOperandVal(1);
42850 if (ShAmt >= BitWidth)
42851 break;
42852
42853 APInt DemandedMask = OriginalDemandedBits.lshr(ShAmt);
42854
42855 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
42856 // single shift. We can do this if the bottom bits (which are shifted
42857 // out) are never demanded.
42858 if (Op0.getOpcode() == X86ISD::VSRLI &&
42859 OriginalDemandedBits.countr_zero() >= ShAmt) {
42860 unsigned Shift2Amt = Op0.getConstantOperandVal(1);
42861 if (Shift2Amt < BitWidth) {
42862 int Diff = ShAmt - Shift2Amt;
42863 if (Diff == 0)
42864 return TLO.CombineTo(Op, Op0.getOperand(0));
42865
42866 unsigned NewOpc = Diff < 0 ? X86ISD::VSRLI : X86ISD::VSHLI;
42867 SDValue NewShift = TLO.DAG.getNode(
42868 NewOpc, SDLoc(Op), VT, Op0.getOperand(0),
42869 TLO.DAG.getTargetConstant(std::abs(Diff), SDLoc(Op), MVT::i8));
42870 return TLO.CombineTo(Op, NewShift);
42871 }
42872 }
42873
42874 // If we are only demanding sign bits then we can use the shift source directly.
42875 unsigned NumSignBits =
42876 TLO.DAG.ComputeNumSignBits(Op0, OriginalDemandedElts, Depth + 1);
42877 unsigned UpperDemandedBits = BitWidth - OriginalDemandedBits.countr_zero();
42878 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits)
42879 return TLO.CombineTo(Op, Op0);
42880
42881 if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
42882 TLO, Depth + 1))
42883 return true;
42884
42885 Known.Zero <<= ShAmt;
42886 Known.One <<= ShAmt;
42887
42888 // Low bits known zero.
42889 Known.Zero.setLowBits(ShAmt);
42890 return false;
42891 }
42892 case X86ISD::VSRLI: {
42893 unsigned ShAmt = Op.getConstantOperandVal(1);
42894 if (ShAmt >= BitWidth)
42895 break;
42896
42897 APInt DemandedMask = OriginalDemandedBits << ShAmt;
42898
42899 if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask,
42900 OriginalDemandedElts, Known, TLO, Depth + 1))
42901 return true;
42902
42903 Known.Zero.lshrInPlace(ShAmt);
42904 Known.One.lshrInPlace(ShAmt);
42905
42906 // High bits known zero.
42907 Known.Zero.setHighBits(ShAmt);
42908 return false;
42909 }
42910 case X86ISD::VSRAI: {
42911 SDValue Op0 = Op.getOperand(0);
42912 SDValue Op1 = Op.getOperand(1);
42913
42914 unsigned ShAmt = Op1->getAsZExtVal();
42915 if (ShAmt >= BitWidth)
42916 break;
42917
42918 APInt DemandedMask = OriginalDemandedBits << ShAmt;
42919
42920 // If we just want the sign bit then we don't need to shift it.
42921 if (OriginalDemandedBits.isSignMask())
42922 return TLO.CombineTo(Op, Op0);
42923
42924 // fold (VSRAI (VSHLI X, C1), C1) --> X iff NumSignBits(X) > C1
42925 if (Op0.getOpcode() == X86ISD::VSHLI &&
42926 Op.getOperand(1) == Op0.getOperand(1)) {
42927 SDValue Op00 = Op0.getOperand(0);
42928 unsigned NumSignBits =
42929 TLO.DAG.ComputeNumSignBits(Op00, OriginalDemandedElts);
42930 if (ShAmt < NumSignBits)
42931 return TLO.CombineTo(Op, Op00);
42932 }
42933
42934 // If any of the demanded bits are produced by the sign extension, we also
42935 // demand the input sign bit.
42936 if (OriginalDemandedBits.countl_zero() < ShAmt)
42937 DemandedMask.setSignBit();
42938
42939 if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
42940 TLO, Depth + 1))
42941 return true;
42942
42943 Known.Zero.lshrInPlace(ShAmt);
42944 Known.One.lshrInPlace(ShAmt);
42945
42946 // If the input sign bit is known to be zero, or if none of the top bits
42947 // are demanded, turn this into an unsigned shift right.
42948 if (Known.Zero[BitWidth - ShAmt - 1] ||
42949 OriginalDemandedBits.countl_zero() >= ShAmt)
42950 return TLO.CombineTo(
42951 Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1));
42952
42953 // High bits are known one.
42954 if (Known.One[BitWidth - ShAmt - 1])
42955 Known.One.setHighBits(ShAmt);
42956 return false;
42957 }
42958 case X86ISD::BLENDV: {
42959 SDValue Sel = Op.getOperand(0);
42960 SDValue LHS = Op.getOperand(1);
42961 SDValue RHS = Op.getOperand(2);
42962
42963 APInt SignMask = APInt::getSignMask(BitWidth);
42964 SDValue NewSel = SimplifyMultipleUseDemandedBits(
42965 Sel, SignMask, OriginalDemandedElts, TLO.DAG, Depth + 1);
42966 SDValue NewLHS = SimplifyMultipleUseDemandedBits(
42967 LHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1);
42968 SDValue NewRHS = SimplifyMultipleUseDemandedBits(
42969 RHS, OriginalDemandedBits, OriginalDemandedElts, TLO.DAG, Depth + 1);
42970
42971 if (NewSel || NewLHS || NewRHS) {
42972 NewSel = NewSel ? NewSel : Sel;
42973 NewLHS = NewLHS ? NewLHS : LHS;
42974 NewRHS = NewRHS ? NewRHS : RHS;
42975 return TLO.CombineTo(Op, TLO.DAG.getNode(X86ISD::BLENDV, SDLoc(Op), VT,
42976 NewSel, NewLHS, NewRHS));
42977 }
42978 break;
42979 }
42980 case X86ISD::PEXTRB:
42981 case X86ISD::PEXTRW: {
42982 SDValue Vec = Op.getOperand(0);
42983 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
42984 MVT VecVT = Vec.getSimpleValueType();
42985 unsigned NumVecElts = VecVT.getVectorNumElements();
42986
42987 if (CIdx && CIdx->getAPIntValue().ult(NumVecElts)) {
42988 unsigned Idx = CIdx->getZExtValue();
42989 unsigned VecBitWidth = VecVT.getScalarSizeInBits();
42990
42991 // If we demand no bits from the vector then we must have demanded
42992 // bits from the implict zext - simplify to zero.
42993 APInt DemandedVecBits = OriginalDemandedBits.trunc(VecBitWidth);
42994 if (DemandedVecBits == 0)
42995 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
42996
42997 APInt KnownUndef, KnownZero;
42998 APInt DemandedVecElts = APInt::getOneBitSet(NumVecElts, Idx);
42999 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
43000 KnownZero, TLO, Depth + 1))
43001 return true;
43002
43003 KnownBits KnownVec;
43004 if (SimplifyDemandedBits(Vec, DemandedVecBits, DemandedVecElts,
43005 KnownVec, TLO, Depth + 1))
43006 return true;
43007
43008 if (SDValue V = SimplifyMultipleUseDemandedBits(
43009 Vec, DemandedVecBits, DemandedVecElts, TLO.DAG, Depth + 1))
43010 return TLO.CombineTo(
43011 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, V, Op.getOperand(1)));
43012
43013 Known = KnownVec.zext(BitWidth);
43014 return false;
43015 }
43016 break;
43017 }
43018 case X86ISD::PINSRB:
43019 case X86ISD::PINSRW: {
43020 SDValue Vec = Op.getOperand(0);
43021 SDValue Scl = Op.getOperand(1);
43022 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
43023 MVT VecVT = Vec.getSimpleValueType();
43024
43025 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
43026 unsigned Idx = CIdx->getZExtValue();
43027 if (!OriginalDemandedElts[Idx])
43028 return TLO.CombineTo(Op, Vec);
43029
43030 KnownBits KnownVec;
43031 APInt DemandedVecElts(OriginalDemandedElts);
43032 DemandedVecElts.clearBit(Idx);
43033 if (SimplifyDemandedBits(Vec, OriginalDemandedBits, DemandedVecElts,
43034 KnownVec, TLO, Depth + 1))
43035 return true;
43036
43037 KnownBits KnownScl;
43038 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
43039 APInt DemandedSclBits = OriginalDemandedBits.zext(NumSclBits);
43040 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
43041 return true;
43042
43043 KnownScl = KnownScl.trunc(VecVT.getScalarSizeInBits());
43044 Known = KnownVec.intersectWith(KnownScl);
43045 return false;
43046 }
43047 break;
43048 }
43049 case X86ISD::PACKSS:
43050 // PACKSS saturates to MIN/MAX integer values. So if we just want the
43051 // sign bit then we can just ask for the source operands sign bit.
43052 // TODO - add known bits handling.
43053 if (OriginalDemandedBits.isSignMask()) {
43054 APInt DemandedLHS, DemandedRHS;
43055 getPackDemandedElts(VT, OriginalDemandedElts, DemandedLHS, DemandedRHS);
43056
43057 KnownBits KnownLHS, KnownRHS;
43058 APInt SignMask = APInt::getSignMask(BitWidth * 2);
43059 if (SimplifyDemandedBits(Op.getOperand(0), SignMask, DemandedLHS,
43060 KnownLHS, TLO, Depth + 1))
43061 return true;
43062 if (SimplifyDemandedBits(Op.getOperand(1), SignMask, DemandedRHS,
43063 KnownRHS, TLO, Depth + 1))
43064 return true;
43065
43066 // Attempt to avoid multi-use ops if we don't need anything from them.
43067 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
43068 Op.getOperand(0), SignMask, DemandedLHS, TLO.DAG, Depth + 1);
43069 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
43070 Op.getOperand(1), SignMask, DemandedRHS, TLO.DAG, Depth + 1);
43071 if (DemandedOp0 || DemandedOp1) {
43072 SDValue Op0 = DemandedOp0 ? DemandedOp0 : Op.getOperand(0);
43073 SDValue Op1 = DemandedOp1 ? DemandedOp1 : Op.getOperand(1);
43074 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, Op0, Op1));
43075 }
43076 }
43077 // TODO - add general PACKSS/PACKUS SimplifyDemandedBits support.
43078 break;
43079 case X86ISD::VBROADCAST: {
43080 SDValue Src = Op.getOperand(0);
43081 MVT SrcVT = Src.getSimpleValueType();
43082 APInt DemandedElts = APInt::getOneBitSet(
43083 SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1, 0);
43084 if (SimplifyDemandedBits(Src, OriginalDemandedBits, DemandedElts, Known,
43085 TLO, Depth + 1))
43086 return true;
43087 // If we don't need the upper bits, attempt to narrow the broadcast source.
43088 // Don't attempt this on AVX512 as it might affect broadcast folding.
43089 // TODO: Should we attempt this for i32/i16 splats? They tend to be slower.
43090 if ((BitWidth == 64) && SrcVT.isScalarInteger() && !Subtarget.hasAVX512() &&
43091 OriginalDemandedBits.countl_zero() >= (BitWidth / 2) &&
43092 Src->hasOneUse()) {
43093 MVT NewSrcVT = MVT::getIntegerVT(BitWidth / 2);
43094 SDValue NewSrc =
43095 TLO.DAG.getNode(ISD::TRUNCATE, SDLoc(Src), NewSrcVT, Src);
43096 MVT NewVT = MVT::getVectorVT(NewSrcVT, VT.getVectorNumElements() * 2);
43097 SDValue NewBcst =
43098 TLO.DAG.getNode(X86ISD::VBROADCAST, SDLoc(Op), NewVT, NewSrc);
43099 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, NewBcst));
43100 }
43101 break;
43102 }
43103 case X86ISD::PCMPGT:
43104 // icmp sgt(0, R) == ashr(R, BitWidth-1).
43105 // iff we only need the sign bit then we can use R directly.
43106 if (OriginalDemandedBits.isSignMask() &&
43107 ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
43108 return TLO.CombineTo(Op, Op.getOperand(1));
43109 break;
43110 case X86ISD::MOVMSK: {
43111 SDValue Src = Op.getOperand(0);
43112 MVT SrcVT = Src.getSimpleValueType();
43113 unsigned SrcBits = SrcVT.getScalarSizeInBits();
43114 unsigned NumElts = SrcVT.getVectorNumElements();
43115
43116 // If we don't need the sign bits at all just return zero.
43117 if (OriginalDemandedBits.countr_zero() >= NumElts)
43118 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
43119
43120 // See if we only demand bits from the lower 128-bit vector.
43121 if (SrcVT.is256BitVector() &&
43122 OriginalDemandedBits.getActiveBits() <= (NumElts / 2)) {
43123 SDValue NewSrc = extract128BitVector(Src, 0, TLO.DAG, SDLoc(Src));
43124 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
43125 }
43126
43127 // Only demand the vector elements of the sign bits we need.
43128 APInt KnownUndef, KnownZero;
43129 APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(NumElts);
43130 if (SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef, KnownZero,
43131 TLO, Depth + 1))
43132 return true;
43133
43134 Known.Zero = KnownZero.zext(BitWidth);
43135 Known.Zero.setHighBits(BitWidth - NumElts);
43136
43137 // MOVMSK only uses the MSB from each vector element.
43138 KnownBits KnownSrc;
43139 APInt DemandedSrcBits = APInt::getSignMask(SrcBits);
43140 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, KnownSrc, TLO,
43141 Depth + 1))
43142 return true;
43143
43144 if (KnownSrc.One[SrcBits - 1])
43145 Known.One.setLowBits(NumElts);
43146 else if (KnownSrc.Zero[SrcBits - 1])
43147 Known.Zero.setLowBits(NumElts);
43148
43149 // Attempt to avoid multi-use os if we don't need anything from it.
43150 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
43151 Src, DemandedSrcBits, DemandedElts, TLO.DAG, Depth + 1))
43152 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
43153 return false;
43154 }
43155 case X86ISD::TESTP: {
43156 SDValue Op0 = Op.getOperand(0);
43157 SDValue Op1 = Op.getOperand(1);
43158 MVT OpVT = Op0.getSimpleValueType();
43159 assert((OpVT.getVectorElementType() == MVT::f32 ||
43160 OpVT.getVectorElementType() == MVT::f64) &&
43161 "Illegal vector type for X86ISD::TESTP");
43162
43163 // TESTPS/TESTPD only demands the sign bits of ALL the elements.
43164 KnownBits KnownSrc;
43165 APInt SignMask = APInt::getSignMask(OpVT.getScalarSizeInBits());
43166 bool AssumeSingleUse = (Op0 == Op1) && Op->isOnlyUserOf(Op0.getNode());
43167 return SimplifyDemandedBits(Op0, SignMask, KnownSrc, TLO, Depth + 1,
43168 AssumeSingleUse) ||
43169 SimplifyDemandedBits(Op1, SignMask, KnownSrc, TLO, Depth + 1,
43170 AssumeSingleUse);
43171 }
43172 case X86ISD::CMOV: {
43173 KnownBits Known2;
43174 if (SimplifyDemandedBits(Op.getOperand(1), OriginalDemandedBits,
43175 OriginalDemandedElts, Known2, TLO, Depth + 1))
43176 return true;
43177 if (SimplifyDemandedBits(Op.getOperand(0), OriginalDemandedBits,
43178 OriginalDemandedElts, Known, TLO, Depth + 1))
43179 return true;
43180
43181 // Only known if known in both the LHS and RHS.
43182 Known = Known.intersectWith(Known2);
43183 break;
43184 }
43185 case X86ISD::BEXTR:
43186 case X86ISD::BEXTRI: {
43187 SDValue Op0 = Op.getOperand(0);
43188 SDValue Op1 = Op.getOperand(1);
43189
43190 // Only bottom 16-bits of the control bits are required.
43191 if (auto *Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
43192 // NOTE: SimplifyDemandedBits won't do this for constants.
43193 uint64_t Val1 = Cst1->getZExtValue();
43194 uint64_t MaskedVal1 = Val1 & 0xFFFF;
43195 if (Opc == X86ISD::BEXTR && MaskedVal1 != Val1) {
43196 SDLoc DL(Op);
43197 return TLO.CombineTo(
43198 Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0,
43199 TLO.DAG.getConstant(MaskedVal1, DL, VT)));
43200 }
43201
43202 unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0);
43203 unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8);
43204
43205 // If the length is 0, the result is 0.
43206 if (Length == 0) {
43207 Known.setAllZero();
43208 return false;
43209 }
43210
43211 if ((Shift + Length) <= BitWidth) {
43212 APInt DemandedMask = APInt::getBitsSet(BitWidth, Shift, Shift + Length);
43213 if (SimplifyDemandedBits(Op0, DemandedMask, Known, TLO, Depth + 1))
43214 return true;
43215
43216 Known = Known.extractBits(Length, Shift);
43217 Known = Known.zextOrTrunc(BitWidth);
43218 return false;
43219 }
43220 } else {
43221 assert(Opc == X86ISD::BEXTR && "Unexpected opcode!");
43222 KnownBits Known1;
43223 APInt DemandedMask(APInt::getLowBitsSet(BitWidth, 16));
43224 if (SimplifyDemandedBits(Op1, DemandedMask, Known1, TLO, Depth + 1))
43225 return true;
43226
43227 // If the length is 0, replace with 0.
43228 KnownBits LengthBits = Known1.extractBits(8, 8);
43229 if (LengthBits.isZero())
43230 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
43231 }
43232
43233 break;
43234 }
43235 case X86ISD::PDEP: {
43236 SDValue Op0 = Op.getOperand(0);
43237 SDValue Op1 = Op.getOperand(1);
43238
43239 unsigned DemandedBitsLZ = OriginalDemandedBits.countl_zero();
43240 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
43241
43242 // If the demanded bits has leading zeroes, we don't demand those from the
43243 // mask.
43244 if (SimplifyDemandedBits(Op1, LoMask, Known, TLO, Depth + 1))
43245 return true;
43246
43247 // The number of possible 1s in the mask determines the number of LSBs of
43248 // operand 0 used. Undemanded bits from the mask don't matter so filter
43249 // them before counting.
43250 KnownBits Known2;
43251 uint64_t Count = (~Known.Zero & LoMask).popcount();
43252 APInt DemandedMask(APInt::getLowBitsSet(BitWidth, Count));
43253 if (SimplifyDemandedBits(Op0, DemandedMask, Known2, TLO, Depth + 1))
43254 return true;
43255
43256 // Zeroes are retained from the mask, but not ones.
43257 Known.One.clearAllBits();
43258 // The result will have at least as many trailing zeros as the non-mask
43259 // operand since bits can only map to the same or higher bit position.
43260 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
43261 return false;
43262 }
43263 }
43264
43265 return TargetLowering::SimplifyDemandedBitsForTargetNode(
43266 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth);
43267}
43268
43269SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
43270 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
43271 SelectionDAG &DAG, unsigned Depth) const {
43272 int NumElts = DemandedElts.getBitWidth();
43273 unsigned Opc = Op.getOpcode();
43274 EVT VT = Op.getValueType();
43275
43276 switch (Opc) {
43277 case X86ISD::PINSRB:
43278 case X86ISD::PINSRW: {
43279 // If we don't demand the inserted element, return the base vector.
43280 SDValue Vec = Op.getOperand(0);
43281 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
43282 MVT VecVT = Vec.getSimpleValueType();
43283 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
43284 !DemandedElts[CIdx->getZExtValue()])
43285 return Vec;
43286 break;
43287 }
43288 case X86ISD::VSHLI: {
43289 // If we are only demanding sign bits then we can use the shift source
43290 // directly.
43291 SDValue Op0 = Op.getOperand(0);
43292 unsigned ShAmt = Op.getConstantOperandVal(1);
43293 unsigned BitWidth = DemandedBits.getBitWidth();
43294 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
43295 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
43296 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= UpperDemandedBits)
43297 return Op0;
43298 break;
43299 }
43300 case X86ISD::VSRAI:
43301 // iff we only need the sign bit then we can use the source directly.
43302 // TODO: generalize where we only demand extended signbits.
43303 if (DemandedBits.isSignMask())
43304 return Op.getOperand(0);
43305 break;
43306 case X86ISD::PCMPGT:
43307 // icmp sgt(0, R) == ashr(R, BitWidth-1).
43308 // iff we only need the sign bit then we can use R directly.
43309 if (DemandedBits.isSignMask() &&
43310 ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
43311 return Op.getOperand(1);
43312 break;
43313 case X86ISD::BLENDV: {
43314 // BLENDV: Cond (MSB) ? LHS : RHS
43315 SDValue Cond = Op.getOperand(0);
43316 SDValue LHS = Op.getOperand(1);
43317 SDValue RHS = Op.getOperand(2);
43318
43319 KnownBits CondKnown = DAG.computeKnownBits(Cond, DemandedElts, Depth + 1);
43320 if (CondKnown.isNegative())
43321 return LHS;
43322 if (CondKnown.isNonNegative())
43323 return RHS;
43324 break;
43325 }
43326 case X86ISD::ANDNP: {
43327 // ANDNP = (~LHS & RHS);
43328 SDValue LHS = Op.getOperand(0);
43329 SDValue RHS = Op.getOperand(1);
43330
43331 KnownBits LHSKnown = DAG.computeKnownBits(LHS, DemandedElts, Depth + 1);
43332 KnownBits RHSKnown = DAG.computeKnownBits(RHS, DemandedElts, Depth + 1);
43333
43334 // If all of the demanded bits are known 0 on LHS and known 0 on RHS, then
43335 // the (inverted) LHS bits cannot contribute to the result of the 'andn' in
43336 // this context, so return RHS.
43337 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.Zero))
43338 return RHS;
43339 break;
43340 }
43341 }
43342
43343 APInt ShuffleUndef, ShuffleZero;
43344 SmallVector<int, 16> ShuffleMask;
43345 SmallVector<SDValue, 2> ShuffleOps;
43346 if (getTargetShuffleInputs(Op, DemandedElts, ShuffleOps, ShuffleMask,
43347 ShuffleUndef, ShuffleZero, DAG, Depth, false)) {
43348 // If all the demanded elts are from one operand and are inline,
43349 // then we can use the operand directly.
43350 int NumOps = ShuffleOps.size();
43351 if (ShuffleMask.size() == (unsigned)NumElts &&
43352 llvm::all_of(ShuffleOps, [VT](SDValue V) {
43353 return VT.getSizeInBits() == V.getValueSizeInBits();
43354 })) {
43355
43356 if (DemandedElts.isSubsetOf(ShuffleUndef))
43357 return DAG.getUNDEF(VT);
43358 if (DemandedElts.isSubsetOf(ShuffleUndef | ShuffleZero))
43359 return getZeroVector(VT.getSimpleVT(), Subtarget, DAG, SDLoc(Op));
43360
43361 // Bitmask that indicates which ops have only been accessed 'inline'.
43362 APInt IdentityOp = APInt::getAllOnes(NumOps);
43363 for (int i = 0; i != NumElts; ++i) {
43364 int M = ShuffleMask[i];
43365 if (!DemandedElts[i] || ShuffleUndef[i])
43366 continue;
43367 int OpIdx = M / NumElts;
43368 int EltIdx = M % NumElts;
43369 if (M < 0 || EltIdx != i) {
43370 IdentityOp.clearAllBits();
43371 break;
43372 }
43373 IdentityOp &= APInt::getOneBitSet(NumOps, OpIdx);
43374 if (IdentityOp == 0)
43375 break;
43376 }
43377 assert((IdentityOp == 0 || IdentityOp.popcount() == 1) &&
43378 "Multiple identity shuffles detected");
43379
43380 if (IdentityOp != 0)
43381 return DAG.getBitcast(VT, ShuffleOps[IdentityOp.countr_zero()]);
43382 }
43383 }
43384
43385 return TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
43386 Op, DemandedBits, DemandedElts, DAG, Depth);
43387}
43388
43389bool X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
43390 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
43391 bool PoisonOnly, unsigned Depth) const {
43392 unsigned NumElts = DemandedElts.getBitWidth();
43393
43394 switch (Op.getOpcode()) {
43395 case X86ISD::PSHUFD:
43396 case X86ISD::VPERMILPI: {
43397 SmallVector<int, 8> Mask;
43398 SmallVector<SDValue, 2> Ops;
43399 if (getTargetShuffleMask(Op, true, Ops, Mask)) {
43400 SmallVector<APInt, 2> DemandedSrcElts(Ops.size(),
43401 APInt::getZero(NumElts));
43402 for (auto M : enumerate(Mask)) {
43403 if (!DemandedElts[M.index()] || M.value() == SM_SentinelZero)
43404 continue;
43405 if (M.value() == SM_SentinelUndef)
43406 return false;
43407 assert(0 <= M.value() && M.value() < (int)(Ops.size() * NumElts) &&
43408 "Shuffle mask index out of range");
43409 DemandedSrcElts[M.value() / NumElts].setBit(M.value() % NumElts);
43410 }
43411 for (auto Op : enumerate(Ops))
43412 if (!DemandedSrcElts[Op.index()].isZero() &&
43413 !DAG.isGuaranteedNotToBeUndefOrPoison(
43414 Op.value(), DemandedSrcElts[Op.index()], PoisonOnly, Depth + 1))
43415 return false;
43416 return true;
43417 }
43418 break;
43419 }
43420 }
43421 return TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
43422 Op, DemandedElts, DAG, PoisonOnly, Depth);
43423}
43424
43425bool X86TargetLowering::canCreateUndefOrPoisonForTargetNode(
43426 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
43427 bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const {
43428
43429 switch (Op.getOpcode()) {
43430 // SSE vector multiplies are either inbounds or saturate.
43431 case X86ISD::VPMADDUBSW:
43432 case X86ISD::VPMADDWD:
43433 // SSE vector shifts handle out of bounds shift amounts.
43434 case X86ISD::VSHLI:
43435 case X86ISD::VSRLI:
43436 case X86ISD::VSRAI:
43437 return false;
43438 case X86ISD::PSHUFD:
43439 case X86ISD::VPERMILPI:
43440 case X86ISD::UNPCKH:
43441 case X86ISD::UNPCKL:
43442 return false;
43443 case ISD::INTRINSIC_WO_CHAIN:
43444 switch (Op->getConstantOperandVal(0)) {
43445 case Intrinsic::x86_sse2_pmadd_wd:
43446 case Intrinsic::x86_avx2_pmadd_wd:
43447 case Intrinsic::x86_avx512_pmaddw_d_512:
43448 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
43449 case Intrinsic::x86_avx2_pmadd_ub_sw:
43450 case Intrinsic::x86_avx512_pmaddubs_w_512:
43451 return false;
43452 }
43453 }
43454 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
43455 Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth);
43456}
43457
43458bool X86TargetLowering::isSplatValueForTargetNode(SDValue Op,
43459 const APInt &DemandedElts,
43460 APInt &UndefElts,
43461 const SelectionDAG &DAG,
43462 unsigned Depth) const {
43463 unsigned NumElts = DemandedElts.getBitWidth();
43464 unsigned Opc = Op.getOpcode();
43465
43466 switch (Opc) {
43467 case X86ISD::VBROADCAST:
43468 case X86ISD::VBROADCAST_LOAD:
43469 UndefElts = APInt::getZero(NumElts);
43470 return true;
43471 }
43472
43473 return TargetLowering::isSplatValueForTargetNode(Op, DemandedElts, UndefElts,
43474 DAG, Depth);
43475}
43476
43477// Helper to peek through bitops/trunc/setcc to determine size of source vector.
43478// Allows combineBitcastvxi1 to determine what size vector generated a <X x i1>.
43479static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
43480 bool AllowTruncate) {
43481 switch (Src.getOpcode()) {
43482 case ISD::TRUNCATE:
43483 if (!AllowTruncate)
43484 return false;
43485 [[fallthrough]];
43486 case ISD::SETCC:
43487 return Src.getOperand(0).getValueSizeInBits() == Size;
43488 case ISD::FREEZE:
43489 return checkBitcastSrcVectorSize(Src.getOperand(0), Size, AllowTruncate);
43490 case ISD::AND:
43491 case ISD::XOR:
43492 case ISD::OR:
43493 return checkBitcastSrcVectorSize(Src.getOperand(0), Size, AllowTruncate) &&
43494 checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate);
43495 case ISD::SELECT:
43496 case ISD::VSELECT:
43497 return Src.getOperand(0).getScalarValueSizeInBits() == 1 &&
43498 checkBitcastSrcVectorSize(Src.getOperand(1), Size, AllowTruncate) &&
43499 checkBitcastSrcVectorSize(Src.getOperand(2), Size, AllowTruncate);
43500 case ISD::BUILD_VECTOR:
43501 return ISD::isBuildVectorAllZeros(Src.getNode()) ||
43502 ISD::isBuildVectorAllOnes(Src.getNode());
43503 }
43504 return false;
43505}
43506
43507// Helper to flip between AND/OR/XOR opcodes and their X86ISD FP equivalents.
43508static unsigned getAltBitOpcode(unsigned Opcode) {
43509 switch(Opcode) {
43510 // clang-format off
43511 case ISD::AND: return X86ISD::FAND;
43512 case ISD::OR: return X86ISD::FOR;
43513 case ISD::XOR: return X86ISD::FXOR;
43514 case X86ISD::ANDNP: return X86ISD::FANDN;
43515 // clang-format on
43516 }
43517 llvm_unreachable("Unknown bitwise opcode");
43518}
43519
43520// Helper to adjust v4i32 MOVMSK expansion to work with SSE1-only targets.
43521static SDValue adjustBitcastSrcVectorSSE1(SelectionDAG &DAG, SDValue Src,
43522 const SDLoc &DL) {
43523 EVT SrcVT = Src.getValueType();
43524 if (SrcVT != MVT::v4i1)
43525 return SDValue();
43526
43527 switch (Src.getOpcode()) {
43528 case ISD::SETCC:
43529 if (Src.getOperand(0).getValueType() == MVT::v4i32 &&
43530 ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode()) &&
43531 cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT) {
43532 SDValue Op0 = Src.getOperand(0);
43533 if (ISD::isNormalLoad(Op0.getNode()))
43534 return DAG.getBitcast(MVT::v4f32, Op0);
43535 if (Op0.getOpcode() == ISD::BITCAST &&
43536 Op0.getOperand(0).getValueType() == MVT::v4f32)
43537 return Op0.getOperand(0);
43538 }
43539 break;
43540 case ISD::AND:
43541 case ISD::XOR:
43542 case ISD::OR: {
43543 SDValue Op0 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(0), DL);
43544 SDValue Op1 = adjustBitcastSrcVectorSSE1(DAG, Src.getOperand(1), DL);
43545 if (Op0 && Op1)
43546 return DAG.getNode(getAltBitOpcode(Src.getOpcode()), DL, MVT::v4f32, Op0,
43547 Op1);
43548 break;
43549 }
43550 }
43551 return SDValue();
43552}
43553
43554// Helper to push sign extension of vXi1 SETCC result through bitops.
43555static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
43556 SDValue Src, const SDLoc &DL) {
43557 switch (Src.getOpcode()) {
43558 case ISD::SETCC:
43559 case ISD::FREEZE:
43560 case ISD::TRUNCATE:
43561 case ISD::BUILD_VECTOR:
43562 return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
43563 case ISD::AND:
43564 case ISD::XOR:
43565 case ISD::OR:
43566 return DAG.getNode(
43567 Src.getOpcode(), DL, SExtVT,
43568 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(0), DL),
43569 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL));
43570 case ISD::SELECT:
43571 case ISD::VSELECT:
43572 return DAG.getSelect(
43573 DL, SExtVT, Src.getOperand(0),
43574 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL),
43575 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(2), DL));
43576 }
43577 llvm_unreachable("Unexpected node type for vXi1 sign extension");
43578}
43579
43580// Try to match patterns such as
43581// (i16 bitcast (v16i1 x))
43582// ->
43583// (i16 movmsk (16i8 sext (v16i1 x)))
43584// before the illegal vector is scalarized on subtargets that don't have legal
43585// vxi1 types.
43586static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
43587 const SDLoc &DL,
43588 const X86Subtarget &Subtarget) {
43589 EVT SrcVT = Src.getValueType();
43590 if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
43591 return SDValue();
43592
43593 // Recognize the IR pattern for the movmsk intrinsic under SSE1 before type
43594 // legalization destroys the v4i32 type.
43595 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2()) {
43596 if (SDValue V = adjustBitcastSrcVectorSSE1(DAG, Src, DL)) {
43597 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32,
43598 DAG.getBitcast(MVT::v4f32, V));
43599 return DAG.getZExtOrTrunc(V, DL, VT);
43600 }
43601 }
43602
43603 // If the input is a truncate from v16i8 or v32i8 go ahead and use a
43604 // movmskb even with avx512. This will be better than truncating to vXi1 and
43605 // using a kmov. This can especially help KNL if the input is a v16i8/v32i8
43606 // vpcmpeqb/vpcmpgtb.
43607 bool PreferMovMsk = Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse() &&
43608 (Src.getOperand(0).getValueType() == MVT::v16i8 ||
43609 Src.getOperand(0).getValueType() == MVT::v32i8 ||
43610 Src.getOperand(0).getValueType() == MVT::v64i8);
43611
43612 // Prefer movmsk for AVX512 for (bitcast (setlt X, 0)) which can be handled
43613 // directly with vpmovmskb/vmovmskps/vmovmskpd.
43614 if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse() &&
43615 cast<CondCodeSDNode>(Src.getOperand(2))->get() == ISD::SETLT &&
43616 ISD::isBuildVectorAllZeros(Src.getOperand(1).getNode())) {
43617 EVT CmpVT = Src.getOperand(0).getValueType();
43618 EVT EltVT = CmpVT.getVectorElementType();
43619 if (CmpVT.getSizeInBits() <= 256 &&
43620 (EltVT == MVT::i8 || EltVT == MVT::i32 || EltVT == MVT::i64))
43621 PreferMovMsk = true;
43622 }
43623
43624 // With AVX512 vxi1 types are legal and we prefer using k-regs.
43625 // MOVMSK is supported in SSE2 or later.
43626 if (!Subtarget.hasSSE2() || (Subtarget.hasAVX512() && !PreferMovMsk))
43627 return SDValue();
43628
43629 // If the upper ops of a concatenation are undef, then try to bitcast the
43630 // lower op and extend.
43631 SmallVector<SDValue, 4> SubSrcOps;
43632 if (collectConcatOps(Src.getNode(), SubSrcOps, DAG) &&
43633 SubSrcOps.size() >= 2) {
43634 SDValue LowerOp = SubSrcOps[0];
43635 ArrayRef<SDValue> UpperOps(std::next(SubSrcOps.begin()), SubSrcOps.end());
43636 if (LowerOp.getOpcode() == ISD::SETCC &&
43637 all_of(UpperOps, [](SDValue Op) { return Op.isUndef(); })) {
43638 EVT SubVT = VT.getIntegerVT(
43639 *DAG.getContext(), LowerOp.getValueType().getVectorMinNumElements());
43640 if (SDValue V = combineBitcastvxi1(DAG, SubVT, LowerOp, DL, Subtarget)) {
43641 EVT IntVT = VT.getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
43642 return DAG.getBitcast(VT, DAG.getNode(ISD::ANY_EXTEND, DL, IntVT, V));
43643 }
43644 }
43645 }
43646
43647 // There are MOVMSK flavors for types v16i8, v32i8, v4f32, v8f32, v4f64 and
43648 // v8f64. So all legal 128-bit and 256-bit vectors are covered except for
43649 // v8i16 and v16i16.
43650 // For these two cases, we can shuffle the upper element bytes to a
43651 // consecutive sequence at the start of the vector and treat the results as
43652 // v16i8 or v32i8, and for v16i8 this is the preferable solution. However,
43653 // for v16i16 this is not the case, because the shuffle is expensive, so we
43654 // avoid sign-extending to this type entirely.
43655 // For example, t0 := (v8i16 sext(v8i1 x)) needs to be shuffled as:
43656 // (v16i8 shuffle <0,2,4,6,8,10,12,14,u,u,...,u> (v16i8 bitcast t0), undef)
43657 MVT SExtVT;
43658 bool PropagateSExt = false;
43659 switch (SrcVT.getSimpleVT().SimpleTy) {
43660 default:
43661 return SDValue();
43662 case MVT::v2i1:
43663 SExtVT = MVT::v2i64;
43664 break;
43665 case MVT::v4i1:
43666 SExtVT = MVT::v4i32;
43667 // For cases such as (i4 bitcast (v4i1 setcc v4i64 v1, v2))
43668 // sign-extend to a 256-bit operation to avoid truncation.
43669 if (Subtarget.hasAVX() &&
43670 checkBitcastSrcVectorSize(Src, 256, Subtarget.hasAVX2())) {
43671 SExtVT = MVT::v4i64;
43672 PropagateSExt = true;
43673 }
43674 break;
43675 case MVT::v8i1:
43676 SExtVT = MVT::v8i16;
43677 // For cases such as (i8 bitcast (v8i1 setcc v8i32 v1, v2)),
43678 // sign-extend to a 256-bit operation to match the compare.
43679 // If the setcc operand is 128-bit, prefer sign-extending to 128-bit over
43680 // 256-bit because the shuffle is cheaper than sign extending the result of
43681 // the compare.
43682 if (Subtarget.hasAVX() && (checkBitcastSrcVectorSize(Src, 256, true) ||
43683 checkBitcastSrcVectorSize(Src, 512, true))) {
43684 SExtVT = MVT::v8i32;
43685 PropagateSExt = true;
43686 }
43687 break;
43688 case MVT::v16i1:
43689 SExtVT = MVT::v16i8;
43690 // For the case (i16 bitcast (v16i1 setcc v16i16 v1, v2)),
43691 // it is not profitable to sign-extend to 256-bit because this will
43692 // require an extra cross-lane shuffle which is more expensive than
43693 // truncating the result of the compare to 128-bits.
43694 break;
43695 case MVT::v32i1:
43696 SExtVT = MVT::v32i8;
43697 break;
43698 case MVT::v64i1:
43699 // If we have AVX512F, but not AVX512BW and the input is truncated from
43700 // v64i8 checked earlier. Then split the input and make two pmovmskbs.
43701 if (Subtarget.hasAVX512()) {
43702 if (Subtarget.hasBWI())
43703 return SDValue();
43704 SExtVT = MVT::v64i8;
43705 break;
43706 }
43707 // Split if this is a <64 x i8> comparison result.
43708 if (checkBitcastSrcVectorSize(Src, 512, false)) {
43709 SExtVT = MVT::v64i8;
43710 break;
43711 }
43712 return SDValue();
43713 };
43714
43715 SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
43716 : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
43717
43718 if (SExtVT == MVT::v16i8 || SExtVT == MVT::v32i8 || SExtVT == MVT::v64i8) {
43719 V = getPMOVMSKB(DL, V, DAG, Subtarget);
43720 } else {
43721 if (SExtVT == MVT::v8i16) {
43722 V = widenSubVector(V, false, Subtarget, DAG, DL, 256);
43723 V = DAG.getNode(ISD::TRUNCATE, DL, MVT::v16i8, V);
43724 }
43725 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
43726 }
43727
43728 EVT IntVT =
43729 EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements());
43730 V = DAG.getZExtOrTrunc(V, DL, IntVT);
43731 return DAG.getBitcast(VT, V);
43732}
43733
43734// Convert a vXi1 constant build vector to the same width scalar integer.
43735static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
43736 EVT SrcVT = Op.getValueType();
43737 assert(SrcVT.getVectorElementType() == MVT::i1 &&
43738 "Expected a vXi1 vector");
43739 assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
43740 "Expected a constant build vector");
43741
43742 APInt Imm(SrcVT.getVectorNumElements(), 0);
43743 for (unsigned Idx = 0, e = Op.getNumOperands(); Idx < e; ++Idx) {
43744 SDValue In = Op.getOperand(Idx);
43745 if (!In.isUndef() && (In->getAsZExtVal() & 0x1))
43746 Imm.setBit(Idx);
43747 }
43748 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), Imm.getBitWidth());
43749 return DAG.getConstant(Imm, SDLoc(Op), IntVT);
43750}
43751
43752static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG,
43753 TargetLowering::DAGCombinerInfo &DCI,
43754 const X86Subtarget &Subtarget) {
43755 assert(N->getOpcode() == ISD::BITCAST && "Expected a bitcast");
43756
43757 if (!DCI.isBeforeLegalizeOps())
43758 return SDValue();
43759
43760 // Only do this if we have k-registers.
43761 if (!Subtarget.hasAVX512())
43762 return SDValue();
43763
43764 EVT DstVT = N->getValueType(0);
43765 SDValue Op = N->getOperand(0);
43766 EVT SrcVT = Op.getValueType();
43767
43768 if (!Op.hasOneUse())
43769 return SDValue();
43770
43771 // Look for logic ops.
43772 if (Op.getOpcode() != ISD::AND &&
43773 Op.getOpcode() != ISD::OR &&
43774 Op.getOpcode() != ISD::XOR)
43775 return SDValue();
43776
43777 // Make sure we have a bitcast between mask registers and a scalar type.
43778 if (!(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&
43779 DstVT.isScalarInteger()) &&
43780 !(DstVT.isVector() && DstVT.getVectorElementType() == MVT::i1 &&
43781 SrcVT.isScalarInteger()))
43782 return SDValue();
43783
43784 SDValue LHS = Op.getOperand(0);
43785 SDValue RHS = Op.getOperand(1);
43786
43787 if (LHS.hasOneUse() && LHS.getOpcode() == ISD::BITCAST &&
43788 LHS.getOperand(0).getValueType() == DstVT)
43789 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, LHS.getOperand(0),
43790 DAG.getBitcast(DstVT, RHS));
43791
43792 if (RHS.hasOneUse() && RHS.getOpcode() == ISD::BITCAST &&
43793 RHS.getOperand(0).getValueType() == DstVT)
43794 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
43795 DAG.getBitcast(DstVT, LHS), RHS.getOperand(0));
43796
43797 // If the RHS is a vXi1 build vector, this is a good reason to flip too.
43798 // Most of these have to move a constant from the scalar domain anyway.
43799 if (ISD::isBuildVectorOfConstantSDNodes(RHS.getNode())) {
43800 RHS = combinevXi1ConstantToInteger(RHS, DAG);
43801 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
43802 DAG.getBitcast(DstVT, LHS), RHS);
43803 }
43804
43805 return SDValue();
43806}
43807
43808static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
43809 const X86Subtarget &Subtarget) {
43810 SDLoc DL(BV);
43811 unsigned NumElts = BV->getNumOperands();
43812 SDValue Splat = BV->getSplatValue();
43813
43814 // Build MMX element from integer GPR or SSE float values.
43815 auto CreateMMXElement = [&](SDValue V) {
43816 if (V.isUndef())
43817 return DAG.getUNDEF(MVT::x86mmx);
43818 if (V.getValueType().isFloatingPoint()) {
43819 if (Subtarget.hasSSE1() && !isa<ConstantFPSDNode>(V)) {
43820 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V);
43821 V = DAG.getBitcast(MVT::v2i64, V);
43822 return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V);
43823 }
43824 V = DAG.getBitcast(MVT::i32, V);
43825 } else {
43826 V = DAG.getAnyExtOrTrunc(V, DL, MVT::i32);
43827 }
43828 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V);
43829 };
43830
43831 // Convert build vector ops to MMX data in the bottom elements.
43832 SmallVector<SDValue, 8> Ops;
43833
43834 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43835
43836 // Broadcast - use (PUNPCKL+)PSHUFW to broadcast single element.
43837 if (Splat) {
43838 if (Splat.isUndef())
43839 return DAG.getUNDEF(MVT::x86mmx);
43840
43841 Splat = CreateMMXElement(Splat);
43842
43843 if (Subtarget.hasSSE1()) {
43844 // Unpack v8i8 to splat i8 elements to lowest 16-bits.
43845 if (NumElts == 8)
43846 Splat = DAG.getNode(
43847 ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
43848 DAG.getTargetConstant(Intrinsic::x86_mmx_punpcklbw, DL,
43849 TLI.getPointerTy(DAG.getDataLayout())),
43850 Splat, Splat);
43851
43852 // Use PSHUFW to repeat 16-bit elements.
43853 unsigned ShufMask = (NumElts > 2 ? 0 : 0x44);
43854 return DAG.getNode(
43855 ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
43856 DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL,
43857 TLI.getPointerTy(DAG.getDataLayout())),
43858 Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8));
43859 }
43860 Ops.append(NumElts, Splat);
43861 } else {
43862 for (unsigned i = 0; i != NumElts; ++i)
43863 Ops.push_back(CreateMMXElement(BV->getOperand(i)));
43864 }
43865
43866 // Use tree of PUNPCKLs to build up general MMX vector.
43867 while (Ops.size() > 1) {
43868 unsigned NumOps = Ops.size();
43869 unsigned IntrinOp =
43870 (NumOps == 2 ? Intrinsic::x86_mmx_punpckldq
43871 : (NumOps == 4 ? Intrinsic::x86_mmx_punpcklwd
43872 : Intrinsic::x86_mmx_punpcklbw));
43873 SDValue Intrin = DAG.getTargetConstant(
43874 IntrinOp, DL, TLI.getPointerTy(DAG.getDataLayout()));
43875 for (unsigned i = 0; i != NumOps; i += 2)
43876 Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin,
43877 Ops[i], Ops[i + 1]);
43878 Ops.resize(NumOps / 2);
43879 }
43880
43881 return Ops[0];
43882}
43883
43884// Recursive function that attempts to find if a bool vector node was originally
43885// a vector/float/double that got truncated/extended/bitcast to/from a scalar
43886// integer. If so, replace the scalar ops with bool vector equivalents back down
43887// the chain.
43888static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL,
43889 SelectionDAG &DAG,
43890 const X86Subtarget &Subtarget,
43891 unsigned Depth = 0) {
43892 if (Depth >= SelectionDAG::MaxRecursionDepth)
43893 return SDValue(); // Limit search depth.
43894
43895 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43896 unsigned Opc = V.getOpcode();
43897 switch (Opc) {
43898 case ISD::BITCAST: {
43899 // Bitcast from a vector/float/double, we can cheaply bitcast to VT.
43900 SDValue Src = V.getOperand(0);
43901 EVT SrcVT = Src.getValueType();
43902 if (SrcVT.isVector() || SrcVT.isFloatingPoint())
43903 return DAG.getBitcast(VT, Src);
43904 break;
43905 }
43906 case ISD::Constant: {
43907 auto *C = cast<ConstantSDNode>(V);
43908 if (C->isZero())
43909 return DAG.getConstant(0, DL, VT);
43910 if (C->isAllOnes())
43911 return DAG.getAllOnesConstant(DL, VT);
43912 break;
43913 }
43914 case ISD::TRUNCATE: {
43915 // If we find a suitable source, a truncated scalar becomes a subvector.
43916 SDValue Src = V.getOperand(0);
43917 EVT NewSrcVT =
43918 EVT::getVectorVT(*DAG.getContext(), MVT::i1, Src.getValueSizeInBits());
43919 if (TLI.isTypeLegal(NewSrcVT))
43920 if (SDValue N0 = combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG,
43921 Subtarget, Depth + 1))
43922 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N0,
43923 DAG.getIntPtrConstant(0, DL));
43924 break;
43925 }
43926 case ISD::ANY_EXTEND:
43927 case ISD::ZERO_EXTEND: {
43928 // If we find a suitable source, an extended scalar becomes a subvector.
43929 SDValue Src = V.getOperand(0);
43930 EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
43931 Src.getScalarValueSizeInBits());
43932 if (TLI.isTypeLegal(NewSrcVT))
43933 if (SDValue N0 = combineBitcastToBoolVector(NewSrcVT, Src, DL, DAG,
43934 Subtarget, Depth + 1))
43935 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
43936 Opc == ISD::ANY_EXTEND ? DAG.getUNDEF(VT)
43937 : DAG.getConstant(0, DL, VT),
43938 N0, DAG.getIntPtrConstant(0, DL));
43939 break;
43940 }
43941 case ISD::OR:
43942 case ISD::XOR: {
43943 // If we find suitable sources, we can just move the op to the vector
43944 // domain.
43945 if (SDValue N0 = combineBitcastToBoolVector(VT, V.getOperand(0), DL, DAG,
43946 Subtarget, Depth + 1))
43947 if (SDValue N1 = combineBitcastToBoolVector(VT, V.getOperand(1), DL, DAG,
43948 Subtarget, Depth + 1))
43949 return DAG.getNode(Opc, DL, VT, N0, N1);
43950 break;
43951 }
43952 case ISD::SHL: {
43953 // If we find a suitable source, a SHL becomes a KSHIFTL.
43954 SDValue Src0 = V.getOperand(0);
43955 if ((VT == MVT::v8i1 && !Subtarget.hasDQI()) ||
43956 ((VT == MVT::v32i1 || VT == MVT::v64i1) && !Subtarget.hasBWI()))
43957 break;
43958
43959 if (auto *Amt = dyn_cast<ConstantSDNode>(V.getOperand(1)))
43960 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget,
43961 Depth + 1))
43962 return DAG.getNode(
43963 X86ISD::KSHIFTL, DL, VT, N0,
43964 DAG.getTargetConstant(Amt->getZExtValue(), DL, MVT::i8));
43965 break;
43966 }
43967 }
43968
43969 // Does the inner bitcast already exist?
43970 if (Depth > 0)
43971 if (SDNode *Alt = DAG.getNodeIfExists(ISD::BITCAST, DAG.getVTList(VT), {V}))
43972 return SDValue(Alt, 0);
43973
43974 return SDValue();
43975}
43976
43977static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
43978 TargetLowering::DAGCombinerInfo &DCI,
43979 const X86Subtarget &Subtarget) {
43980 SDValue N0 = N->getOperand(0);
43981 EVT VT = N->getValueType(0);
43982 EVT SrcVT = N0.getValueType();
43983 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43984
43985 // Try to match patterns such as
43986 // (i16 bitcast (v16i1 x))
43987 // ->
43988 // (i16 movmsk (16i8 sext (v16i1 x)))
43989 // before the setcc result is scalarized on subtargets that don't have legal
43990 // vxi1 types.
43991 if (DCI.isBeforeLegalize()) {
43992 SDLoc dl(N);
43993 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget))
43994 return V;
43995
43996 // If this is a bitcast between a MVT::v4i1/v2i1 and an illegal integer
43997 // type, widen both sides to avoid a trip through memory.
43998 if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() &&
43999 Subtarget.hasAVX512()) {
44000 N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, N0);
44001 N0 = DAG.getBitcast(MVT::v8i1, N0);
44002 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0,
44003 DAG.getIntPtrConstant(0, dl));
44004 }
44005
44006 // If this is a bitcast between a MVT::v4i1/v2i1 and an illegal integer
44007 // type, widen both sides to avoid a trip through memory.
44008 if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() &&
44009 Subtarget.hasAVX512()) {
44010 // Use zeros for the widening if we already have some zeroes. This can
44011 // allow SimplifyDemandedBits to remove scalar ANDs that may be down
44012 // stream of this.
44013 // FIXME: It might make sense to detect a concat_vectors with a mix of
44014 // zeroes and undef and turn it into insert_subvector for i1 vectors as
44015 // a separate combine. What we can't do is canonicalize the operands of
44016 // such a concat or we'll get into a loop with SimplifyDemandedBits.
44017 if (N0.getOpcode() == ISD::CONCAT_VECTORS) {
44018 SDValue LastOp = N0.getOperand(N0.getNumOperands() - 1);
44019 if (ISD::isBuildVectorAllZeros(LastOp.getNode())) {
44020 SrcVT = LastOp.getValueType();
44021 unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
44022 SmallVector<SDValue, 4> Ops(N0->ops());
44023 Ops.resize(NumConcats, DAG.getConstant(0, dl, SrcVT));
44024 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
44025 N0 = DAG.getBitcast(MVT::i8, N0);
44026 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
44027 }
44028 }
44029
44030 unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
44031 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT));
44032 Ops[0] = N0;
44033 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
44034 N0 = DAG.getBitcast(MVT::i8, N0);
44035 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
44036 }
44037 } else {
44038 // If we're bitcasting from iX to vXi1, see if the integer originally
44039 // began as a vXi1 and whether we can remove the bitcast entirely.
44040 if (VT.isVector() && VT.getScalarType() == MVT::i1 &&
44041 SrcVT.isScalarInteger() && TLI.isTypeLegal(VT)) {
44042 if (SDValue V =
44043 combineBitcastToBoolVector(VT, N0, SDLoc(N), DAG, Subtarget))
44044 return V;
44045 }
44046 }
44047
44048 // Look for (i8 (bitcast (v8i1 (extract_subvector (v16i1 X), 0)))) and
44049 // replace with (i8 (trunc (i16 (bitcast (v16i1 X))))). This can occur
44050 // due to insert_subvector legalization on KNL. By promoting the copy to i16
44051 // we can help with known bits propagation from the vXi1 domain to the
44052 // scalar domain.
44053 if (VT == MVT::i8 && SrcVT == MVT::v8i1 && Subtarget.hasAVX512() &&
44054 !Subtarget.hasDQI() && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
44055 N0.getOperand(0).getValueType() == MVT::v16i1 &&
44056 isNullConstant(N0.getOperand(1)))
44057 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT,
44058 DAG.getBitcast(MVT::i16, N0.getOperand(0)));
44059
44060 // Canonicalize (bitcast (vbroadcast_load)) so that the output of the bitcast
44061 // and the vbroadcast_load are both integer or both fp. In some cases this
44062 // will remove the bitcast entirely.
44063 if (N0.getOpcode() == X86ISD::VBROADCAST_LOAD && N0.hasOneUse() &&
44064 VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) {
44065 auto *BCast = cast<MemIntrinsicSDNode>(N0);
44066 unsigned SrcVTSize = SrcVT.getScalarSizeInBits();
44067 unsigned MemSize = BCast->getMemoryVT().getScalarSizeInBits();
44068 // Don't swap i8/i16 since don't have fp types that size.
44069 if (MemSize >= 32) {
44070 MVT MemVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(MemSize)
44071 : MVT::getIntegerVT(MemSize);
44072 MVT LoadVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(SrcVTSize)
44073 : MVT::getIntegerVT(SrcVTSize);
44074 LoadVT = MVT::getVectorVT(LoadVT, SrcVT.getVectorNumElements());
44075
44076 SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
44077 SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() };
44078 SDValue ResNode =
44079 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
44080 MemVT, BCast->getMemOperand());
44081 DAG.ReplaceAllUsesOfValueWith(SDValue(BCast, 1), ResNode.getValue(1));
44082 return DAG.getBitcast(VT, ResNode);
44083 }
44084 }
44085
44086 // Since MMX types are special and don't usually play with other vector types,
44087 // it's better to handle them early to be sure we emit efficient code by
44088 // avoiding store-load conversions.
44089 if (VT == MVT::x86mmx) {
44090 // Detect MMX constant vectors.
44091 APInt UndefElts;
44092 SmallVector<APInt, 1> EltBits;
44093 if (getTargetConstantBitsFromNode(N0, 64, UndefElts, EltBits,
44094 /*AllowWholeUndefs*/ true,
44095 /*AllowPartialUndefs*/ true)) {
44096 SDLoc DL(N0);
44097 // Handle zero-extension of i32 with MOVD.
44098 if (EltBits[0].countl_zero() >= 32)
44099 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT,
44100 DAG.getConstant(EltBits[0].trunc(32), DL, MVT::i32));
44101 // Else, bitcast to a double.
44102 // TODO - investigate supporting sext 32-bit immediates on x86_64.
44103 APFloat F64(APFloat::IEEEdouble(), EltBits[0]);
44104 return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64));
44105 }
44106
44107 // Detect bitcasts to x86mmx low word.
44108 if (N0.getOpcode() == ISD::BUILD_VECTOR &&
44109 (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) &&
44110 N0.getOperand(0).getValueType() == SrcVT.getScalarType()) {
44111 bool LowUndef = true, AllUndefOrZero = true;
44112 for (unsigned i = 1, e = SrcVT.getVectorNumElements(); i != e; ++i) {
44113 SDValue Op = N0.getOperand(i);
44114 LowUndef &= Op.isUndef() || (i >= e/2);
44115 AllUndefOrZero &= isNullConstantOrUndef(Op);
44116 }
44117 if (AllUndefOrZero) {
44118 SDValue N00 = N0.getOperand(0);
44119 SDLoc dl(N00);
44120 N00 = LowUndef ? DAG.getAnyExtOrTrunc(N00, dl, MVT::i32)
44121 : DAG.getZExtOrTrunc(N00, dl, MVT::i32);
44122 return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00);
44123 }
44124 }
44125
44126 // Detect bitcasts of 64-bit build vectors and convert to a
44127 // MMX UNPCK/PSHUFW which takes MMX type inputs with the value in the
44128 // lowest element.
44129 if (N0.getOpcode() == ISD::BUILD_VECTOR &&
44130 (SrcVT == MVT::v2f32 || SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 ||
44131 SrcVT == MVT::v8i8))
44132 return createMMXBuildVector(cast<BuildVectorSDNode>(N0), DAG, Subtarget);
44133
44134 // Detect bitcasts between element or subvector extraction to x86mmx.
44135 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
44136 N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) &&
44137 isNullConstant(N0.getOperand(1))) {
44138 SDValue N00 = N0.getOperand(0);
44139 if (N00.getValueType().is128BitVector())
44140 return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT,
44141 DAG.getBitcast(MVT::v2i64, N00));
44142 }
44143
44144 // Detect bitcasts from FP_TO_SINT to x86mmx.
44145 if (SrcVT == MVT::v2i32 && N0.getOpcode() == ISD::FP_TO_SINT) {
44146 SDLoc DL(N0);
44147 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
44148 DAG.getUNDEF(MVT::v2i32));
44149 return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT,
44150 DAG.getBitcast(MVT::v2i64, Res));
44151 }
44152 }
44153
44154 // Try to remove a bitcast of constant vXi1 vector. We have to legalize
44155 // most of these to scalar anyway.
44156 if (Subtarget.hasAVX512() && VT.isScalarInteger() &&
44157 SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&
44158 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
44159 return combinevXi1ConstantToInteger(N0, DAG);
44160 }
44161
44162 if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() && VT.isVector() &&
44163 VT.getVectorElementType() == MVT::i1) {
44164 if (auto *C = dyn_cast<ConstantSDNode>(N0)) {
44165 if (C->isAllOnes())
44166 return DAG.getConstant(1, SDLoc(N0), VT);
44167 if (C->isZero())
44168 return DAG.getConstant(0, SDLoc(N0), VT);
44169 }
44170 }
44171
44172 // Look for MOVMSK that is maybe truncated and then bitcasted to vXi1.
44173 // Turn it into a sign bit compare that produces a k-register. This avoids
44174 // a trip through a GPR.
44175 if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() &&
44176 VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
44177 isPowerOf2_32(VT.getVectorNumElements())) {
44178 unsigned NumElts = VT.getVectorNumElements();
44179 SDValue Src = N0;
44180
44181 // Peek through truncate.
44182 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse())
44183 Src = N0.getOperand(0);
44184
44185 if (Src.getOpcode() == X86ISD::MOVMSK && Src.hasOneUse()) {
44186 SDValue MovmskIn = Src.getOperand(0);
44187 MVT MovmskVT = MovmskIn.getSimpleValueType();
44188 unsigned MovMskElts = MovmskVT.getVectorNumElements();
44189
44190 // We allow extra bits of the movmsk to be used since they are known zero.
44191 // We can't convert a VPMOVMSKB without avx512bw.
44192 if (MovMskElts <= NumElts &&
44193 (Subtarget.hasBWI() || MovmskVT.getVectorElementType() != MVT::i8)) {
44194 EVT IntVT = EVT(MovmskVT).changeVectorElementTypeToInteger();
44195 MovmskIn = DAG.getBitcast(IntVT, MovmskIn);
44196 SDLoc dl(N);
44197 MVT CmpVT = MVT::getVectorVT(MVT::i1, MovMskElts);
44198 SDValue Cmp = DAG.getSetCC(dl, CmpVT, MovmskIn,
44199 DAG.getConstant(0, dl, IntVT), ISD::SETLT);
44200 if (EVT(CmpVT) == VT)
44201 return Cmp;
44202
44203 // Pad with zeroes up to original VT to replace the zeroes that were
44204 // being used from the MOVMSK.
44205 unsigned NumConcats = NumElts / MovMskElts;
44206 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, CmpVT));
44207 Ops[0] = Cmp;
44208 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Ops);
44209 }
44210 }
44211 }
44212
44213 // Try to remove bitcasts from input and output of mask arithmetic to
44214 // remove GPR<->K-register crossings.
44215 if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
44216 return V;
44217
44218 // Convert a bitcasted integer logic operation that has one bitcasted
44219 // floating-point operand into a floating-point logic operation. This may
44220 // create a load of a constant, but that is cheaper than materializing the
44221 // constant in an integer register and transferring it to an SSE register or
44222 // transferring the SSE operand to integer register and back.
44223 unsigned FPOpcode;
44224 switch (N0.getOpcode()) {
44225 // clang-format off
44226 case ISD::AND: FPOpcode = X86ISD::FAND; break;
44227 case ISD::OR: FPOpcode = X86ISD::FOR; break;
44228 case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
44229 default: return SDValue();
44230 // clang-format on
44231 }
44232
44233 // Check if we have a bitcast from another integer type as well.
44234 if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||
44235 (Subtarget.hasSSE2() && VT == MVT::f64) ||
44236 (Subtarget.hasFP16() && VT == MVT::f16) ||
44237 (Subtarget.hasSSE2() && VT.isInteger() && VT.isVector() &&
44238 TLI.isTypeLegal(VT))))
44239 return SDValue();
44240
44241 SDValue LogicOp0 = N0.getOperand(0);
44242 SDValue LogicOp1 = N0.getOperand(1);
44243 SDLoc DL0(N0);
44244
44245 // bitcast(logic(bitcast(X), Y)) --> logic'(X, bitcast(Y))
44246 if (N0.hasOneUse() && LogicOp0.getOpcode() == ISD::BITCAST &&
44247 LogicOp0.hasOneUse() && LogicOp0.getOperand(0).hasOneUse() &&
44248 LogicOp0.getOperand(0).getValueType() == VT &&
44249 !isa<ConstantSDNode>(LogicOp0.getOperand(0))) {
44250 SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1);
44251 unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode();
44252 return DAG.getNode(Opcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1);
44253 }
44254 // bitcast(logic(X, bitcast(Y))) --> logic'(bitcast(X), Y)
44255 if (N0.hasOneUse() && LogicOp1.getOpcode() == ISD::BITCAST &&
44256 LogicOp1.hasOneUse() && LogicOp1.getOperand(0).hasOneUse() &&
44257 LogicOp1.getOperand(0).getValueType() == VT &&
44258 !isa<ConstantSDNode>(LogicOp1.getOperand(0))) {
44259 SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0);
44260 unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode();
44261 return DAG.getNode(Opcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0);
44262 }
44263
44264 return SDValue();
44265}
44266
44267// (mul (zext a), (sext, b))
44268static bool detectExtMul(SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0,
44269 SDValue &Op1) {
44270 Op0 = Mul.getOperand(0);
44271 Op1 = Mul.getOperand(1);
44272
44273 // The operand1 should be signed extend
44274 if (Op0.getOpcode() == ISD::SIGN_EXTEND)
44275 std::swap(Op0, Op1);
44276
44277 auto IsFreeTruncation = [](SDValue &Op) -> bool {
44278 if ((Op.getOpcode() == ISD::ZERO_EXTEND ||
44279 Op.getOpcode() == ISD::SIGN_EXTEND) &&
44280 Op.getOperand(0).getScalarValueSizeInBits() <= 8)
44281 return true;
44282
44283 auto *BV = dyn_cast<BuildVectorSDNode>(Op);
44284 return (BV && BV->isConstant());
44285 };
44286
44287 // (dpbusd (zext a), (sext, b)). Since the first operand should be unsigned
44288 // value, we need to check Op0 is zero extended value. Op1 should be signed
44289 // value, so we just check the signed bits.
44290 if ((IsFreeTruncation(Op0) &&
44291 DAG.computeKnownBits(Op0).countMaxActiveBits() <= 8) &&
44292 (IsFreeTruncation(Op1) && DAG.ComputeMaxSignificantBits(Op1) <= 8))
44293 return true;
44294
44295 return false;
44296}
44297
44298// Given a ABS node, detect the following pattern:
44299// (ABS (SUB (ZERO_EXTEND a), (ZERO_EXTEND b))).
44300// This is useful as it is the input into a SAD pattern.
44301static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
44302 SDValue AbsOp1 = Abs->getOperand(0);
44303 if (AbsOp1.getOpcode() != ISD::SUB)
44304 return false;
44305
44306 Op0 = AbsOp1.getOperand(0);
44307 Op1 = AbsOp1.getOperand(1);
44308
44309 // Check if the operands of the sub are zero-extended from vectors of i8.
44310 if (Op0.getOpcode() != ISD::ZERO_EXTEND ||
44311 Op0.getOperand(0).getValueType().getVectorElementType() != MVT::i8 ||
44312 Op1.getOpcode() != ISD::ZERO_EXTEND ||
44313 Op1.getOperand(0).getValueType().getVectorElementType() != MVT::i8)
44314 return false;
44315
44316 return true;
44317}
44318
44319static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS,
44320 unsigned &LogBias, const SDLoc &DL,
44321 const X86Subtarget &Subtarget) {
44322 // Extend or truncate to MVT::i8 first.
44323 MVT Vi8VT =
44324 MVT::getVectorVT(MVT::i8, LHS.getValueType().getVectorElementCount());
44325 LHS = DAG.getZExtOrTrunc(LHS, DL, Vi8VT);
44326 RHS = DAG.getSExtOrTrunc(RHS, DL, Vi8VT);
44327
44328 // VPDPBUSD(<16 x i32>C, <16 x i8>A, <16 x i8>B). For each dst element
44329 // C[0] = C[0] + A[0]B[0] + A[1]B[1] + A[2]B[2] + A[3]B[3].
44330 // The src A, B element type is i8, but the dst C element type is i32.
44331 // When we calculate the reduce stage, we use src vector type vXi8 for it
44332 // so we need logbias 2 to avoid extra 2 stages.
44333 LogBias = 2;
44334
44335 unsigned RegSize = std::max(128u, (unsigned)Vi8VT.getSizeInBits());
44336 if (Subtarget.hasVNNI() && !Subtarget.hasVLX())
44337 RegSize = std::max(512u, RegSize);
44338
44339 // "Zero-extend" the i8 vectors. This is not a per-element zext, rather we
44340 // fill in the missing vector elements with 0.
44341 unsigned NumConcat = RegSize / Vi8VT.getSizeInBits();
44342 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, Vi8VT));
44343 Ops[0] = LHS;
44344 MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8);
44345 SDValue DpOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
44346 Ops[0] = RHS;
44347 SDValue DpOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
44348
44349 // Actually build the DotProduct, split as 256/512 bits for
44350 // AVXVNNI/AVX512VNNI.
44351 auto DpBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
44352 ArrayRef<SDValue> Ops) {
44353 MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
44354 return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops);
44355 };
44356 MVT DpVT = MVT::getVectorVT(MVT::i32, RegSize / 32);
44357 SDValue Zero = DAG.getConstant(0, DL, DpVT);
44358
44359 return SplitOpsAndApply(DAG, Subtarget, DL, DpVT, {Zero, DpOp0, DpOp1},
44360 DpBuilder, false);
44361}
44362
44363// Given two zexts of <k x i8> to <k x i32>, create a PSADBW of the inputs
44364// to these zexts.
44365static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0,
44366 const SDValue &Zext1, const SDLoc &DL,
44367 const X86Subtarget &Subtarget) {
44368 // Find the appropriate width for the PSADBW.
44369 EVT InVT = Zext0.getOperand(0).getValueType();
44370 unsigned RegSize = std::max(128u, (unsigned)InVT.getSizeInBits());
44371
44372 // "Zero-extend" the i8 vectors. This is not a per-element zext, rather we
44373 // fill in the missing vector elements with 0.
44374 unsigned NumConcat = RegSize / InVT.getSizeInBits();
44375 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT));
44376 Ops[0] = Zext0.getOperand(0);
44377 MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8);
44378 SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
44379 Ops[0] = Zext1.getOperand(0);
44380 SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
44381
44382 // Actually build the SAD, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
44383 auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
44384 ArrayRef<SDValue> Ops) {
44385 MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
44386 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops);
44387 };
44388 MVT SadVT = MVT::getVectorVT(MVT::i64, RegSize / 64);
44389 return SplitOpsAndApply(DAG, Subtarget, DL, SadVT, { SadOp0, SadOp1 },
44390 PSADBWBuilder);
44391}
44392
44393// Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
44394// PHMINPOSUW.
44395static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG,
44396 const X86Subtarget &Subtarget) {
44397 // Bail without SSE41.
44398 if (!Subtarget.hasSSE41())
44399 return SDValue();
44400
44401 EVT ExtractVT = Extract->getValueType(0);
44402 if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)
44403 return SDValue();
44404
44405 // Check for SMAX/SMIN/UMAX/UMIN horizontal reduction patterns.
44406 ISD::NodeType BinOp;
44407 SDValue Src = DAG.matchBinOpReduction(
44408 Extract, BinOp, {ISD::SMAX, ISD::SMIN, ISD::UMAX, ISD::UMIN}, true);
44409 if (!Src)
44410 return SDValue();
44411
44412 EVT SrcVT = Src.getValueType();
44413 EVT SrcSVT = SrcVT.getScalarType();
44414 if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0)
44415 return SDValue();
44416
44417 SDLoc DL(Extract);
44418 SDValue MinPos = Src;
44419
44420 // First, reduce the source down to 128-bit, applying BinOp to lo/hi.
44421 while (SrcVT.getSizeInBits() > 128) {
44422 SDValue Lo, Hi;
44423 std::tie(Lo, Hi) = splitVector(MinPos, DAG, DL);
44424 SrcVT = Lo.getValueType();
44425 MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
44426 }
44427 assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) ||
44428 (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
44429 "Unexpected value type");
44430
44431 // PHMINPOSUW applies to UMIN(v8i16), for SMIN/SMAX/UMAX we must apply a mask
44432 // to flip the value accordingly.
44433 SDValue Mask;
44434 unsigned MaskEltsBits = ExtractVT.getSizeInBits();
44435 if (BinOp == ISD::SMAX)
44436 Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
44437 else if (BinOp == ISD::SMIN)
44438 Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
44439 else if (BinOp == ISD::UMAX)
44440 Mask = DAG.getAllOnesConstant(DL, SrcVT);
44441
44442 if (Mask)
44443 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
44444
44445 // For v16i8 cases we need to perform UMIN on pairs of byte elements,
44446 // shuffling each upper element down and insert zeros. This means that the
44447 // v16i8 UMIN will leave the upper element as zero, performing zero-extension
44448 // ready for the PHMINPOS.
44449 if (ExtractVT == MVT::i8) {
44450 SDValue Upper = DAG.getVectorShuffle(
44451 SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
44452 {1, 16, 3, 16, 5, 16, 7, 16, 9, 16, 11, 16, 13, 16, 15, 16});
44453 MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
44454 }
44455
44456 // Perform the PHMINPOS on a v8i16 vector,
44457 MinPos = DAG.getBitcast(MVT::v8i16, MinPos);
44458 MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
44459 MinPos = DAG.getBitcast(SrcVT, MinPos);
44460
44461 if (Mask)
44462 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
44463
44464 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
44465 DAG.getIntPtrConstant(0, DL));
44466}
44467
44468// Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
44469static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG,
44470 const X86Subtarget &Subtarget) {
44471 // Bail without SSE2.
44472 if (!Subtarget.hasSSE2())
44473 return SDValue();
44474
44475 EVT ExtractVT = Extract->getValueType(0);
44476 unsigned BitWidth = ExtractVT.getSizeInBits();
44477 if (ExtractVT != MVT::i64 && ExtractVT != MVT::i32 && ExtractVT != MVT::i16 &&
44478 ExtractVT != MVT::i8 && ExtractVT != MVT::i1)
44479 return SDValue();
44480
44481 // Check for OR(any_of)/AND(all_of)/XOR(parity) horizontal reduction patterns.
44482 ISD::NodeType BinOp;
44483 SDValue Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::OR, ISD::AND});
44484 if (!Match && ExtractVT == MVT::i1)
44485 Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::XOR});
44486 if (!Match)
44487 return SDValue();
44488
44489 // EXTRACT_VECTOR_ELT can require implicit extension of the vector element
44490 // which we can't support here for now.
44491 if (Match.getScalarValueSizeInBits() != BitWidth)
44492 return SDValue();
44493
44494 SDValue Movmsk;
44495 SDLoc DL(Extract);
44496 EVT MatchVT = Match.getValueType();
44497 unsigned NumElts = MatchVT.getVectorNumElements();
44498 unsigned MaxElts = Subtarget.hasInt256() ? 32 : 16;
44499 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44500 LLVMContext &Ctx = *DAG.getContext();
44501
44502 if (ExtractVT == MVT::i1) {
44503 // Special case for (pre-legalization) vXi1 reductions.
44504 if (NumElts > 64 || !isPowerOf2_32(NumElts))
44505 return SDValue();
44506 if (Match.getOpcode() == ISD::SETCC) {
44507 ISD::CondCode CC = cast<CondCodeSDNode>(Match.getOperand(2))->get();
44508 if ((BinOp == ISD::AND && CC == ISD::CondCode::SETEQ) ||
44509 (BinOp == ISD::OR && CC == ISD::CondCode::SETNE)) {
44510 // For all_of(setcc(x,y,eq)) - use (iX)x == (iX)y.
44511 // For any_of(setcc(x,y,ne)) - use (iX)x != (iX)y.
44512 X86::CondCode X86CC;
44513 SDValue LHS = DAG.getFreeze(Match.getOperand(0));
44514 SDValue RHS = DAG.getFreeze(Match.getOperand(1));
44515 APInt Mask = APInt::getAllOnes(LHS.getScalarValueSizeInBits());
44516 if (SDValue V = LowerVectorAllEqual(DL, LHS, RHS, CC, Mask, Subtarget,
44517 DAG, X86CC))
44518 return DAG.getNode(ISD::TRUNCATE, DL, ExtractVT,
44519 getSETCC(X86CC, V, DL, DAG));
44520 }
44521 }
44522 if (TLI.isTypeLegal(MatchVT)) {
44523 // If this is a legal AVX512 predicate type then we can just bitcast.
44524 EVT MovmskVT = EVT::getIntegerVT(Ctx, NumElts);
44525 Movmsk = DAG.getBitcast(MovmskVT, Match);
44526 } else {
44527 // Use combineBitcastvxi1 to create the MOVMSK.
44528 while (NumElts > MaxElts) {
44529 SDValue Lo, Hi;
44530 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
44531 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
44532 NumElts /= 2;
44533 }
44534 EVT MovmskVT = EVT::getIntegerVT(Ctx, NumElts);
44535 Movmsk = combineBitcastvxi1(DAG, MovmskVT, Match, DL, Subtarget);
44536 }
44537 if (!Movmsk)
44538 return SDValue();
44539 Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32);
44540 } else {
44541 // FIXME: Better handling of k-registers or 512-bit vectors?
44542 unsigned MatchSizeInBits = Match.getValueSizeInBits();
44543 if (!(MatchSizeInBits == 128 ||
44544 (MatchSizeInBits == 256 && Subtarget.hasAVX())))
44545 return SDValue();
44546
44547 // Make sure this isn't a vector of 1 element. The perf win from using
44548 // MOVMSK diminishes with less elements in the reduction, but it is
44549 // generally better to get the comparison over to the GPRs as soon as
44550 // possible to reduce the number of vector ops.
44551 if (Match.getValueType().getVectorNumElements() < 2)
44552 return SDValue();
44553
44554 // Check that we are extracting a reduction of all sign bits.
44555 if (DAG.ComputeNumSignBits(Match) != BitWidth)
44556 return SDValue();
44557
44558 if (MatchSizeInBits == 256 && BitWidth < 32 && !Subtarget.hasInt256()) {
44559 SDValue Lo, Hi;
44560 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
44561 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
44562 MatchSizeInBits = Match.getValueSizeInBits();
44563 }
44564
44565 // For 32/64 bit comparisons use MOVMSKPS/MOVMSKPD, else PMOVMSKB.
44566 MVT MaskSrcVT;
44567 if (64 == BitWidth || 32 == BitWidth)
44568 MaskSrcVT = MVT::getVectorVT(MVT::getFloatingPointVT(BitWidth),
44569 MatchSizeInBits / BitWidth);
44570 else
44571 MaskSrcVT = MVT::getVectorVT(MVT::i8, MatchSizeInBits / 8);
44572
44573 SDValue BitcastLogicOp = DAG.getBitcast(MaskSrcVT, Match);
44574 Movmsk = getPMOVMSKB(DL, BitcastLogicOp, DAG, Subtarget);
44575 NumElts = MaskSrcVT.getVectorNumElements();
44576 }
44577 assert((NumElts <= 32 || NumElts == 64) &&
44578 "Not expecting more than 64 elements");
44579
44580 MVT CmpVT = NumElts == 64 ? MVT::i64 : MVT::i32;
44581 if (BinOp == ISD::XOR) {
44582 // parity -> (PARITY(MOVMSK X))
44583 SDValue Result = DAG.getNode(ISD::PARITY, DL, CmpVT, Movmsk);
44584 return DAG.getZExtOrTrunc(Result, DL, ExtractVT);
44585 }
44586
44587 SDValue CmpC;
44588 ISD::CondCode CondCode;
44589 if (BinOp == ISD::OR) {
44590 // any_of -> MOVMSK != 0
44591 CmpC = DAG.getConstant(0, DL, CmpVT);
44592 CondCode = ISD::CondCode::SETNE;
44593 } else {
44594 // all_of -> MOVMSK == ((1 << NumElts) - 1)
44595 CmpC = DAG.getConstant(APInt::getLowBitsSet(CmpVT.getSizeInBits(), NumElts),
44596 DL, CmpVT);
44597 CondCode = ISD::CondCode::SETEQ;
44598 }
44599
44600 // The setcc produces an i8 of 0/1, so extend that to the result width and
44601 // negate to get the final 0/-1 mask value.
44602 EVT SetccVT = TLI.getSetCCResultType(DAG.getDataLayout(), Ctx, CmpVT);
44603 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode);
44604 SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT);
44605 return DAG.getNegative(Zext, DL, ExtractVT);
44606}
44607
44608static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
44609 const X86Subtarget &Subtarget) {
44610 if (!Subtarget.hasVNNI() && !Subtarget.hasAVXVNNI())
44611 return SDValue();
44612
44613 EVT ExtractVT = Extract->getValueType(0);
44614 // Verify the type we're extracting is i32, as the output element type of
44615 // vpdpbusd is i32.
44616 if (ExtractVT != MVT::i32)
44617 return SDValue();
44618
44619 EVT VT = Extract->getOperand(0).getValueType();
44620 if (!isPowerOf2_32(VT.getVectorNumElements()))
44621 return SDValue();
44622
44623 // Match shuffle + add pyramid.
44624 ISD::NodeType BinOp;
44625 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
44626
44627 // We can't combine to vpdpbusd for zext, because each of the 4 multiplies
44628 // done by vpdpbusd compute a signed 16-bit product that will be sign extended
44629 // before adding into the accumulator.
44630 // TODO:
44631 // We also need to verify that the multiply has at least 2x the number of bits
44632 // of the input. We shouldn't match
44633 // (sign_extend (mul (vXi9 (zext (vXi8 X))), (vXi9 (zext (vXi8 Y)))).
44634 // if (Root && (Root.getOpcode() == ISD::SIGN_EXTEND))
44635 // Root = Root.getOperand(0);
44636
44637 // If there was a match, we want Root to be a mul.
44638 if (!Root || Root.getOpcode() != ISD::MUL)
44639 return SDValue();
44640
44641 // Check whether we have an extend and mul pattern
44642 SDValue LHS, RHS;
44643 if (!detectExtMul(DAG, Root, LHS, RHS))
44644 return SDValue();
44645
44646 // Create the dot product instruction.
44647 SDLoc DL(Extract);
44648 unsigned StageBias;
44649 SDValue DP = createVPDPBUSD(DAG, LHS, RHS, StageBias, DL, Subtarget);
44650
44651 // If the original vector was wider than 4 elements, sum over the results
44652 // in the DP vector.
44653 unsigned Stages = Log2_32(VT.getVectorNumElements());
44654 EVT DpVT = DP.getValueType();
44655
44656 if (Stages > StageBias) {
44657 unsigned DpElems = DpVT.getVectorNumElements();
44658
44659 for (unsigned i = Stages - StageBias; i > 0; --i) {
44660 SmallVector<int, 16> Mask(DpElems, -1);
44661 for (unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
44662 Mask[j] = MaskEnd + j;
44663
44664 SDValue Shuffle =
44665 DAG.getVectorShuffle(DpVT, DL, DP, DAG.getUNDEF(DpVT), Mask);
44666 DP = DAG.getNode(ISD::ADD, DL, DpVT, DP, Shuffle);
44667 }
44668 }
44669
44670 // Return the lowest ExtractSizeInBits bits.
44671 EVT ResVT =
44672 EVT::getVectorVT(*DAG.getContext(), ExtractVT,
44673 DpVT.getSizeInBits() / ExtractVT.getSizeInBits());
44674 DP = DAG.getBitcast(ResVT, DP);
44675 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, DP,
44676 Extract->getOperand(1));
44677}
44678
44679static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
44680 const X86Subtarget &Subtarget) {
44681 // PSADBW is only supported on SSE2 and up.
44682 if (!Subtarget.hasSSE2())
44683 return SDValue();
44684
44685 EVT ExtractVT = Extract->getValueType(0);
44686 // Verify the type we're extracting is either i32 or i64.
44687 // FIXME: Could support other types, but this is what we have coverage for.
44688 if (ExtractVT != MVT::i32 && ExtractVT != MVT::i64)
44689 return SDValue();
44690
44691 EVT VT = Extract->getOperand(0).getValueType();
44692 if (!isPowerOf2_32(VT.getVectorNumElements()))
44693 return SDValue();
44694
44695 // Match shuffle + add pyramid.
44696 ISD::NodeType BinOp;
44697 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
44698
44699 // The operand is expected to be zero extended from i8
44700 // (verified in detectZextAbsDiff).
44701 // In order to convert to i64 and above, additional any/zero/sign
44702 // extend is expected.
44703 // The zero extend from 32 bit has no mathematical effect on the result.
44704 // Also the sign extend is basically zero extend
44705 // (extends the sign bit which is zero).
44706 // So it is correct to skip the sign/zero extend instruction.
44707 if (Root && (Root.getOpcode() == ISD::SIGN_EXTEND ||
44708 Root.getOpcode() == ISD::ZERO_EXTEND ||
44709 Root.getOpcode() == ISD::ANY_EXTEND))
44710 Root = Root.getOperand(0);
44711
44712 // If there was a match, we want Root to be a select that is the root of an
44713 // abs-diff pattern.
44714 if (!Root || Root.getOpcode() != ISD::ABS)
44715 return SDValue();
44716
44717 // Check whether we have an abs-diff pattern feeding into the select.
44718 SDValue Zext0, Zext1;
44719 if (!detectZextAbsDiff(Root, Zext0, Zext1))
44720 return SDValue();
44721
44722 // Create the SAD instruction.
44723 SDLoc DL(Extract);
44724 SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget);
44725
44726 // If the original vector was wider than 8 elements, sum over the results
44727 // in the SAD vector.
44728 unsigned Stages = Log2_32(VT.getVectorNumElements());
44729 EVT SadVT = SAD.getValueType();
44730 if (Stages > 3) {
44731 unsigned SadElems = SadVT.getVectorNumElements();
44732
44733 for(unsigned i = Stages - 3; i > 0; --i) {
44734 SmallVector<int, 16> Mask(SadElems, -1);
44735 for(unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
44736 Mask[j] = MaskEnd + j;
44737
44738 SDValue Shuffle =
44739 DAG.getVectorShuffle(SadVT, DL, SAD, DAG.getUNDEF(SadVT), Mask);
44740 SAD = DAG.getNode(ISD::ADD, DL, SadVT, SAD, Shuffle);
44741 }
44742 }
44743
44744 unsigned ExtractSizeInBits = ExtractVT.getSizeInBits();
44745 // Return the lowest ExtractSizeInBits bits.
44746 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), ExtractVT,
44747 SadVT.getSizeInBits() / ExtractSizeInBits);
44748 SAD = DAG.getBitcast(ResVT, SAD);
44749 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, SAD,
44750 Extract->getOperand(1));
44751}
44752
44753// If this extract is from a loaded vector value and will be used as an
44754// integer, that requires a potentially expensive XMM -> GPR transfer.
44755// Additionally, if we can convert to a scalar integer load, that will likely
44756// be folded into a subsequent integer op.
44757// Note: SrcVec might not have a VecVT type, but it must be the same size.
44758// Note: Unlike the related fold for this in DAGCombiner, this is not limited
44759// to a single-use of the loaded vector. For the reasons above, we
44760// expect this to be profitable even if it creates an extra load.
44761static SDValue
44762combineExtractFromVectorLoad(SDNode *N, EVT VecVT, SDValue SrcVec, uint64_t Idx,
44763 const SDLoc &dl, SelectionDAG &DAG,
44764 TargetLowering::DAGCombinerInfo &DCI) {
44765 assert(N->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
44766 "Only EXTRACT_VECTOR_ELT supported so far");
44767
44768 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44769 EVT VT = N->getValueType(0);
44770
44771 bool LikelyUsedAsVector = any_of(N->uses(), [](SDNode *Use) {
44772 return Use->getOpcode() == ISD::STORE ||
44773 Use->getOpcode() == ISD::INSERT_VECTOR_ELT ||
44774 Use->getOpcode() == ISD::SCALAR_TO_VECTOR;
44775 });
44776
44777 auto *LoadVec = dyn_cast<LoadSDNode>(SrcVec);
44778 if (LoadVec && ISD::isNormalLoad(LoadVec) && VT.isInteger() &&
44779 VecVT.getVectorElementType() == VT &&
44780 VecVT.getSizeInBits() == SrcVec.getValueSizeInBits() &&
44781 DCI.isAfterLegalizeDAG() && !LikelyUsedAsVector && LoadVec->isSimple()) {
44782 SDValue NewPtr = TLI.getVectorElementPointer(
44783 DAG, LoadVec->getBasePtr(), VecVT, DAG.getVectorIdxConstant(Idx, dl));
44784 unsigned PtrOff = VT.getSizeInBits() * Idx / 8;
44785 MachinePointerInfo MPI = LoadVec->getPointerInfo().getWithOffset(PtrOff);
44786 Align Alignment = commonAlignment(LoadVec->getAlign(), PtrOff);
44787 SDValue Load =
44788 DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment,
44789 LoadVec->getMemOperand()->getFlags(), LoadVec->getAAInfo());
44790 DAG.makeEquivalentMemoryOrdering(LoadVec, Load);
44791 return Load;
44792 }
44793
44794 return SDValue();
44795}
44796
44797// Attempt to peek through a target shuffle and extract the scalar from the
44798// source.
44799static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
44800 TargetLowering::DAGCombinerInfo &DCI,
44801 const X86Subtarget &Subtarget) {
44802 if (DCI.isBeforeLegalizeOps())
44803 return SDValue();
44804
44805 SDLoc dl(N);
44806 SDValue Src = N->getOperand(0);
44807 SDValue Idx = N->getOperand(1);
44808
44809 EVT VT = N->getValueType(0);
44810 EVT SrcVT = Src.getValueType();
44811 EVT SrcSVT = SrcVT.getVectorElementType();
44812 unsigned SrcEltBits = SrcSVT.getSizeInBits();
44813 unsigned NumSrcElts = SrcVT.getVectorNumElements();
44814
44815 // Don't attempt this for boolean mask vectors or unknown extraction indices.
44816 if (SrcSVT == MVT::i1 || !isa<ConstantSDNode>(Idx))
44817 return SDValue();
44818
44819 const APInt &IdxC = N->getConstantOperandAPInt(1);
44820 if (IdxC.uge(NumSrcElts))
44821 return SDValue();
44822
44823 SDValue SrcBC = peekThroughBitcasts(Src);
44824
44825 // Handle extract(bitcast(broadcast(scalar_value))).
44826 if (X86ISD::VBROADCAST == SrcBC.getOpcode()) {
44827 SDValue SrcOp = SrcBC.getOperand(0);
44828 EVT SrcOpVT = SrcOp.getValueType();
44829 if (SrcOpVT.isScalarInteger() && VT.isInteger() &&
44830 (SrcOpVT.getSizeInBits() % SrcEltBits) == 0) {
44831 unsigned Scale = SrcOpVT.getSizeInBits() / SrcEltBits;
44832 unsigned Offset = IdxC.urem(Scale) * SrcEltBits;
44833 // TODO support non-zero offsets.
44834 if (Offset == 0) {
44835 SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, SrcVT.getScalarType());
44836 SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, VT);
44837 return SrcOp;
44838 }
44839 }
44840 }
44841
44842 // If we're extracting a single element from a broadcast load and there are
44843 // no other users, just create a single load.
44844 if (SrcBC.getOpcode() == X86ISD::VBROADCAST_LOAD && SrcBC.hasOneUse()) {
44845 auto *MemIntr = cast<MemIntrinsicSDNode>(SrcBC);
44846 unsigned SrcBCWidth = SrcBC.getScalarValueSizeInBits();
44847 if (MemIntr->getMemoryVT().getSizeInBits() == SrcBCWidth &&
44848 VT.getSizeInBits() == SrcBCWidth && SrcEltBits == SrcBCWidth) {
44849 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
44850 MemIntr->getBasePtr(),
44851 MemIntr->getPointerInfo(),
44852 MemIntr->getOriginalAlign(),
44853 MemIntr->getMemOperand()->getFlags());
44854 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
44855 return Load;
44856 }
44857 }
44858
44859 // Handle extract(bitcast(scalar_to_vector(scalar_value))) for integers.
44860 // TODO: Move to DAGCombine?
44861 if (SrcBC.getOpcode() == ISD::SCALAR_TO_VECTOR && VT.isInteger() &&
44862 SrcBC.getValueType().isInteger() &&
44863 (SrcBC.getScalarValueSizeInBits() % SrcEltBits) == 0 &&
44864 SrcBC.getScalarValueSizeInBits() ==
44865 SrcBC.getOperand(0).getValueSizeInBits()) {
44866 unsigned Scale = SrcBC.getScalarValueSizeInBits() / SrcEltBits;
44867 if (IdxC.ult(Scale)) {
44868 unsigned Offset = IdxC.getZExtValue() * SrcVT.getScalarSizeInBits();
44869 SDValue Scl = SrcBC.getOperand(0);
44870 EVT SclVT = Scl.getValueType();
44871 if (Offset) {
44872 Scl = DAG.getNode(ISD::SRL, dl, SclVT, Scl,
44873 DAG.getShiftAmountConstant(Offset, SclVT, dl));
44874 }
44875 Scl = DAG.getZExtOrTrunc(Scl, dl, SrcVT.getScalarType());
44876 Scl = DAG.getZExtOrTrunc(Scl, dl, VT);
44877 return Scl;
44878 }
44879 }
44880
44881 // Handle extract(truncate(x)) for 0'th index.
44882 // TODO: Treat this as a faux shuffle?
44883 // TODO: When can we use this for general indices?
44884 if (ISD::TRUNCATE == Src.getOpcode() && IdxC == 0 &&
44885 (SrcVT.getSizeInBits() % 128) == 0) {
44886 Src = extract128BitVector(Src.getOperand(0), 0, DAG, dl);
44887 MVT ExtractVT = MVT::getVectorVT(SrcSVT.getSimpleVT(), 128 / SrcEltBits);
44888 return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src),
44889 Idx);
44890 }
44891
44892 // We can only legally extract other elements from 128-bit vectors and in
44893 // certain circumstances, depending on SSE-level.
44894 // TODO: Investigate float/double extraction if it will be just stored.
44895 auto GetLegalExtract = [&Subtarget, &DAG, &dl](SDValue Vec, EVT VecVT,
44896 unsigned Idx) {
44897 EVT VecSVT = VecVT.getScalarType();
44898 if ((VecVT.is256BitVector() || VecVT.is512BitVector()) &&
44899 (VecSVT == MVT::i8 || VecSVT == MVT::i16 || VecSVT == MVT::i32 ||
44900 VecSVT == MVT::i64)) {
44901 unsigned EltSizeInBits = VecSVT.getSizeInBits();
44902 unsigned NumEltsPerLane = 128 / EltSizeInBits;
44903 unsigned LaneOffset = (Idx & ~(NumEltsPerLane - 1)) * EltSizeInBits;
44904 unsigned LaneIdx = LaneOffset / Vec.getScalarValueSizeInBits();
44905 VecVT = EVT::getVectorVT(*DAG.getContext(), VecSVT, NumEltsPerLane);
44906 Vec = extract128BitVector(Vec, LaneIdx, DAG, dl);
44907 Idx &= (NumEltsPerLane - 1);
44908 }
44909 if ((VecVT == MVT::v4i32 || VecVT == MVT::v2i64) &&
44910 ((Idx == 0 && Subtarget.hasSSE2()) || Subtarget.hasSSE41())) {
44911 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VecVT.getScalarType(),
44912 DAG.getBitcast(VecVT, Vec),
44913 DAG.getIntPtrConstant(Idx, dl));
44914 }
44915 if ((VecVT == MVT::v8i16 && Subtarget.hasSSE2()) ||
44916 (VecVT == MVT::v16i8 && Subtarget.hasSSE41())) {
44917 unsigned OpCode = (VecVT == MVT::v8i16 ? X86ISD::PEXTRW : X86ISD::PEXTRB);
44918 return DAG.getNode(OpCode, dl, MVT::i32, DAG.getBitcast(VecVT, Vec),
44919 DAG.getTargetConstant(Idx, dl, MVT::i8));
44920 }
44921 return SDValue();
44922 };
44923
44924 // Resolve the target shuffle inputs and mask.
44925 SmallVector<int, 16> Mask;
44926 SmallVector<SDValue, 2> Ops;
44927 if (!getTargetShuffleInputs(SrcBC, Ops, Mask, DAG))
44928 return SDValue();
44929
44930 // Shuffle inputs must be the same size as the result.
44931 if (llvm::any_of(Ops, [SrcVT](SDValue Op) {
44932 return SrcVT.getSizeInBits() != Op.getValueSizeInBits();
44933 }))
44934 return SDValue();
44935
44936 // Attempt to narrow/widen the shuffle mask to the correct size.
44937 if (Mask.size() != NumSrcElts) {
44938 if ((NumSrcElts % Mask.size()) == 0) {
44939 SmallVector<int, 16> ScaledMask;
44940 int Scale = NumSrcElts / Mask.size();
44941 narrowShuffleMaskElts(Scale, Mask, ScaledMask);
44942 Mask = std::move(ScaledMask);
44943 } else if ((Mask.size() % NumSrcElts) == 0) {
44944 // Simplify Mask based on demanded element.
44945 int ExtractIdx = (int)IdxC.getZExtValue();
44946 int Scale = Mask.size() / NumSrcElts;
44947 int Lo = Scale * ExtractIdx;
44948 int Hi = Scale * (ExtractIdx + 1);
44949 for (int i = 0, e = (int)Mask.size(); i != e; ++i)
44950 if (i < Lo || Hi <= i)
44951 Mask[i] = SM_SentinelUndef;
44952
44953 SmallVector<int, 16> WidenedMask;
44954 while (Mask.size() > NumSrcElts &&
44955 canWidenShuffleElements(Mask, WidenedMask))
44956 Mask = std::move(WidenedMask);
44957 }
44958 }
44959
44960 // If narrowing/widening failed, see if we can extract+zero-extend.
44961 int ExtractIdx;
44962 EVT ExtractVT;
44963 if (Mask.size() == NumSrcElts) {
44964 ExtractIdx = Mask[IdxC.getZExtValue()];
44965 ExtractVT = SrcVT;
44966 } else {
44967 unsigned Scale = Mask.size() / NumSrcElts;
44968 if ((Mask.size() % NumSrcElts) != 0 || SrcVT.isFloatingPoint())
44969 return SDValue();
44970 unsigned ScaledIdx = Scale * IdxC.getZExtValue();
44971 if (!isUndefOrZeroInRange(Mask, ScaledIdx + 1, Scale - 1))
44972 return SDValue();
44973 ExtractIdx = Mask[ScaledIdx];
44974 EVT ExtractSVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltBits / Scale);
44975 ExtractVT = EVT::getVectorVT(*DAG.getContext(), ExtractSVT, Mask.size());
44976 assert(SrcVT.getSizeInBits() == ExtractVT.getSizeInBits() &&
44977 "Failed to widen vector type");
44978 }
44979
44980 // If the shuffle source element is undef/zero then we can just accept it.
44981 if (ExtractIdx == SM_SentinelUndef)
44982 return DAG.getUNDEF(VT);
44983
44984 if (ExtractIdx == SM_SentinelZero)
44985 return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, dl, VT)
44986 : DAG.getConstant(0, dl, VT);
44987
44988 SDValue SrcOp = Ops[ExtractIdx / Mask.size()];
44989 ExtractIdx = ExtractIdx % Mask.size();
44990 if (SDValue V = GetLegalExtract(SrcOp, ExtractVT, ExtractIdx))
44991 return DAG.getZExtOrTrunc(V, dl, VT);
44992
44993 if (N->getOpcode() == ISD::EXTRACT_VECTOR_ELT && ExtractVT == SrcVT)
44994 if (SDValue V = combineExtractFromVectorLoad(
44995 N, SrcVT, peekThroughBitcasts(SrcOp), ExtractIdx, dl, DAG, DCI))
44996 return V;
44997
44998 return SDValue();
44999}
45000
45001/// Extracting a scalar FP value from vector element 0 is free, so extract each
45002/// operand first, then perform the math as a scalar op.
45003static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG,
45004 const X86Subtarget &Subtarget) {
45005 assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Expected extract");
45006 SDValue Vec = ExtElt->getOperand(0);
45007 SDValue Index = ExtElt->getOperand(1);
45008 EVT VT = ExtElt->getValueType(0);
45009 EVT VecVT = Vec.getValueType();
45010
45011 // TODO: If this is a unary/expensive/expand op, allow extraction from a
45012 // non-zero element because the shuffle+scalar op will be cheaper?
45013 if (!Vec.hasOneUse() || !isNullConstant(Index) || VecVT.getScalarType() != VT)
45014 return SDValue();
45015
45016 // Vector FP compares don't fit the pattern of FP math ops (propagate, not
45017 // extract, the condition code), so deal with those as a special-case.
45018 if (Vec.getOpcode() == ISD::SETCC && VT == MVT::i1) {
45019 EVT OpVT = Vec.getOperand(0).getValueType().getScalarType();
45020 if (OpVT != MVT::f32 && OpVT != MVT::f64)
45021 return SDValue();
45022
45023 // extract (setcc X, Y, CC), 0 --> setcc (extract X, 0), (extract Y, 0), CC
45024 SDLoc DL(ExtElt);
45025 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
45026 Vec.getOperand(0), Index);
45027 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
45028 Vec.getOperand(1), Index);
45029 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2));
45030 }
45031
45032 if (!(VT == MVT::f16 && Subtarget.hasFP16()) && VT != MVT::f32 &&
45033 VT != MVT::f64)
45034 return SDValue();
45035
45036 // Vector FP selects don't fit the pattern of FP math ops (because the
45037 // condition has a different type and we have to change the opcode), so deal
45038 // with those here.
45039 // FIXME: This is restricted to pre type legalization by ensuring the setcc
45040 // has i1 elements. If we loosen this we need to convert vector bool to a
45041 // scalar bool.
45042 if (Vec.getOpcode() == ISD::VSELECT &&
45043 Vec.getOperand(0).getOpcode() == ISD::SETCC &&
45044 Vec.getOperand(0).getValueType().getScalarType() == MVT::i1 &&
45045 Vec.getOperand(0).getOperand(0).getValueType() == VecVT) {
45046 // ext (sel Cond, X, Y), 0 --> sel (ext Cond, 0), (ext X, 0), (ext Y, 0)
45047 SDLoc DL(ExtElt);
45048 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
45049 Vec.getOperand(0).getValueType().getScalarType(),
45050 Vec.getOperand(0), Index);
45051 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
45052 Vec.getOperand(1), Index);
45053 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
45054 Vec.getOperand(2), Index);
45055 return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2);
45056 }
45057
45058 // TODO: This switch could include FNEG and the x86-specific FP logic ops
45059 // (FAND, FANDN, FOR, FXOR). But that may require enhancements to avoid
45060 // missed load folding and fma+fneg combining.
45061 switch (Vec.getOpcode()) {
45062 case ISD::FMA: // Begin 3 operands
45063 case ISD::FMAD:
45064 case ISD::FADD: // Begin 2 operands
45065 case ISD::FSUB:
45066 case ISD::FMUL:
45067 case ISD::FDIV:
45068 case ISD::FREM:
45069 case ISD::FCOPYSIGN:
45070 case ISD::FMINNUM:
45071 case ISD::FMAXNUM:
45072 case ISD::FMINNUM_IEEE:
45073 case ISD::FMAXNUM_IEEE:
45074 case ISD::FMAXIMUM:
45075 case ISD::FMINIMUM:
45076 case X86ISD::FMAX:
45077 case X86ISD::FMIN:
45078 case ISD::FABS: // Begin 1 operand
45079 case ISD::FSQRT:
45080 case ISD::FRINT:
45081 case ISD::FCEIL:
45082 case ISD::FTRUNC:
45083 case ISD::FNEARBYINT:
45084 case ISD::FROUNDEVEN:
45085 case ISD::FROUND:
45086 case ISD::FFLOOR:
45087 case X86ISD::FRCP:
45088 case X86ISD::FRSQRT: {
45089 // extract (fp X, Y, ...), 0 --> fp (extract X, 0), (extract Y, 0), ...
45090 SDLoc DL(ExtElt);
45091 SmallVector<SDValue, 4> ExtOps;
45092 for (SDValue Op : Vec->ops())
45093 ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index));
45094 return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps);
45095 }
45096 default:
45097 return SDValue();
45098 }
45099 llvm_unreachable("All opcodes should return within switch");
45100}
45101
45102/// Try to convert a vector reduction sequence composed of binops and shuffles
45103/// into horizontal ops.
45104static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
45105 const X86Subtarget &Subtarget) {
45106 assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unexpected caller");
45107
45108 // We need at least SSE2 to anything here.
45109 if (!Subtarget.hasSSE2())
45110 return SDValue();
45111
45112 ISD::NodeType Opc;
45113 SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
45114 {ISD::ADD, ISD::MUL, ISD::FADD}, true);
45115 if (!Rdx)
45116 return SDValue();
45117
45118 SDValue Index = ExtElt->getOperand(1);
45119 assert(isNullConstant(Index) &&
45120 "Reduction doesn't end in an extract from index 0");
45121
45122 EVT VT = ExtElt->getValueType(0);
45123 EVT VecVT = Rdx.getValueType();
45124 if (VecVT.getScalarType() != VT)
45125 return SDValue();
45126
45127 SDLoc DL(ExtElt);
45128 unsigned NumElts = VecVT.getVectorNumElements();
45129 unsigned EltSizeInBits = VecVT.getScalarSizeInBits();
45130
45131 // Extend v4i8/v8i8 vector to v16i8, with undef upper 64-bits.
45132 auto WidenToV16I8 = [&](SDValue V, bool ZeroExtend) {
45133 if (V.getValueType() == MVT::v4i8) {
45134 if (ZeroExtend && Subtarget.hasSSE41()) {
45135 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,
45136 DAG.getConstant(0, DL, MVT::v4i32),
45137 DAG.getBitcast(MVT::i32, V),
45138 DAG.getIntPtrConstant(0, DL));
45139 return DAG.getBitcast(MVT::v16i8, V);
45140 }
45141 V = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, V,
45142 ZeroExtend ? DAG.getConstant(0, DL, MVT::v4i8)
45143 : DAG.getUNDEF(MVT::v4i8));
45144 }
45145 return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V,
45146 DAG.getUNDEF(MVT::v8i8));
45147 };
45148
45149 // vXi8 mul reduction - promote to vXi16 mul reduction.
45150 if (Opc == ISD::MUL) {
45151 if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))
45152 return SDValue();
45153 if (VecVT.getSizeInBits() >= 128) {
45154 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
45155 SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
45156 SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
45157 Lo = DAG.getBitcast(WideVT, Lo);
45158 Hi = DAG.getBitcast(WideVT, Hi);
45159 Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi);
45160 while (Rdx.getValueSizeInBits() > 128) {
45161 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
45162 Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi);
45163 }
45164 } else {
45165 Rdx = WidenToV16I8(Rdx, false);
45166 Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));
45167 Rdx = DAG.getBitcast(MVT::v8i16, Rdx);
45168 }
45169 if (NumElts >= 8)
45170 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
45171 DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
45172 {4, 5, 6, 7, -1, -1, -1, -1}));
45173 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
45174 DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
45175 {2, 3, -1, -1, -1, -1, -1, -1}));
45176 Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
45177 DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
45178 {1, -1, -1, -1, -1, -1, -1, -1}));
45179 Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
45180 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
45181 }
45182
45183 // vXi8 add reduction - sub 128-bit vector.
45184 if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {
45185 Rdx = WidenToV16I8(Rdx, true);
45186 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
45187 DAG.getConstant(0, DL, MVT::v16i8));
45188 Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
45189 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
45190 }
45191
45192 // Must be a >=128-bit vector with pow2 elements.
45193 if ((VecVT.getSizeInBits() % 128) != 0 || !isPowerOf2_32(NumElts))
45194 return SDValue();
45195
45196 // vXi8 add reduction - sum lo/hi halves then use PSADBW.
45197 if (VT == MVT::i8) {
45198 while (Rdx.getValueSizeInBits() > 128) {
45199 SDValue Lo, Hi;
45200 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
45201 VecVT = Lo.getValueType();
45202 Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
45203 }
45204 assert(VecVT == MVT::v16i8 && "v16i8 reduction expected");
45205
45206 SDValue Hi = DAG.getVectorShuffle(
45207 MVT::v16i8, DL, Rdx, Rdx,
45208 {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1});
45209 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi);
45210 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
45211 getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
45212 Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
45213 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
45214 }
45215
45216 // See if we can use vXi8 PSADBW add reduction for larger zext types.
45217 // If the source vector values are 0-255, then we can use PSADBW to
45218 // sum+zext v8i8 subvectors to vXi64, then perform the reduction.
45219 // TODO: See if its worth avoiding vXi16/i32 truncations?
45220 if (Opc == ISD::ADD && NumElts >= 4 && EltSizeInBits >= 16 &&
45221 DAG.computeKnownBits(Rdx).getMaxValue().ule(255) &&
45222 (EltSizeInBits == 16 || Rdx.getOpcode() == ISD::ZERO_EXTEND ||
45223 Subtarget.hasAVX512())) {
45224 if (Rdx.getValueType() == MVT::v8i16) {
45225 Rdx = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Rdx,
45226 DAG.getUNDEF(MVT::v8i16));
45227 } else {
45228 EVT ByteVT = VecVT.changeVectorElementType(MVT::i8);
45229 Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx);
45230 if (ByteVT.getSizeInBits() < 128)
45231 Rdx = WidenToV16I8(Rdx, true);
45232 }
45233
45234 // Build the PSADBW, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
45235 auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
45236 ArrayRef<SDValue> Ops) {
45237 MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
45238 SDValue Zero = DAG.getConstant(0, DL, Ops[0].getValueType());
45239 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero);
45240 };
45241 MVT SadVT = MVT::getVectorVT(MVT::i64, Rdx.getValueSizeInBits() / 64);
45242 Rdx = SplitOpsAndApply(DAG, Subtarget, DL, SadVT, {Rdx}, PSADBWBuilder);
45243
45244 // TODO: We could truncate to vXi16/vXi32 before performing the reduction.
45245 while (Rdx.getValueSizeInBits() > 128) {
45246 SDValue Lo, Hi;
45247 std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
45248 VecVT = Lo.getValueType();
45249 Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
45250 }
45251 assert(Rdx.getValueType() == MVT::v2i64 && "v2i64 reduction expected");
45252
45253 if (NumElts > 8) {
45254 SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1});
45255 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Rdx, RdxHi);
45256 }
45257
45258 VecVT = MVT::getVectorVT(VT.getSimpleVT(), 128 / VT.getSizeInBits());
45259 Rdx = DAG.getBitcast(VecVT, Rdx);
45260 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
45261 }
45262
45263 // Only use (F)HADD opcodes if they aren't microcoded or minimizes codesize.
45264 if (!shouldUseHorizontalOp(true, DAG, Subtarget))
45265 return SDValue();
45266
45267 unsigned HorizOpcode = Opc == ISD::ADD ? X86ISD::HADD : X86ISD::FHADD;
45268
45269 // 256-bit horizontal instructions operate on 128-bit chunks rather than
45270 // across the whole vector, so we need an extract + hop preliminary stage.
45271 // This is the only step where the operands of the hop are not the same value.
45272 // TODO: We could extend this to handle 512-bit or even longer vectors.
45273 if (((VecVT == MVT::v16i16 || VecVT == MVT::v8i32) && Subtarget.hasSSSE3()) ||
45274 ((VecVT == MVT::v8f32 || VecVT == MVT::v4f64) && Subtarget.hasSSE3())) {
45275 unsigned NumElts = VecVT.getVectorNumElements();
45276 SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
45277 SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
45278 Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo);
45279 VecVT = Rdx.getValueType();
45280 }
45281 if (!((VecVT == MVT::v8i16 || VecVT == MVT::v4i32) && Subtarget.hasSSSE3()) &&
45282 !((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3()))
45283 return SDValue();
45284
45285 // extract (add (shuf X), X), 0 --> extract (hadd X, X), 0
45286 unsigned ReductionSteps = Log2_32(VecVT.getVectorNumElements());
45287 for (unsigned i = 0; i != ReductionSteps; ++i)
45288 Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx);
45289
45290 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
45291}
45292
45293/// Detect vector gather/scatter index generation and convert it from being a
45294/// bunch of shuffles and extracts into a somewhat faster sequence.
45295/// For i686, the best sequence is apparently storing the value and loading
45296/// scalars back, while for x64 we should use 64-bit extracts and shifts.
45297static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
45298 TargetLowering::DAGCombinerInfo &DCI,
45299 const X86Subtarget &Subtarget) {
45300 if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget))
45301 return NewOp;
45302
45303 SDValue InputVector = N->getOperand(0);
45304 SDValue EltIdx = N->getOperand(1);
45305 auto *CIdx = dyn_cast<ConstantSDNode>(EltIdx);
45306
45307 EVT SrcVT = InputVector.getValueType();
45308 EVT VT = N->getValueType(0);
45309 SDLoc dl(InputVector);
45310 bool IsPextr = N->getOpcode() != ISD::EXTRACT_VECTOR_ELT;
45311 unsigned NumSrcElts = SrcVT.getVectorNumElements();
45312 unsigned NumEltBits = VT.getScalarSizeInBits();
45313 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45314
45315 if (CIdx && CIdx->getAPIntValue().uge(NumSrcElts))
45316 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
45317
45318 // Integer Constant Folding.
45319 if (CIdx && VT.isInteger()) {
45320 APInt UndefVecElts;
45321 SmallVector<APInt, 16> EltBits;
45322 unsigned VecEltBitWidth = SrcVT.getScalarSizeInBits();
45323 if (getTargetConstantBitsFromNode(InputVector, VecEltBitWidth, UndefVecElts,
45324 EltBits, /*AllowWholeUndefs*/ true,
45325 /*AllowPartialUndefs*/ false)) {
45326 uint64_t Idx = CIdx->getZExtValue();
45327 if (UndefVecElts[Idx])
45328 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
45329 return DAG.getConstant(EltBits[Idx].zext(NumEltBits), dl, VT);
45330 }
45331
45332 // Convert extract_element(bitcast(<X x i1>) -> bitcast(extract_subvector()).
45333 // Improves lowering of bool masks on rust which splits them into byte array.
45334 if (InputVector.getOpcode() == ISD::BITCAST && (NumEltBits % 8) == 0) {
45335 SDValue Src = peekThroughBitcasts(InputVector);
45336 if (Src.getValueType().getScalarType() == MVT::i1 &&
45337 TLI.isTypeLegal(Src.getValueType())) {
45338 MVT SubVT = MVT::getVectorVT(MVT::i1, NumEltBits);
45339 SDValue Sub = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Src,
45340 DAG.getIntPtrConstant(CIdx->getZExtValue() * NumEltBits, dl));
45341 return DAG.getBitcast(VT, Sub);
45342 }
45343 }
45344 }
45345
45346 if (IsPextr) {
45347 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumEltBits),
45348 DCI))
45349 return SDValue(N, 0);
45350
45351 // PEXTR*(PINSR*(v, s, c), c) -> s (with implicit zext handling).
45352 if ((InputVector.getOpcode() == X86ISD::PINSRB ||
45353 InputVector.getOpcode() == X86ISD::PINSRW) &&
45354 InputVector.getOperand(2) == EltIdx) {
45355 assert(SrcVT == InputVector.getOperand(0).getValueType() &&
45356 "Vector type mismatch");
45357 SDValue Scl = InputVector.getOperand(1);
45358 Scl = DAG.getNode(ISD::TRUNCATE, dl, SrcVT.getScalarType(), Scl);
45359 return DAG.getZExtOrTrunc(Scl, dl, VT);
45360 }
45361
45362 // TODO - Remove this once we can handle the implicit zero-extension of
45363 // X86ISD::PEXTRW/X86ISD::PEXTRB in combinePredicateReduction and
45364 // combineBasicSADPattern.
45365 return SDValue();
45366 }
45367
45368 // Detect mmx extraction of all bits as a i64. It works better as a bitcast.
45369 if (VT == MVT::i64 && SrcVT == MVT::v1i64 &&
45370 InputVector.getOpcode() == ISD::BITCAST &&
45371 InputVector.getOperand(0).getValueType() == MVT::x86mmx &&
45372 isNullConstant(EltIdx) && InputVector.hasOneUse())
45373 return DAG.getBitcast(VT, InputVector);
45374
45375 // Detect mmx to i32 conversion through a v2i32 elt extract.
45376 if (VT == MVT::i32 && SrcVT == MVT::v2i32 &&
45377 InputVector.getOpcode() == ISD::BITCAST &&
45378 InputVector.getOperand(0).getValueType() == MVT::x86mmx &&
45379 isNullConstant(EltIdx) && InputVector.hasOneUse())
45380 return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32,
45381 InputVector.getOperand(0));
45382
45383 // Check whether this extract is the root of a sum of absolute differences
45384 // pattern. This has to be done here because we really want it to happen
45385 // pre-legalization,
45386 if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
45387 return SAD;
45388
45389 if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget))
45390 return VPDPBUSD;
45391
45392 // Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK.
45393 if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
45394 return Cmp;
45395
45396 // Attempt to replace min/max v8i16/v16i8 reductions with PHMINPOSUW.
45397 if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget))
45398 return MinMax;
45399
45400 // Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
45401 if (SDValue V = combineArithReduction(N, DAG, Subtarget))
45402 return V;
45403
45404 if (SDValue V = scalarizeExtEltFP(N, DAG, Subtarget))
45405 return V;
45406
45407 if (CIdx)
45408 if (SDValue V = combineExtractFromVectorLoad(
45409 N, InputVector.getValueType(), InputVector, CIdx->getZExtValue(),
45410 dl, DAG, DCI))
45411 return V;
45412
45413 // Attempt to extract a i1 element by using MOVMSK to extract the signbits
45414 // and then testing the relevant element.
45415 //
45416 // Note that we only combine extracts on the *same* result number, i.e.
45417 // t0 = merge_values a0, a1, a2, a3
45418 // i1 = extract_vector_elt t0, Constant:i64<2>
45419 // i1 = extract_vector_elt t0, Constant:i64<3>
45420 // but not
45421 // i1 = extract_vector_elt t0:1, Constant:i64<2>
45422 // since the latter would need its own MOVMSK.
45423 if (SrcVT.getScalarType() == MVT::i1) {
45424 bool IsVar = !CIdx;
45425 SmallVector<SDNode *, 16> BoolExtracts;
45426 unsigned ResNo = InputVector.getResNo();
45427 auto IsBoolExtract = [&BoolExtracts, &ResNo, &IsVar](SDNode *Use) {
45428 if (Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
45429 Use->getOperand(0).getResNo() == ResNo &&
45430 Use->getValueType(0) == MVT::i1) {
45431 BoolExtracts.push_back(Use);
45432 IsVar |= !isa<ConstantSDNode>(Use->getOperand(1));
45433 return true;
45434 }
45435 return false;
45436 };
45437 // TODO: Can we drop the oneuse check for constant extracts?
45438 if (all_of(InputVector->uses(), IsBoolExtract) &&
45439 (IsVar || BoolExtracts.size() > 1)) {
45440 EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcElts);
45441 if (SDValue BC =
45442 combineBitcastvxi1(DAG, BCVT, InputVector, dl, Subtarget)) {
45443 for (SDNode *Use : BoolExtracts) {
45444 // extractelement vXi1 X, MaskIdx --> ((movmsk X) & Mask) == Mask
45445 // Mask = 1 << MaskIdx
45446 SDValue MaskIdx = DAG.getZExtOrTrunc(Use->getOperand(1), dl, MVT::i8);
45447 SDValue MaskBit = DAG.getConstant(1, dl, BCVT);
45448 SDValue Mask = DAG.getNode(ISD::SHL, dl, BCVT, MaskBit, MaskIdx);
45449 SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask);
45450 Res = DAG.getSetCC(dl, MVT::i1, Res, Mask, ISD::SETEQ);
45451 DCI.CombineTo(Use, Res);
45452 }
45453 return SDValue(N, 0);
45454 }
45455 }
45456 }
45457
45458 // Attempt to fold extract(trunc(x),c) -> trunc(extract(x,c)).
45459 if (CIdx && InputVector.getOpcode() == ISD::TRUNCATE) {
45460 SDValue TruncSrc = InputVector.getOperand(0);
45461 EVT TruncSVT = TruncSrc.getValueType().getScalarType();
45462 if (DCI.isBeforeLegalize() && TLI.isTypeLegal(TruncSVT)) {
45463 SDValue NewExt =
45464 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TruncSVT, TruncSrc, EltIdx);
45465 return DAG.getAnyExtOrTrunc(NewExt, dl, VT);
45466 }
45467 }
45468
45469 return SDValue();
45470}
45471
45472// Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
45473// This is more or less the reverse of combineBitcastvxi1.
45474static SDValue combineToExtendBoolVectorInReg(
45475 unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG,
45476 TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) {
45477 if (Opcode != ISD::SIGN_EXTEND && Opcode != ISD::ZERO_EXTEND &&
45478 Opcode != ISD::ANY_EXTEND)
45479 return SDValue();
45480 if (!DCI.isBeforeLegalizeOps())
45481 return SDValue();
45482 if (!Subtarget.hasSSE2() || Subtarget.hasAVX512())
45483 return SDValue();
45484
45485 EVT SVT = VT.getScalarType();
45486 EVT InSVT = N0.getValueType().getScalarType();
45487 unsigned EltSizeInBits = SVT.getSizeInBits();
45488
45489 // Input type must be extending a bool vector (bit-casted from a scalar
45490 // integer) to legal integer types.
45491 if (!VT.isVector())
45492 return SDValue();
45493 if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16 && SVT != MVT::i8)
45494 return SDValue();
45495 if (InSVT != MVT::i1 || N0.getOpcode() != ISD::BITCAST)
45496 return SDValue();
45497
45498 SDValue N00 = N0.getOperand(0);
45499 EVT SclVT = N00.getValueType();
45500 if (!SclVT.isScalarInteger())
45501 return SDValue();
45502
45503 SDValue Vec;
45504 SmallVector<int> ShuffleMask;
45505 unsigned NumElts = VT.getVectorNumElements();
45506 assert(NumElts == SclVT.getSizeInBits() && "Unexpected bool vector size");
45507
45508 // Broadcast the scalar integer to the vector elements.
45509 if (NumElts > EltSizeInBits) {
45510 // If the scalar integer is greater than the vector element size, then we
45511 // must split it down into sub-sections for broadcasting. For example:
45512 // i16 -> v16i8 (i16 -> v8i16 -> v16i8) with 2 sub-sections.
45513 // i32 -> v32i8 (i32 -> v8i32 -> v32i8) with 4 sub-sections.
45514 assert((NumElts % EltSizeInBits) == 0 && "Unexpected integer scale");
45515 unsigned Scale = NumElts / EltSizeInBits;
45516 EVT BroadcastVT = EVT::getVectorVT(*DAG.getContext(), SclVT, EltSizeInBits);
45517 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
45518 Vec = DAG.getBitcast(VT, Vec);
45519
45520 for (unsigned i = 0; i != Scale; ++i)
45521 ShuffleMask.append(EltSizeInBits, i);
45522 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
45523 } else if (Subtarget.hasAVX2() && NumElts < EltSizeInBits &&
45524 (SclVT == MVT::i8 || SclVT == MVT::i16 || SclVT == MVT::i32)) {
45525 // If we have register broadcast instructions, use the scalar size as the
45526 // element type for the shuffle. Then cast to the wider element type. The
45527 // widened bits won't be used, and this might allow the use of a broadcast
45528 // load.
45529 assert((EltSizeInBits % NumElts) == 0 && "Unexpected integer scale");
45530 unsigned Scale = EltSizeInBits / NumElts;
45531 EVT BroadcastVT =
45532 EVT::getVectorVT(*DAG.getContext(), SclVT, NumElts * Scale);
45533 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
45534 ShuffleMask.append(NumElts * Scale, 0);
45535 Vec = DAG.getVectorShuffle(BroadcastVT, DL, Vec, Vec, ShuffleMask);
45536 Vec = DAG.getBitcast(VT, Vec);
45537 } else {
45538 // For smaller scalar integers, we can simply any-extend it to the vector
45539 // element size (we don't care about the upper bits) and broadcast it to all
45540 // elements.
45541 SDValue Scl = DAG.getAnyExtOrTrunc(N00, DL, SVT);
45542 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
45543 ShuffleMask.append(NumElts, 0);
45544 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
45545 }
45546
45547 // Now, mask the relevant bit in each element.
45548 SmallVector<SDValue, 32> Bits;
45549 for (unsigned i = 0; i != NumElts; ++i) {
45550 int BitIdx = (i % EltSizeInBits);
45551 APInt Bit = APInt::getBitsSet(EltSizeInBits, BitIdx, BitIdx + 1);
45552 Bits.push_back(DAG.getConstant(Bit, DL, SVT));
45553 }
45554 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);
45555 Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask);
45556
45557 // Compare against the bitmask and extend the result.
45558 EVT CCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
45559 Vec = DAG.getSetCC(DL, CCVT, Vec, BitMask, ISD::SETEQ);
45560 Vec = DAG.getSExtOrTrunc(Vec, DL, VT);
45561
45562 // For SEXT, this is now done, otherwise shift the result down for
45563 // zero-extension.
45564 if (Opcode == ISD::SIGN_EXTEND)
45565 return Vec;
45566 return DAG.getNode(ISD::SRL, DL, VT, Vec,
45567 DAG.getConstant(EltSizeInBits - 1, DL, VT));
45568}
45569
45570/// If a vector select has an operand that is -1 or 0, try to simplify the
45571/// select to a bitwise logic operation.
45572/// TODO: Move to DAGCombiner, possibly using TargetLowering::hasAndNot()?
45573static SDValue
45574combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
45575 TargetLowering::DAGCombinerInfo &DCI,
45576 const X86Subtarget &Subtarget) {
45577 SDValue Cond = N->getOperand(0);
45578 SDValue LHS = N->getOperand(1);
45579 SDValue RHS = N->getOperand(2);
45580 EVT VT = LHS.getValueType();
45581 EVT CondVT = Cond.getValueType();
45582 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45583
45584 if (N->getOpcode() != ISD::VSELECT)
45585 return SDValue();
45586
45587 assert(CondVT.isVector() && "Vector select expects a vector selector!");
45588
45589 // TODO: Use isNullOrNullSplat() to distinguish constants with undefs?
45590 // TODO: Can we assert that both operands are not zeros (because that should
45591 // get simplified at node creation time)?
45592 bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
45593 bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
45594
45595 // If both inputs are 0/undef, create a complete zero vector.
45596 // FIXME: As noted above this should be handled by DAGCombiner/getNode.
45597 if (TValIsAllZeros && FValIsAllZeros) {
45598 if (VT.isFloatingPoint())
45599 return DAG.getConstantFP(0.0, DL, VT);
45600 return DAG.getConstant(0, DL, VT);
45601 }
45602
45603 // To use the condition operand as a bitwise mask, it must have elements that
45604 // are the same size as the select elements. Ie, the condition operand must
45605 // have already been promoted from the IR select condition type <N x i1>.
45606 // Don't check if the types themselves are equal because that excludes
45607 // vector floating-point selects.
45608 if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
45609 return SDValue();
45610
45611 // Try to invert the condition if true value is not all 1s and false value is
45612 // not all 0s. Only do this if the condition has one use.
45613 bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode());
45614 if (!TValIsAllOnes && !FValIsAllZeros && Cond.hasOneUse() &&
45615 // Check if the selector will be produced by CMPP*/PCMP*.
45616 Cond.getOpcode() == ISD::SETCC &&
45617 // Check if SETCC has already been promoted.
45618 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
45619 CondVT) {
45620 bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
45621
45622 if (TValIsAllZeros || FValIsAllOnes) {
45623 SDValue CC = Cond.getOperand(2);
45624 ISD::CondCode NewCC = ISD::getSetCCInverse(
45625 cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
45626 Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
45627 NewCC);
45628 std::swap(LHS, RHS);
45629 TValIsAllOnes = FValIsAllOnes;
45630 FValIsAllZeros = TValIsAllZeros;
45631 }
45632 }
45633
45634 // Cond value must be 'sign splat' to be converted to a logical op.
45635 if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
45636 return SDValue();
45637
45638 // vselect Cond, 111..., 000... -> Cond
45639 if (TValIsAllOnes && FValIsAllZeros)
45640 return DAG.getBitcast(VT, Cond);
45641
45642 if (!TLI.isTypeLegal(CondVT))
45643 return SDValue();
45644
45645 // vselect Cond, 111..., X -> or Cond, X
45646 if (TValIsAllOnes) {
45647 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
45648 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
45649 return DAG.getBitcast(VT, Or);
45650 }
45651
45652 // vselect Cond, X, 000... -> and Cond, X
45653 if (FValIsAllZeros) {
45654 SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
45655 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
45656 return DAG.getBitcast(VT, And);
45657 }
45658
45659 // vselect Cond, 000..., X -> andn Cond, X
45660 if (TValIsAllZeros) {
45661 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
45662 SDValue AndN;
45663 // The canonical form differs for i1 vectors - x86andnp is not used
45664 if (CondVT.getScalarType() == MVT::i1)
45665 AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT),
45666 CastRHS);
45667 else
45668 AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS);
45669 return DAG.getBitcast(VT, AndN);
45670 }
45671
45672 return SDValue();
45673}
45674
45675/// If both arms of a vector select are concatenated vectors, split the select,
45676/// and concatenate the result to eliminate a wide (256-bit) vector instruction:
45677/// vselect Cond, (concat T0, T1), (concat F0, F1) -->
45678/// concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)
45679static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
45680 const X86Subtarget &Subtarget) {
45681 unsigned Opcode = N->getOpcode();
45682 if (Opcode != X86ISD::BLENDV && Opcode != ISD::VSELECT)
45683 return SDValue();
45684
45685 // TODO: Split 512-bit vectors too?
45686 EVT VT = N->getValueType(0);
45687 if (!VT.is256BitVector())
45688 return SDValue();
45689
45690 // TODO: Split as long as any 2 of the 3 operands are concatenated?
45691 SDValue Cond = N->getOperand(0);
45692 SDValue TVal = N->getOperand(1);
45693 SDValue FVal = N->getOperand(2);
45694 if (!TVal.hasOneUse() || !FVal.hasOneUse() ||
45695 !isFreeToSplitVector(TVal.getNode(), DAG) ||
45696 !isFreeToSplitVector(FVal.getNode(), DAG))
45697 return SDValue();
45698
45699 auto makeBlend = [Opcode](SelectionDAG &DAG, const SDLoc &DL,
45700 ArrayRef<SDValue> Ops) {
45701 return DAG.getNode(Opcode, DL, Ops[1].getValueType(), Ops);
45702 };
45703 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {Cond, TVal, FVal}, makeBlend,
45704 /*CheckBWI*/ false);
45705}
45706
45707static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG,
45708 const SDLoc &DL) {
45709 SDValue Cond = N->getOperand(0);
45710 SDValue LHS = N->getOperand(1);
45711 SDValue RHS = N->getOperand(2);
45712
45713 auto *TrueC = dyn_cast<ConstantSDNode>(LHS);
45714 auto *FalseC = dyn_cast<ConstantSDNode>(RHS);
45715 if (!TrueC || !FalseC)
45716 return SDValue();
45717
45718 // Don't do this for crazy integer types.
45719 EVT VT = N->getValueType(0);
45720 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
45721 return SDValue();
45722
45723 // We're going to use the condition bit in math or logic ops. We could allow
45724 // this with a wider condition value (post-legalization it becomes an i8),
45725 // but if nothing is creating selects that late, it doesn't matter.
45726 if (Cond.getValueType() != MVT::i1)
45727 return SDValue();
45728
45729 // A power-of-2 multiply is just a shift. LEA also cheaply handles multiply by
45730 // 3, 5, or 9 with i32/i64, so those get transformed too.
45731 // TODO: For constants that overflow or do not differ by power-of-2 or small
45732 // multiplier, convert to 'and' + 'add'.
45733 const APInt &TrueVal = TrueC->getAPIntValue();
45734 const APInt &FalseVal = FalseC->getAPIntValue();
45735
45736 // We have a more efficient lowering for "(X == 0) ? Y : -1" using SBB.
45737 if ((TrueVal.isAllOnes() || FalseVal.isAllOnes()) &&
45738 Cond.getOpcode() == ISD::SETCC && isNullConstant(Cond.getOperand(1))) {
45739 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
45740 if (CC == ISD::SETEQ || CC == ISD::SETNE)
45741 return SDValue();
45742 }
45743
45744 bool OV;
45745 APInt Diff = TrueVal.ssub_ov(FalseVal, OV);
45746 if (OV)
45747 return SDValue();
45748
45749 APInt AbsDiff = Diff.abs();
45750 if (AbsDiff.isPowerOf2() ||
45751 ((VT == MVT::i32 || VT == MVT::i64) &&
45752 (AbsDiff == 3 || AbsDiff == 5 || AbsDiff == 9))) {
45753
45754 // We need a positive multiplier constant for shift/LEA codegen. The 'not'
45755 // of the condition can usually be folded into a compare predicate, but even
45756 // without that, the sequence should be cheaper than a CMOV alternative.
45757 if (TrueVal.slt(FalseVal)) {
45758 Cond = DAG.getNOT(DL, Cond, MVT::i1);
45759 std::swap(TrueC, FalseC);
45760 }
45761
45762 // select Cond, TC, FC --> (zext(Cond) * (TC - FC)) + FC
45763 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
45764
45765 // Multiply condition by the difference if non-one.
45766 if (!AbsDiff.isOne())
45767 R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT));
45768
45769 // Add the base if non-zero.
45770 if (!FalseC->isZero())
45771 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
45772
45773 return R;
45774 }
45775
45776 return SDValue();
45777}
45778
45779/// If this is a *dynamic* select (non-constant condition) and we can match
45780/// this node with one of the variable blend instructions, restructure the
45781/// condition so that blends can use the high (sign) bit of each element.
45782/// This function will also call SimplifyDemandedBits on already created
45783/// BLENDV to perform additional simplifications.
45784static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
45785 const SDLoc &DL,
45786 TargetLowering::DAGCombinerInfo &DCI,
45787 const X86Subtarget &Subtarget) {
45788 SDValue Cond = N->getOperand(0);
45789 if ((N->getOpcode() != ISD::VSELECT &&
45790 N->getOpcode() != X86ISD::BLENDV) ||
45791 ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
45792 return SDValue();
45793
45794 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45795 unsigned BitWidth = Cond.getScalarValueSizeInBits();
45796 EVT VT = N->getValueType(0);
45797
45798 // We can only handle the cases where VSELECT is directly legal on the
45799 // subtarget. We custom lower VSELECT nodes with constant conditions and
45800 // this makes it hard to see whether a dynamic VSELECT will correctly
45801 // lower, so we both check the operation's status and explicitly handle the
45802 // cases where a *dynamic* blend will fail even though a constant-condition
45803 // blend could be custom lowered.
45804 // FIXME: We should find a better way to handle this class of problems.
45805 // Potentially, we should combine constant-condition vselect nodes
45806 // pre-legalization into shuffles and not mark as many types as custom
45807 // lowered.
45808 if (!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
45809 return SDValue();
45810 // FIXME: We don't support i16-element blends currently. We could and
45811 // should support them by making *all* the bits in the condition be set
45812 // rather than just the high bit and using an i8-element blend.
45813 if (VT.getVectorElementType() == MVT::i16)
45814 return SDValue();
45815 // Dynamic blending was only available from SSE4.1 onward.
45816 if (VT.is128BitVector() && !Subtarget.hasSSE41())
45817 return SDValue();
45818 // Byte blends are only available in AVX2
45819 if (VT == MVT::v32i8 && !Subtarget.hasAVX2())
45820 return SDValue();
45821 // There are no 512-bit blend instructions that use sign bits.
45822 if (VT.is512BitVector())
45823 return SDValue();
45824
45825 // Don't optimize before the condition has been transformed to a legal type
45826 // and don't ever optimize vector selects that map to AVX512 mask-registers.
45827 if (BitWidth < 8 || BitWidth > 64)
45828 return SDValue();
45829
45830 auto OnlyUsedAsSelectCond = [](SDValue Cond) {
45831 for (SDNode::use_iterator UI = Cond->use_begin(), UE = Cond->use_end();
45832 UI != UE; ++UI)
45833 if ((UI->getOpcode() != ISD::VSELECT &&
45834 UI->getOpcode() != X86ISD::BLENDV) ||
45835 UI.getOperandNo() != 0)
45836 return false;
45837
45838 return true;
45839 };
45840
45841 APInt DemandedBits(APInt::getSignMask(BitWidth));
45842
45843 if (OnlyUsedAsSelectCond(Cond)) {
45844 KnownBits Known;
45845 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
45846 !DCI.isBeforeLegalizeOps());
45847 if (!TLI.SimplifyDemandedBits(Cond, DemandedBits, Known, TLO, 0, true))
45848 return SDValue();
45849
45850 // If we changed the computation somewhere in the DAG, this change will
45851 // affect all users of Cond. Update all the nodes so that we do not use
45852 // the generic VSELECT anymore. Otherwise, we may perform wrong
45853 // optimizations as we messed with the actual expectation for the vector
45854 // boolean values.
45855 for (SDNode *U : Cond->uses()) {
45856 if (U->getOpcode() == X86ISD::BLENDV)
45857 continue;
45858
45859 SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0),
45860 Cond, U->getOperand(1), U->getOperand(2));
45861 DAG.ReplaceAllUsesOfValueWith(SDValue(U, 0), SB);
45862 DCI.AddToWorklist(U);
45863 }
45864 DCI.CommitTargetLoweringOpt(TLO);
45865 return SDValue(N, 0);
45866 }
45867
45868 // Otherwise we can still at least try to simplify multiple use bits.
45869 if (SDValue V = TLI.SimplifyMultipleUseDemandedBits(Cond, DemandedBits, DAG))
45870 return DAG.getNode(X86ISD::BLENDV, DL, N->getValueType(0), V,
45871 N->getOperand(1), N->getOperand(2));
45872
45873 return SDValue();
45874}
45875
45876// Try to match:
45877// (or (and (M, (sub 0, X)), (pandn M, X)))
45878// which is a special case of:
45879// (select M, (sub 0, X), X)
45880// Per:
45881// http://graphics.stanford.edu/~seander/bithacks.html#ConditionalNegate
45882// We know that, if fNegate is 0 or 1:
45883// (fNegate ? -v : v) == ((v ^ -fNegate) + fNegate)
45884//
45885// Here, we have a mask, M (all 1s or 0), and, similarly, we know that:
45886// ((M & 1) ? -X : X) == ((X ^ -(M & 1)) + (M & 1))
45887// ( M ? -X : X) == ((X ^ M ) + (M & 1))
45888// This lets us transform our vselect to:
45889// (add (xor X, M), (and M, 1))
45890// And further to:
45891// (sub (xor X, M), M)
45892static SDValue combineLogicBlendIntoConditionalNegate(
45893 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
45894 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
45895 EVT MaskVT = Mask.getValueType();
45896 assert(MaskVT.isInteger() &&
45897 DAG.ComputeNumSignBits(Mask) == MaskVT.getScalarSizeInBits() &&
45898 "Mask must be zero/all-bits");
45899
45900 if (X.getValueType() != MaskVT || Y.getValueType() != MaskVT)
45901 return SDValue();
45902 if (!DAG.getTargetLoweringInfo().isOperationLegal(ISD::SUB, MaskVT))
45903 return SDValue();
45904
45905 auto IsNegV = [](SDNode *N, SDValue V) {
45906 return N->getOpcode() == ISD::SUB && N->getOperand(1) == V &&
45907 ISD::isBuildVectorAllZeros(N->getOperand(0).getNode());
45908 };
45909
45910 SDValue V;
45911 if (IsNegV(Y.getNode(), X))
45912 V = X;
45913 else if (IsNegV(X.getNode(), Y))
45914 V = Y;
45915 else
45916 return SDValue();
45917
45918 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask);
45919 SDValue SubOp2 = Mask;
45920
45921 // If the negate was on the false side of the select, then
45922 // the operands of the SUB need to be swapped. PR 27251.
45923 // This is because the pattern being matched above is
45924 // (vselect M, (sub (0, X), X) -> (sub (xor X, M), M)
45925 // but if the pattern matched was
45926 // (vselect M, X, (sub (0, X))), that is really negation of the pattern
45927 // above, -(vselect M, (sub 0, X), X), and therefore the replacement
45928 // pattern also needs to be a negation of the replacement pattern above.
45929 // And -(sub X, Y) is just sub (Y, X), so swapping the operands of the
45930 // sub accomplishes the negation of the replacement pattern.
45931 if (V == Y)
45932 std::swap(SubOp1, SubOp2);
45933
45934 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2);
45935 return DAG.getBitcast(VT, Res);
45936}
45937
45938static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
45939 const X86Subtarget &Subtarget) {
45940 if (!Subtarget.hasAVX512())
45941 return SDValue();
45942 if (N->getOpcode() != ISD::VSELECT)
45943 return SDValue();
45944
45945 SDValue Cond = N->getOperand(0);
45946 SDValue LHS = N->getOperand(1);
45947 SDValue RHS = N->getOperand(2);
45948
45949 if (canCombineAsMaskOperation(LHS, Subtarget))
45950 return SDValue();
45951
45952 if (!canCombineAsMaskOperation(RHS, Subtarget))
45953 return SDValue();
45954
45955 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
45956 return SDValue();
45957
45958 // Commute LHS and RHS to create opportunity to select mask instruction.
45959 // (vselect M, L, R) -> (vselect ~M, R, L)
45960 ISD::CondCode NewCC =
45961 ISD::getSetCCInverse(cast<CondCodeSDNode>(Cond.getOperand(2))->get(),
45962 Cond.getOperand(0).getValueType());
45963 Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), Cond.getOperand(0),
45964 Cond.getOperand(1), NewCC);
45965 return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS);
45966}
45967
45968/// Do target-specific dag combines on SELECT and VSELECT nodes.
45969static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
45970 TargetLowering::DAGCombinerInfo &DCI,
45971 const X86Subtarget &Subtarget) {
45972 SDLoc DL(N);
45973 SDValue Cond = N->getOperand(0);
45974 SDValue LHS = N->getOperand(1);
45975 SDValue RHS = N->getOperand(2);
45976
45977 // Try simplification again because we use this function to optimize
45978 // BLENDV nodes that are not handled by the generic combiner.
45979 if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS))
45980 return V;
45981
45982 // When avx512 is available the lhs operand of select instruction can be
45983 // folded with mask instruction, while the rhs operand can't. Commute the
45984 // lhs and rhs of the select instruction to create the opportunity of
45985 // folding.
45986 if (SDValue V = commuteSelect(N, DAG, DL, Subtarget))
45987 return V;
45988
45989 EVT VT = LHS.getValueType();
45990 EVT CondVT = Cond.getValueType();
45991 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45992 bool CondConstantVector = ISD::isBuildVectorOfConstantSDNodes(Cond.getNode());
45993
45994 // Attempt to combine (select M, (sub 0, X), X) -> (sub (xor X, M), M).
45995 // Limit this to cases of non-constant masks that createShuffleMaskFromVSELECT
45996 // can't catch, plus vXi8 cases where we'd likely end up with BLENDV.
45997 if (CondVT.isVector() && CondVT.isInteger() &&
45998 CondVT.getScalarSizeInBits() == VT.getScalarSizeInBits() &&
45999 (!CondConstantVector || CondVT.getScalarType() == MVT::i8) &&
46000 DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits())
46001 if (SDValue V = combineLogicBlendIntoConditionalNegate(VT, Cond, RHS, LHS,
46002 DL, DAG, Subtarget))
46003 return V;
46004
46005 // Convert vselects with constant condition into shuffles.
46006 if (CondConstantVector && DCI.isBeforeLegalizeOps() &&
46007 (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV)) {
46008 SmallVector<int, 64> Mask;
46009 if (createShuffleMaskFromVSELECT(Mask, Cond,
46010 N->getOpcode() == X86ISD::BLENDV))
46011 return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);
46012 }
46013
46014 // fold vselect(cond, pshufb(x), pshufb(y)) -> or (pshufb(x), pshufb(y))
46015 // by forcing the unselected elements to zero.
46016 // TODO: Can we handle more shuffles with this?
46017 if (N->getOpcode() == ISD::VSELECT && CondVT.isVector() &&
46018 LHS.getOpcode() == X86ISD::PSHUFB && RHS.getOpcode() == X86ISD::PSHUFB &&
46019 LHS.hasOneUse() && RHS.hasOneUse()) {
46020 MVT SimpleVT = VT.getSimpleVT();
46021 SmallVector<SDValue, 1> LHSOps, RHSOps;
46022 SmallVector<int, 64> LHSMask, RHSMask, CondMask;
46023 if (createShuffleMaskFromVSELECT(CondMask, Cond) &&
46024 getTargetShuffleMask(LHS, true, LHSOps, LHSMask) &&
46025 getTargetShuffleMask(RHS, true, RHSOps, RHSMask)) {
46026 int NumElts = VT.getVectorNumElements();
46027 for (int i = 0; i != NumElts; ++i) {
46028 // getConstVector sets negative shuffle mask values as undef, so ensure
46029 // we hardcode SM_SentinelZero values to zero (0x80).
46030 if (CondMask[i] < NumElts) {
46031 LHSMask[i] = isUndefOrZero(LHSMask[i]) ? 0x80 : LHSMask[i];
46032 RHSMask[i] = 0x80;
46033 } else {
46034 LHSMask[i] = 0x80;
46035 RHSMask[i] = isUndefOrZero(RHSMask[i]) ? 0x80 : RHSMask[i];
46036 }
46037 }
46038 LHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, LHS.getOperand(0),
46039 getConstVector(LHSMask, SimpleVT, DAG, DL, true));
46040 RHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, RHS.getOperand(0),
46041 getConstVector(RHSMask, SimpleVT, DAG, DL, true));
46042 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
46043 }
46044 }
46045
46046 // If we have SSE[12] support, try to form min/max nodes. SSE min/max
46047 // instructions match the semantics of the common C idiom x<y?x:y but not
46048 // x<=y?x:y, because of how they handle negative zero (which can be
46049 // ignored in unsafe-math mode).
46050 // We also try to create v2f32 min/max nodes, which we later widen to v4f32.
46051 if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&
46052 VT != MVT::f80 && VT != MVT::f128 && !isSoftF16(VT, Subtarget) &&
46053 (TLI.isTypeLegal(VT) || VT == MVT::v2f32) &&
46054 (Subtarget.hasSSE2() ||
46055 (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) {
46056 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
46057
46058 unsigned Opcode = 0;
46059 // Check for x CC y ? x : y.
46060 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
46061 DAG.isEqualTo(RHS, Cond.getOperand(1))) {
46062 switch (CC) {
46063 default: break;
46064 case ISD::SETULT:
46065 // Converting this to a min would handle NaNs incorrectly, and swapping
46066 // the operands would cause it to handle comparisons between positive
46067 // and negative zero incorrectly.
46068 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
46069 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
46070 !(DAG.isKnownNeverZeroFloat(LHS) ||
46071 DAG.isKnownNeverZeroFloat(RHS)))
46072 break;
46073 std::swap(LHS, RHS);
46074 }
46075 Opcode = X86ISD::FMIN;
46076 break;
46077 case ISD::SETOLE:
46078 // Converting this to a min would handle comparisons between positive
46079 // and negative zero incorrectly.
46080 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
46081 !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
46082 break;
46083 Opcode = X86ISD::FMIN;
46084 break;
46085 case ISD::SETULE:
46086 // Converting this to a min would handle both negative zeros and NaNs
46087 // incorrectly, but we can swap the operands to fix both.
46088 std::swap(LHS, RHS);
46089 [[fallthrough]];
46090 case ISD::SETOLT:
46091 case ISD::SETLT:
46092 case ISD::SETLE:
46093 Opcode = X86ISD::FMIN;
46094 break;
46095
46096 case ISD::SETOGE:
46097 // Converting this to a max would handle comparisons between positive
46098 // and negative zero incorrectly.
46099 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
46100 !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
46101 break;
46102 Opcode = X86ISD::FMAX;
46103 break;
46104 case ISD::SETUGT:
46105 // Converting this to a max would handle NaNs incorrectly, and swapping
46106 // the operands would cause it to handle comparisons between positive
46107 // and negative zero incorrectly.
46108 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
46109 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
46110 !(DAG.isKnownNeverZeroFloat(LHS) ||
46111 DAG.isKnownNeverZeroFloat(RHS)))
46112 break;
46113 std::swap(LHS, RHS);
46114 }
46115 Opcode = X86ISD::FMAX;
46116 break;
46117 case ISD::SETUGE:
46118 // Converting this to a max would handle both negative zeros and NaNs
46119 // incorrectly, but we can swap the operands to fix both.
46120 std::swap(LHS, RHS);
46121 [[fallthrough]];
46122 case ISD::SETOGT:
46123 case ISD::SETGT:
46124 case ISD::SETGE:
46125 Opcode = X86ISD::FMAX;
46126 break;
46127 }
46128 // Check for x CC y ? y : x -- a min/max with reversed arms.
46129 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
46130 DAG.isEqualTo(RHS, Cond.getOperand(0))) {
46131 switch (CC) {
46132 default: break;
46133 case ISD::SETOGE:
46134 // Converting this to a min would handle comparisons between positive
46135 // and negative zero incorrectly, and swapping the operands would
46136 // cause it to handle NaNs incorrectly.
46137 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
46138 !(DAG.isKnownNeverZeroFloat(LHS) ||
46139 DAG.isKnownNeverZeroFloat(RHS))) {
46140 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
46141 break;
46142 std::swap(LHS, RHS);
46143 }
46144 Opcode = X86ISD::FMIN;
46145 break;
46146 case ISD::SETUGT:
46147 // Converting this to a min would handle NaNs incorrectly.
46148 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
46149 break;
46150 Opcode = X86ISD::FMIN;
46151 break;
46152 case ISD::SETUGE:
46153 // Converting this to a min would handle both negative zeros and NaNs
46154 // incorrectly, but we can swap the operands to fix both.
46155 std::swap(LHS, RHS);
46156 [[fallthrough]];
46157 case ISD::SETOGT:
46158 case ISD::SETGT:
46159 case ISD::SETGE:
46160 Opcode = X86ISD::FMIN;
46161 break;
46162
46163 case ISD::SETULT:
46164 // Converting this to a max would handle NaNs incorrectly.
46165 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
46166 break;
46167 Opcode = X86ISD::FMAX;
46168 break;
46169 case ISD::SETOLE:
46170 // Converting this to a max would handle comparisons between positive
46171 // and negative zero incorrectly, and swapping the operands would
46172 // cause it to handle NaNs incorrectly.
46173 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
46174 !DAG.isKnownNeverZeroFloat(LHS) &&
46175 !DAG.isKnownNeverZeroFloat(RHS)) {
46176 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
46177 break;
46178 std::swap(LHS, RHS);
46179 }
46180 Opcode = X86ISD::FMAX;
46181 break;
46182 case ISD::SETULE:
46183 // Converting this to a max would handle both negative zeros and NaNs
46184 // incorrectly, but we can swap the operands to fix both.
46185 std::swap(LHS, RHS);
46186 [[fallthrough]];
46187 case ISD::SETOLT:
46188 case ISD::SETLT:
46189 case ISD::SETLE:
46190 Opcode = X86ISD::FMAX;
46191 break;
46192 }
46193 }
46194
46195 if (Opcode)
46196 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
46197 }
46198
46199 // Some mask scalar intrinsics rely on checking if only one bit is set
46200 // and implement it in C code like this:
46201 // A[0] = (U & 1) ? A[0] : W[0];
46202 // This creates some redundant instructions that break pattern matching.
46203 // fold (select (setcc (and (X, 1), 0, seteq), Y, Z)) -> select(and(X, 1),Z,Y)
46204 if (Subtarget.hasAVX512() && N->getOpcode() == ISD::SELECT &&
46205 Cond.getOpcode() == ISD::SETCC && (VT == MVT::f32 || VT == MVT::f64)) {
46206 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
46207 SDValue AndNode = Cond.getOperand(0);
46208 if (AndNode.getOpcode() == ISD::AND && CC == ISD::SETEQ &&
46209 isNullConstant(Cond.getOperand(1)) &&
46210 isOneConstant(AndNode.getOperand(1))) {
46211 // LHS and RHS swapped due to
46212 // setcc outputting 1 when AND resulted in 0 and vice versa.
46213 AndNode = DAG.getZExtOrTrunc(AndNode, DL, MVT::i8);
46214 return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS);
46215 }
46216 }
46217
46218 // v16i8 (select v16i1, v16i8, v16i8) does not have a proper
46219 // lowering on KNL. In this case we convert it to
46220 // v16i8 (select v16i8, v16i8, v16i8) and use AVX instruction.
46221 // The same situation all vectors of i8 and i16 without BWI.
46222 // Make sure we extend these even before type legalization gets a chance to
46223 // split wide vectors.
46224 // Since SKX these selects have a proper lowering.
46225 if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && CondVT.isVector() &&
46226 CondVT.getVectorElementType() == MVT::i1 &&
46227 (VT.getVectorElementType() == MVT::i8 ||
46228 VT.getVectorElementType() == MVT::i16)) {
46229 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
46230 return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS);
46231 }
46232
46233 // AVX512 - Extend select with zero to merge with target shuffle.
46234 // select(mask, extract_subvector(shuffle(x)), zero) -->
46235 // extract_subvector(select(insert_subvector(mask), shuffle(x), zero))
46236 // TODO - support non target shuffles as well.
46237 if (Subtarget.hasAVX512() && CondVT.isVector() &&
46238 CondVT.getVectorElementType() == MVT::i1) {
46239 auto SelectableOp = [&TLI](SDValue Op) {
46240 return Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
46241 isTargetShuffle(Op.getOperand(0).getOpcode()) &&
46242 isNullConstant(Op.getOperand(1)) &&
46243 TLI.isTypeLegal(Op.getOperand(0).getValueType()) &&
46244 Op.hasOneUse() && Op.getOperand(0).hasOneUse();
46245 };
46246
46247 bool SelectableLHS = SelectableOp(LHS);
46248 bool SelectableRHS = SelectableOp(RHS);
46249 bool ZeroLHS = ISD::isBuildVectorAllZeros(LHS.getNode());
46250 bool ZeroRHS = ISD::isBuildVectorAllZeros(RHS.getNode());
46251
46252 if ((SelectableLHS && ZeroRHS) || (SelectableRHS && ZeroLHS)) {
46253 EVT SrcVT = SelectableLHS ? LHS.getOperand(0).getValueType()
46254 : RHS.getOperand(0).getValueType();
46255 EVT SrcCondVT = SrcVT.changeVectorElementType(MVT::i1);
46256 LHS = insertSubVector(DAG.getUNDEF(SrcVT), LHS, 0, DAG, DL,
46257 VT.getSizeInBits());
46258 RHS = insertSubVector(DAG.getUNDEF(SrcVT), RHS, 0, DAG, DL,
46259 VT.getSizeInBits());
46260 Cond = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcCondVT,
46261 DAG.getUNDEF(SrcCondVT), Cond,
46262 DAG.getIntPtrConstant(0, DL));
46263 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS);
46264 return extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
46265 }
46266 }
46267
46268 if (SDValue V = combineSelectOfTwoConstants(N, DAG, DL))
46269 return V;
46270
46271 if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC &&
46272 Cond.hasOneUse()) {
46273 EVT CondVT = Cond.getValueType();
46274 SDValue Cond0 = Cond.getOperand(0);
46275 SDValue Cond1 = Cond.getOperand(1);
46276 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
46277
46278 // Canonicalize min/max:
46279 // (x > 0) ? x : 0 -> (x >= 0) ? x : 0
46280 // (x < -1) ? x : -1 -> (x <= -1) ? x : -1
46281 // This allows use of COND_S / COND_NS (see TranslateX86CC) which eliminates
46282 // the need for an extra compare against zero. e.g.
46283 // (a - b) > 0 : (a - b) ? 0 -> (a - b) >= 0 : (a - b) ? 0
46284 // subl %esi, %edi
46285 // testl %edi, %edi
46286 // movl $0, %eax
46287 // cmovgl %edi, %eax
46288 // =>
46289 // xorl %eax, %eax
46290 // subl %esi, $edi
46291 // cmovsl %eax, %edi
46292 //
46293 // We can also canonicalize
46294 // (x s> 1) ? x : 1 -> (x s>= 1) ? x : 1 -> (x s> 0) ? x : 1
46295 // (x u> 1) ? x : 1 -> (x u>= 1) ? x : 1 -> (x != 0) ? x : 1
46296 // This allows the use of a test instruction for the compare.
46297 if (LHS == Cond0 && RHS == Cond1) {
46298 if ((CC == ISD::SETGT && (isNullConstant(RHS) || isOneConstant(RHS))) ||
46299 (CC == ISD::SETLT && isAllOnesConstant(RHS))) {
46300 ISD::CondCode NewCC = CC == ISD::SETGT ? ISD::SETGE : ISD::SETLE;
46301 Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC);
46302 return DAG.getSelect(DL, VT, Cond, LHS, RHS);
46303 }
46304 if (CC == ISD::SETUGT && isOneConstant(RHS)) {
46305 ISD::CondCode NewCC = ISD::SETUGE;
46306 Cond = DAG.getSetCC(SDLoc(Cond), CondVT, Cond0, Cond1, NewCC);
46307 return DAG.getSelect(DL, VT, Cond, LHS, RHS);
46308 }
46309 }
46310
46311 // Similar to DAGCombine's select(or(CC0,CC1),X,Y) fold but for legal types.
46312 // fold eq + gt/lt nested selects into ge/le selects
46313 // select (cmpeq Cond0, Cond1), LHS, (select (cmpugt Cond0, Cond1), LHS, Y)
46314 // --> (select (cmpuge Cond0, Cond1), LHS, Y)
46315 // select (cmpslt Cond0, Cond1), LHS, (select (cmpeq Cond0, Cond1), LHS, Y)
46316 // --> (select (cmpsle Cond0, Cond1), LHS, Y)
46317 // .. etc ..
46318 if (RHS.getOpcode() == ISD::SELECT && RHS.getOperand(1) == LHS &&
46319 RHS.getOperand(0).getOpcode() == ISD::SETCC) {
46320 SDValue InnerSetCC = RHS.getOperand(0);
46321 ISD::CondCode InnerCC =
46322 cast<CondCodeSDNode>(InnerSetCC.getOperand(2))->get();
46323 if ((CC == ISD::SETEQ || InnerCC == ISD::SETEQ) &&
46324 Cond0 == InnerSetCC.getOperand(0) &&
46325 Cond1 == InnerSetCC.getOperand(1)) {
46326 ISD::CondCode NewCC;
46327 switch (CC == ISD::SETEQ ? InnerCC : CC) {
46328 // clang-format off
46329 case ISD::SETGT: NewCC = ISD::SETGE; break;
46330 case ISD::SETLT: NewCC = ISD::SETLE; break;
46331 case ISD::SETUGT: NewCC = ISD::SETUGE; break;
46332 case ISD::SETULT: NewCC = ISD::SETULE; break;
46333 default: NewCC = ISD::SETCC_INVALID; break;
46334 // clang-format on
46335 }
46336 if (NewCC != ISD::SETCC_INVALID) {
46337 Cond = DAG.getSetCC(DL, CondVT, Cond0, Cond1, NewCC);
46338 return DAG.getSelect(DL, VT, Cond, LHS, RHS.getOperand(2));
46339 }
46340 }
46341 }
46342 }
46343
46344 // Check if the first operand is all zeros and Cond type is vXi1.
46345 // If this an avx512 target we can improve the use of zero masking by
46346 // swapping the operands and inverting the condition.
46347 if (N->getOpcode() == ISD::VSELECT && Cond.hasOneUse() &&
46348 Subtarget.hasAVX512() && CondVT.getVectorElementType() == MVT::i1 &&
46349 ISD::isBuildVectorAllZeros(LHS.getNode()) &&
46350 !ISD::isBuildVectorAllZeros(RHS.getNode())) {
46351 // Invert the cond to not(cond) : xor(op,allones)=not(op)
46352 SDValue CondNew = DAG.getNOT(DL, Cond, CondVT);
46353 // Vselect cond, op1, op2 = Vselect not(cond), op2, op1
46354 return DAG.getSelect(DL, VT, CondNew, RHS, LHS);
46355 }
46356
46357 // Attempt to convert a (vXi1 bitcast(iX Cond)) selection mask before it might
46358 // get split by legalization.
46359 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::BITCAST &&
46360 CondVT.getVectorElementType() == MVT::i1 &&
46361 TLI.isTypeLegal(VT.getScalarType())) {
46362 EVT ExtCondVT = VT.changeVectorElementTypeToInteger();
46363 if (SDValue ExtCond = combineToExtendBoolVectorInReg(
46364 ISD::SIGN_EXTEND, DL, ExtCondVT, Cond, DAG, DCI, Subtarget)) {
46365 ExtCond = DAG.getNode(ISD::TRUNCATE, DL, CondVT, ExtCond);
46366 return DAG.getSelect(DL, VT, ExtCond, LHS, RHS);
46367 }
46368 }
46369
46370 // Exploits AVX2 VSHLV/VSRLV instructions for efficient unsigned vector shifts
46371 // with out-of-bounds clamping.
46372
46373 // Unlike general shift instructions (SHL/SRL), AVX2's VSHLV/VSRLV handle
46374 // shift amounts exceeding the element bitwidth. VSHLV/VSRLV clamps the amount
46375 // to bitwidth-1 for unsigned shifts, effectively performing a maximum left
46376 // shift of bitwidth-1 positions. and returns zero for unsigned right shifts
46377 // exceeding bitwidth-1.
46378 if (N->getOpcode() == ISD::VSELECT) {
46379 using namespace llvm::SDPatternMatch;
46380 // fold select(icmp_ult(amt,BW),shl(x,amt),0) -> avx2 psllv(x,amt)
46381 // fold select(icmp_ult(amt,BW),srl(x,amt),0) -> avx2 psrlv(x,amt)
46382 if ((LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SHL) &&
46383 supportedVectorVarShift(VT, Subtarget, LHS.getOpcode()) &&
46384 ISD::isConstantSplatVectorAllZeros(RHS.getNode()) &&
46385 sd_match(Cond, m_SetCC(m_Specific(LHS.getOperand(1)),
46386 m_SpecificInt(VT.getScalarSizeInBits()),
46387 m_SpecificCondCode(ISD::SETULT)))) {
46388 return DAG.getNode(LHS.getOpcode() == ISD::SRL ? X86ISD::VSRLV
46389 : X86ISD::VSHLV,
46390 DL, VT, LHS.getOperand(0), LHS.getOperand(1));
46391 }
46392 // fold select(icmp_uge(amt,BW),0,shl(x,amt)) -> avx2 psllv(x,amt)
46393 // fold select(icmp_uge(amt,BW),0,srl(x,amt)) -> avx2 psrlv(x,amt)
46394 if ((RHS.getOpcode() == ISD::SRL || RHS.getOpcode() == ISD::SHL) &&
46395 supportedVectorVarShift(VT, Subtarget, RHS.getOpcode()) &&
46396 ISD::isConstantSplatVectorAllZeros(LHS.getNode()) &&
46397 sd_match(Cond, m_SetCC(m_Specific(RHS.getOperand(1)),
46398 m_SpecificInt(VT.getScalarSizeInBits()),
46399 m_SpecificCondCode(ISD::SETUGE)))) {
46400 return DAG.getNode(RHS.getOpcode() == ISD::SRL ? X86ISD::VSRLV
46401 : X86ISD::VSHLV,
46402 DL, VT, RHS.getOperand(0), RHS.getOperand(1));
46403 }
46404 }
46405
46406 // Early exit check
46407 if (!TLI.isTypeLegal(VT) || isSoftF16(VT, Subtarget))
46408 return SDValue();
46409
46410 if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DL, DCI, Subtarget))
46411 return V;
46412
46413 if (SDValue V = combineVSelectToBLENDV(N, DAG, DL, DCI, Subtarget))
46414 return V;
46415
46416 if (SDValue V = narrowVectorSelect(N, DAG, DL, Subtarget))
46417 return V;
46418
46419 // select(~Cond, X, Y) -> select(Cond, Y, X)
46420 if (CondVT.getScalarType() != MVT::i1) {
46421 if (SDValue CondNot = IsNOT(Cond, DAG))
46422 return DAG.getNode(N->getOpcode(), DL, VT,
46423 DAG.getBitcast(CondVT, CondNot), RHS, LHS);
46424
46425 // pcmpgt(X, -1) -> pcmpgt(0, X) to help select/blendv just use the
46426 // signbit.
46427 if (Cond.getOpcode() == X86ISD::PCMPGT &&
46428 ISD::isBuildVectorAllOnes(Cond.getOperand(1).getNode()) &&
46429 Cond.hasOneUse()) {
46430 Cond = DAG.getNode(X86ISD::PCMPGT, DL, CondVT,
46431 DAG.getConstant(0, DL, CondVT), Cond.getOperand(0));
46432 return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS);
46433 }
46434 }
46435
46436 // Try to optimize vXi1 selects if both operands are either all constants or
46437 // bitcasts from scalar integer type. In that case we can convert the operands
46438 // to integer and use an integer select which will be converted to a CMOV.
46439 // We need to take a little bit of care to avoid creating an i64 type after
46440 // type legalization.
46441 if (N->getOpcode() == ISD::SELECT && VT.isVector() &&
46442 VT.getVectorElementType() == MVT::i1 &&
46443 (DCI.isBeforeLegalize() || (VT != MVT::v64i1 || Subtarget.is64Bit()))) {
46444 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());
46445 if (DCI.isBeforeLegalize() || TLI.isTypeLegal(IntVT)) {
46446 bool LHSIsConst = ISD::isBuildVectorOfConstantSDNodes(LHS.getNode());
46447 bool RHSIsConst = ISD::isBuildVectorOfConstantSDNodes(RHS.getNode());
46448
46449 if ((LHSIsConst || (LHS.getOpcode() == ISD::BITCAST &&
46450 LHS.getOperand(0).getValueType() == IntVT)) &&
46451 (RHSIsConst || (RHS.getOpcode() == ISD::BITCAST &&
46452 RHS.getOperand(0).getValueType() == IntVT))) {
46453 if (LHSIsConst)
46454 LHS = combinevXi1ConstantToInteger(LHS, DAG);
46455 else
46456 LHS = LHS.getOperand(0);
46457
46458 if (RHSIsConst)
46459 RHS = combinevXi1ConstantToInteger(RHS, DAG);
46460 else
46461 RHS = RHS.getOperand(0);
46462
46463 SDValue Select = DAG.getSelect(DL, IntVT, Cond, LHS, RHS);
46464 return DAG.getBitcast(VT, Select);
46465 }
46466 }
46467 }
46468
46469 // If this is "((X & C) == 0) ? Y : Z" and C is a constant mask vector of
46470 // single bits, then invert the predicate and swap the select operands.
46471 // This can lower using a vector shift bit-hack rather than mask and compare.
46472 if (DCI.isBeforeLegalize() && !Subtarget.hasAVX512() &&
46473 N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
46474 Cond.hasOneUse() && CondVT.getVectorElementType() == MVT::i1 &&
46475 Cond.getOperand(0).getOpcode() == ISD::AND &&
46476 isNullOrNullSplat(Cond.getOperand(1)) &&
46477 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ &&
46478 Cond.getOperand(0).getValueType() == VT) {
46479 // The 'and' mask must be composed of power-of-2 constants.
46480 SDValue And = Cond.getOperand(0);
46481 auto *C = isConstOrConstSplat(And.getOperand(1));
46482 if (C && C->getAPIntValue().isPowerOf2()) {
46483 // vselect (X & C == 0), LHS, RHS --> vselect (X & C != 0), RHS, LHS
46484 SDValue NotCond =
46485 DAG.getSetCC(DL, CondVT, And, Cond.getOperand(1), ISD::SETNE);
46486 return DAG.getSelect(DL, VT, NotCond, RHS, LHS);
46487 }
46488
46489 // If we have a non-splat but still powers-of-2 mask, AVX1 can use pmulld
46490 // and AVX2 can use vpsllv{dq}. 8-bit lacks a proper shift or multiply.
46491 // 16-bit lacks a proper blendv.
46492 unsigned EltBitWidth = VT.getScalarSizeInBits();
46493 bool CanShiftBlend =
46494 TLI.isTypeLegal(VT) && ((Subtarget.hasAVX() && EltBitWidth == 32) ||
46495 (Subtarget.hasAVX2() && EltBitWidth == 64) ||
46496 (Subtarget.hasXOP()));
46497 if (CanShiftBlend &&
46498 ISD::matchUnaryPredicate(And.getOperand(1), [](ConstantSDNode *C) {
46499 return C->getAPIntValue().isPowerOf2();
46500 })) {
46501 // Create a left-shift constant to get the mask bits over to the sign-bit.
46502 SDValue Mask = And.getOperand(1);
46503 SmallVector<int, 32> ShlVals;
46504 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
46505 auto *MaskVal = cast<ConstantSDNode>(Mask.getOperand(i));
46506 ShlVals.push_back(EltBitWidth - 1 -
46507 MaskVal->getAPIntValue().exactLogBase2());
46508 }
46509 // vsel ((X & C) == 0), LHS, RHS --> vsel ((shl X, C') < 0), RHS, LHS
46510 SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL);
46511 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt);
46512 SDValue NewCond =
46513 DAG.getSetCC(DL, CondVT, Shl, Cond.getOperand(1), ISD::SETLT);
46514 return DAG.getSelect(DL, VT, NewCond, RHS, LHS);
46515 }
46516 }
46517
46518 return SDValue();
46519}
46520
46521/// Combine:
46522/// (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S)
46523/// to:
46524/// (brcond/cmov/setcc .., (LADD x, 1), COND_LE)
46525/// i.e., reusing the EFLAGS produced by the LOCKed instruction.
46526/// Note that this is only legal for some op/cc combinations.
46527static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
46528 SelectionDAG &DAG,
46529 const X86Subtarget &Subtarget) {
46530 // This combine only operates on CMP-like nodes.
46531 if (!(Cmp.getOpcode() == X86ISD::CMP ||
46532 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
46533 return SDValue();
46534
46535 // Can't replace the cmp if it has more uses than the one we're looking at.
46536 // FIXME: We would like to be able to handle this, but would need to make sure
46537 // all uses were updated.
46538 if (!Cmp.hasOneUse())
46539 return SDValue();
46540
46541 // This only applies to variations of the common case:
46542 // (icmp slt x, 0) -> (icmp sle (add x, 1), 0)
46543 // (icmp sge x, 0) -> (icmp sgt (add x, 1), 0)
46544 // (icmp sle x, 0) -> (icmp slt (sub x, 1), 0)
46545 // (icmp sgt x, 0) -> (icmp sge (sub x, 1), 0)
46546 // Using the proper condcodes (see below), overflow is checked for.
46547
46548 // FIXME: We can generalize both constraints:
46549 // - XOR/OR/AND (if they were made to survive AtomicExpand)
46550 // - LHS != 1
46551 // if the result is compared.
46552
46553 SDValue CmpLHS = Cmp.getOperand(0);
46554 SDValue CmpRHS = Cmp.getOperand(1);
46555 EVT CmpVT = CmpLHS.getValueType();
46556
46557 if (!CmpLHS.hasOneUse())
46558 return SDValue();
46559
46560 unsigned Opc = CmpLHS.getOpcode();
46561 if (Opc != ISD::ATOMIC_LOAD_ADD && Opc != ISD::ATOMIC_LOAD_SUB)
46562 return SDValue();
46563
46564 SDValue OpRHS = CmpLHS.getOperand(2);
46565 auto *OpRHSC = dyn_cast<ConstantSDNode>(OpRHS);
46566 if (!OpRHSC)
46567 return SDValue();
46568
46569 APInt Addend = OpRHSC->getAPIntValue();
46570 if (Opc == ISD::ATOMIC_LOAD_SUB)
46571 Addend = -Addend;
46572
46573 auto *CmpRHSC = dyn_cast<ConstantSDNode>(CmpRHS);
46574 if (!CmpRHSC)
46575 return SDValue();
46576
46577 APInt Comparison = CmpRHSC->getAPIntValue();
46578 APInt NegAddend = -Addend;
46579
46580 // See if we can adjust the CC to make the comparison match the negated
46581 // addend.
46582 if (Comparison != NegAddend) {
46583 APInt IncComparison = Comparison + 1;
46584 if (IncComparison == NegAddend) {
46585 if (CC == X86::COND_A && !Comparison.isMaxValue()) {
46586 Comparison = IncComparison;
46587 CC = X86::COND_AE;
46588 } else if (CC == X86::COND_LE && !Comparison.isMaxSignedValue()) {
46589 Comparison = IncComparison;
46590 CC = X86::COND_L;
46591 }
46592 }
46593 APInt DecComparison = Comparison - 1;
46594 if (DecComparison == NegAddend) {
46595 if (CC == X86::COND_AE && !Comparison.isMinValue()) {
46596 Comparison = DecComparison;
46597 CC = X86::COND_A;
46598 } else if (CC == X86::COND_L && !Comparison.isMinSignedValue()) {
46599 Comparison = DecComparison;
46600 CC = X86::COND_LE;
46601 }
46602 }
46603 }
46604
46605 // If the addend is the negation of the comparison value, then we can do
46606 // a full comparison by emitting the atomic arithmetic as a locked sub.
46607 if (Comparison == NegAddend) {
46608 // The CC is fine, but we need to rewrite the LHS of the comparison as an
46609 // atomic sub.
46610 auto *AN = cast<AtomicSDNode>(CmpLHS.getNode());
46611 auto AtomicSub = DAG.getAtomic(
46612 ISD::ATOMIC_LOAD_SUB, SDLoc(CmpLHS), CmpVT,
46613 /*Chain*/ CmpLHS.getOperand(0), /*LHS*/ CmpLHS.getOperand(1),
46614 /*RHS*/ DAG.getConstant(NegAddend, SDLoc(CmpRHS), CmpVT),
46615 AN->getMemOperand());
46616 auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget);
46617 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT));
46618 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
46619 return LockOp;
46620 }
46621
46622 // We can handle comparisons with zero in a number of cases by manipulating
46623 // the CC used.
46624 if (!Comparison.isZero())
46625 return SDValue();
46626
46627 if (CC == X86::COND_S && Addend == 1)
46628 CC = X86::COND_LE;
46629 else if (CC == X86::COND_NS && Addend == 1)
46630 CC = X86::COND_G;
46631 else if (CC == X86::COND_G && Addend == -1)
46632 CC = X86::COND_GE;
46633 else if (CC == X86::COND_LE && Addend == -1)
46634 CC = X86::COND_L;
46635 else
46636 return SDValue();
46637
46638 SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget);
46639 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0), DAG.getUNDEF(CmpVT));
46640 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
46641 return LockOp;
46642}
46643
46644// Check whether we're just testing the signbit, and whether we can simplify
46645// this by tracking where the signbit came from.
46646static SDValue checkSignTestSetCCCombine(SDValue Cmp, X86::CondCode &CC,
46647 SelectionDAG &DAG) {
46648 if (CC != X86::COND_S && CC != X86::COND_NS)
46649 return SDValue();
46650
46651 if (!Cmp.hasOneUse())
46652 return SDValue();
46653
46654 SDValue Src;
46655 if (Cmp.getOpcode() == X86ISD::CMP) {
46656 // CMP(X,0) -> signbit test
46657 if (!isNullConstant(Cmp.getOperand(1)))
46658 return SDValue();
46659 Src = Cmp.getOperand(0);
46660 // Peek through a SRA node as we just need the signbit.
46661 // TODO: Remove one use limit once sdiv-fix regressions are fixed.
46662 // TODO: Use SimplifyDemandedBits instead of just SRA?
46663 if (Src.getOpcode() != ISD::SRA || !Src.hasOneUse())
46664 return SDValue();
46665 Src = Src.getOperand(0);
46666 } else if (Cmp.getOpcode() == X86ISD::OR) {
46667 // OR(X,Y) -> see if only one operand contributes to the signbit.
46668 // TODO: XOR(X,Y) -> see if only one operand contributes to the signbit.
46669 if (DAG.SignBitIsZero(Cmp.getOperand(0)))
46670 Src = Cmp.getOperand(1);
46671 else if (DAG.SignBitIsZero(Cmp.getOperand(1)))
46672 Src = Cmp.getOperand(0);
46673 else
46674 return SDValue();
46675 } else {
46676 return SDValue();
46677 }
46678
46679 // Replace with a TEST on the MSB.
46680 SDLoc DL(Cmp);
46681 MVT SrcVT = Src.getSimpleValueType();
46682 APInt BitMask = APInt::getSignMask(SrcVT.getScalarSizeInBits());
46683
46684 // If Src came from a SHL (probably from an expanded SIGN_EXTEND_INREG), then
46685 // peek through and adjust the TEST bit.
46686 if (Src.getOpcode() == ISD::SHL) {
46687 if (std::optional<uint64_t> ShiftAmt = DAG.getValidShiftAmount(Src)) {
46688 Src = Src.getOperand(0);
46689 BitMask.lshrInPlace(*ShiftAmt);
46690 }
46691 }
46692
46693 SDValue Mask = DAG.getNode(ISD::AND, DL, SrcVT, Src,
46694 DAG.getConstant(BitMask, DL, SrcVT));
46695 CC = CC == X86::COND_S ? X86::COND_NE : X86::COND_E;
46696 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Mask,
46697 DAG.getConstant(0, DL, SrcVT));
46698}
46699
46700// Check whether a boolean test is testing a boolean value generated by
46701// X86ISD::SETCC. If so, return the operand of that SETCC and proper condition
46702// code.
46703//
46704// Simplify the following patterns:
46705// (Op (CMP (SETCC Cond EFLAGS) 1) EQ) or
46706// (Op (CMP (SETCC Cond EFLAGS) 0) NEQ)
46707// to (Op EFLAGS Cond)
46708//
46709// (Op (CMP (SETCC Cond EFLAGS) 0) EQ) or
46710// (Op (CMP (SETCC Cond EFLAGS) 1) NEQ)
46711// to (Op EFLAGS !Cond)
46712//
46713// where Op could be BRCOND or CMOV.
46714//
46715static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
46716 // This combine only operates on CMP-like nodes.
46717 if (!(Cmp.getOpcode() == X86ISD::CMP ||
46718 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
46719 return SDValue();
46720
46721 // Quit if not used as a boolean value.
46722 if (CC != X86::COND_E && CC != X86::COND_NE)
46723 return SDValue();
46724
46725 // Check CMP operands. One of them should be 0 or 1 and the other should be
46726 // an SetCC or extended from it.
46727 SDValue Op1 = Cmp.getOperand(0);
46728 SDValue Op2 = Cmp.getOperand(1);
46729
46730 SDValue SetCC;
46731 const ConstantSDNode* C = nullptr;
46732 bool needOppositeCond = (CC == X86::COND_E);
46733 bool checkAgainstTrue = false; // Is it a comparison against 1?
46734
46735 if ((C = dyn_cast<ConstantSDNode>(Op1)))
46736 SetCC = Op2;
46737 else if ((C = dyn_cast<ConstantSDNode>(Op2)))
46738 SetCC = Op1;
46739 else // Quit if all operands are not constants.
46740 return SDValue();
46741
46742 if (C->getZExtValue() == 1) {
46743 needOppositeCond = !needOppositeCond;
46744 checkAgainstTrue = true;
46745 } else if (C->getZExtValue() != 0)
46746 // Quit if the constant is neither 0 or 1.
46747 return SDValue();
46748
46749 bool truncatedToBoolWithAnd = false;
46750 // Skip (zext $x), (trunc $x), or (and $x, 1) node.
46751 while (SetCC.getOpcode() == ISD::ZERO_EXTEND ||
46752 SetCC.getOpcode() == ISD::TRUNCATE ||
46753 SetCC.getOpcode() == ISD::AND) {
46754 if (SetCC.getOpcode() == ISD::AND) {
46755 int OpIdx = -1;
46756 if (isOneConstant(SetCC.getOperand(0)))
46757 OpIdx = 1;
46758 if (isOneConstant(SetCC.getOperand(1)))
46759 OpIdx = 0;
46760 if (OpIdx < 0)
46761 break;
46762 SetCC = SetCC.getOperand(OpIdx);
46763 truncatedToBoolWithAnd = true;
46764 } else
46765 SetCC = SetCC.getOperand(0);
46766 }
46767
46768 switch (SetCC.getOpcode()) {
46769 case X86ISD::SETCC_CARRY:
46770 // Since SETCC_CARRY gives output based on R = CF ? ~0 : 0, it's unsafe to
46771 // simplify it if the result of SETCC_CARRY is not canonicalized to 0 or 1,
46772 // i.e. it's a comparison against true but the result of SETCC_CARRY is not
46773 // truncated to i1 using 'and'.
46774 if (checkAgainstTrue && !truncatedToBoolWithAnd)
46775 break;
46776 assert(X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B &&
46777 "Invalid use of SETCC_CARRY!");
46778 [[fallthrough]];
46779 case X86ISD::SETCC:
46780 // Set the condition code or opposite one if necessary.
46781 CC = X86::CondCode(SetCC.getConstantOperandVal(0));
46782 if (needOppositeCond)
46783 CC = X86::GetOppositeBranchCondition(CC);
46784 return SetCC.getOperand(1);
46785 case X86ISD::CMOV: {
46786 // Check whether false/true value has canonical one, i.e. 0 or 1.
46787 ConstantSDNode *FVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(0));
46788 ConstantSDNode *TVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(1));
46789 // Quit if true value is not a constant.
46790 if (!TVal)
46791 return SDValue();
46792 // Quit if false value is not a constant.
46793 if (!FVal) {
46794 SDValue Op = SetCC.getOperand(0);
46795 // Skip 'zext' or 'trunc' node.
46796 if (Op.getOpcode() == ISD::ZERO_EXTEND ||
46797 Op.getOpcode() == ISD::TRUNCATE)
46798 Op = Op.getOperand(0);
46799 // A special case for rdrand/rdseed, where 0 is set if false cond is
46800 // found.
46801 if ((Op.getOpcode() != X86ISD::RDRAND &&
46802 Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
46803 return SDValue();
46804 }
46805 // Quit if false value is not the constant 0 or 1.
46806 bool FValIsFalse = true;
46807 if (FVal && FVal->getZExtValue() != 0) {
46808 if (FVal->getZExtValue() != 1)
46809 return SDValue();
46810 // If FVal is 1, opposite cond is needed.
46811 needOppositeCond = !needOppositeCond;
46812 FValIsFalse = false;
46813 }
46814 // Quit if TVal is not the constant opposite of FVal.
46815 if (FValIsFalse && TVal->getZExtValue() != 1)
46816 return SDValue();
46817 if (!FValIsFalse && TVal->getZExtValue() != 0)
46818 return SDValue();
46819 CC = X86::CondCode(SetCC.getConstantOperandVal(2));
46820 if (needOppositeCond)
46821 CC = X86::GetOppositeBranchCondition(CC);
46822 return SetCC.getOperand(3);
46823 }
46824 }
46825
46826 return SDValue();
46827}
46828
46829/// Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
46830/// Match:
46831/// (X86or (X86setcc) (X86setcc))
46832/// (X86cmp (and (X86setcc) (X86setcc)), 0)
46833static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
46834 X86::CondCode &CC1, SDValue &Flags,
46835 bool &isAnd) {
46836 if (Cond->getOpcode() == X86ISD::CMP) {
46837 if (!isNullConstant(Cond->getOperand(1)))
46838 return false;
46839
46840 Cond = Cond->getOperand(0);
46841 }
46842
46843 isAnd = false;
46844
46845 SDValue SetCC0, SetCC1;
46846 switch (Cond->getOpcode()) {
46847 default: return false;
46848 case ISD::AND:
46849 case X86ISD::AND:
46850 isAnd = true;
46851 [[fallthrough]];
46852 case ISD::OR:
46853 case X86ISD::OR:
46854 SetCC0 = Cond->getOperand(0);
46855 SetCC1 = Cond->getOperand(1);
46856 break;
46857 };
46858
46859 // Make sure we have SETCC nodes, using the same flags value.
46860 if (SetCC0.getOpcode() != X86ISD::SETCC ||
46861 SetCC1.getOpcode() != X86ISD::SETCC ||
46862 SetCC0->getOperand(1) != SetCC1->getOperand(1))
46863 return false;
46864
46865 CC0 = (X86::CondCode)SetCC0->getConstantOperandVal(0);
46866 CC1 = (X86::CondCode)SetCC1->getConstantOperandVal(0);
46867 Flags = SetCC0->getOperand(1);
46868 return true;
46869}
46870
46871// When legalizing carry, we create carries via add X, -1
46872// If that comes from an actual carry, via setcc, we use the
46873// carry directly.
46874static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
46875 if (EFLAGS.getOpcode() == X86ISD::ADD) {
46876 if (isAllOnesConstant(EFLAGS.getOperand(1))) {
46877 bool FoundAndLSB = false;
46878 SDValue Carry = EFLAGS.getOperand(0);
46879 while (Carry.getOpcode() == ISD::TRUNCATE ||
46880 Carry.getOpcode() == ISD::ZERO_EXTEND ||
46881 (Carry.getOpcode() == ISD::AND &&
46882 isOneConstant(Carry.getOperand(1)))) {
46883 FoundAndLSB |= Carry.getOpcode() == ISD::AND;
46884 Carry = Carry.getOperand(0);
46885 }
46886 if (Carry.getOpcode() == X86ISD::SETCC ||
46887 Carry.getOpcode() == X86ISD::SETCC_CARRY) {
46888 // TODO: Merge this code with equivalent in combineAddOrSubToADCOrSBB?
46889 uint64_t CarryCC = Carry.getConstantOperandVal(0);
46890 SDValue CarryOp1 = Carry.getOperand(1);
46891 if (CarryCC == X86::COND_B)
46892 return CarryOp1;
46893 if (CarryCC == X86::COND_A) {
46894 // Try to convert COND_A into COND_B in an attempt to facilitate
46895 // materializing "setb reg".
46896 //
46897 // Do not flip "e > c", where "c" is a constant, because Cmp
46898 // instruction cannot take an immediate as its first operand.
46899 //
46900 if (CarryOp1.getOpcode() == X86ISD::SUB &&
46901 CarryOp1.getNode()->hasOneUse() &&
46902 CarryOp1.getValueType().isInteger() &&
46903 !isa<ConstantSDNode>(CarryOp1.getOperand(1))) {
46904 SDValue SubCommute =
46905 DAG.getNode(X86ISD::SUB, SDLoc(CarryOp1), CarryOp1->getVTList(),
46906 CarryOp1.getOperand(1), CarryOp1.getOperand(0));
46907 return SDValue(SubCommute.getNode(), CarryOp1.getResNo());
46908 }
46909 }
46910 // If this is a check of the z flag of an add with 1, switch to the
46911 // C flag.
46912 if (CarryCC == X86::COND_E &&
46913 CarryOp1.getOpcode() == X86ISD::ADD &&
46914 isOneConstant(CarryOp1.getOperand(1)))
46915 return CarryOp1;
46916 } else if (FoundAndLSB) {
46917 SDLoc DL(Carry);
46918 SDValue BitNo = DAG.getConstant(0, DL, Carry.getValueType());
46919 if (Carry.getOpcode() == ISD::SRL) {
46920 BitNo = Carry.getOperand(1);
46921 Carry = Carry.getOperand(0);
46922 }
46923 return getBT(Carry, BitNo, DL, DAG);
46924 }
46925 }
46926 }
46927
46928 return SDValue();
46929}
46930
46931/// If we are inverting an PTEST/TESTP operand, attempt to adjust the CC
46932/// to avoid the inversion.
46933static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC,
46934 SelectionDAG &DAG,
46935 const X86Subtarget &Subtarget) {
46936 // TODO: Handle X86ISD::KTEST/X86ISD::KORTEST.
46937 if (EFLAGS.getOpcode() != X86ISD::PTEST &&
46938 EFLAGS.getOpcode() != X86ISD::TESTP)
46939 return SDValue();
46940
46941 // PTEST/TESTP sets EFLAGS as:
46942 // TESTZ: ZF = (Op0 & Op1) == 0
46943 // TESTC: CF = (~Op0 & Op1) == 0
46944 // TESTNZC: ZF == 0 && CF == 0
46945 MVT VT = EFLAGS.getSimpleValueType();
46946 SDValue Op0 = EFLAGS.getOperand(0);
46947 SDValue Op1 = EFLAGS.getOperand(1);
46948 MVT OpVT = Op0.getSimpleValueType();
46949 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
46950
46951 // TEST*(~X,Y) == TEST*(X,Y)
46952 if (SDValue NotOp0 = IsNOT(Op0, DAG)) {
46953 X86::CondCode InvCC;
46954 switch (CC) {
46955 case X86::COND_B:
46956 // testc -> testz.
46957 InvCC = X86::COND_E;
46958 break;
46959 case X86::COND_AE:
46960 // !testc -> !testz.
46961 InvCC = X86::COND_NE;
46962 break;
46963 case X86::COND_E:
46964 // testz -> testc.
46965 InvCC = X86::COND_B;
46966 break;
46967 case X86::COND_NE:
46968 // !testz -> !testc.
46969 InvCC = X86::COND_AE;
46970 break;
46971 case X86::COND_A:
46972 case X86::COND_BE:
46973 // testnzc -> testnzc (no change).
46974 InvCC = CC;
46975 break;
46976 default:
46977 InvCC = X86::COND_INVALID;
46978 break;
46979 }
46980
46981 if (InvCC != X86::COND_INVALID) {
46982 CC = InvCC;
46983 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
46984 DAG.getBitcast(OpVT, NotOp0), Op1);
46985 }
46986 }
46987
46988 if (CC == X86::COND_B || CC == X86::COND_AE) {
46989 // TESTC(X,~X) == TESTC(X,-1)
46990 if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
46991 if (peekThroughBitcasts(NotOp1) == peekThroughBitcasts(Op0)) {
46992 SDLoc DL(EFLAGS);
46993 return DAG.getNode(
46994 EFLAGS.getOpcode(), DL, VT, DAG.getBitcast(OpVT, NotOp1),
46995 DAG.getBitcast(OpVT,
46996 DAG.getAllOnesConstant(DL, NotOp1.getValueType())));
46997 }
46998 }
46999 }
47000
47001 if (CC == X86::COND_E || CC == X86::COND_NE) {
47002 // TESTZ(X,~Y) == TESTC(Y,X)
47003 if (SDValue NotOp1 = IsNOT(Op1, DAG)) {
47004 CC = (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
47005 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
47006 DAG.getBitcast(OpVT, NotOp1), Op0);
47007 }
47008
47009 if (Op0 == Op1) {
47010 SDValue BC = peekThroughBitcasts(Op0);
47011 EVT BCVT = BC.getValueType();
47012
47013 // TESTZ(AND(X,Y),AND(X,Y)) == TESTZ(X,Y)
47014 if (BC.getOpcode() == ISD::AND || BC.getOpcode() == X86ISD::FAND) {
47015 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
47016 DAG.getBitcast(OpVT, BC.getOperand(0)),
47017 DAG.getBitcast(OpVT, BC.getOperand(1)));
47018 }
47019
47020 // TESTZ(AND(~X,Y),AND(~X,Y)) == TESTC(X,Y)
47021 if (BC.getOpcode() == X86ISD::ANDNP || BC.getOpcode() == X86ISD::FANDN) {
47022 CC = (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
47023 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
47024 DAG.getBitcast(OpVT, BC.getOperand(0)),
47025 DAG.getBitcast(OpVT, BC.getOperand(1)));
47026 }
47027
47028 // If every element is an all-sign value, see if we can use TESTP/MOVMSK
47029 // to more efficiently extract the sign bits and compare that.
47030 // TODO: Handle TESTC with comparison inversion.
47031 // TODO: Can we remove SimplifyMultipleUseDemandedBits and rely on
47032 // TESTP/MOVMSK combines to make sure its never worse than PTEST?
47033 if (BCVT.isVector() && TLI.isTypeLegal(BCVT)) {
47034 unsigned EltBits = BCVT.getScalarSizeInBits();
47035 if (DAG.ComputeNumSignBits(BC) == EltBits) {
47036 assert(VT == MVT::i32 && "Expected i32 EFLAGS comparison result");
47037 APInt SignMask = APInt::getSignMask(EltBits);
47038 if (SDValue Res =
47039 TLI.SimplifyMultipleUseDemandedBits(BC, SignMask, DAG)) {
47040 // For vXi16 cases we need to use pmovmksb and extract every other
47041 // sign bit.
47042 SDLoc DL(EFLAGS);
47043 if ((EltBits == 32 || EltBits == 64) && Subtarget.hasAVX()) {
47044 MVT FloatSVT = MVT::getFloatingPointVT(EltBits);
47045 MVT FloatVT =
47046 MVT::getVectorVT(FloatSVT, OpVT.getSizeInBits() / EltBits);
47047 Res = DAG.getBitcast(FloatVT, Res);
47048 return DAG.getNode(X86ISD::TESTP, SDLoc(EFLAGS), VT, Res, Res);
47049 } else if (EltBits == 16) {
47050 MVT MovmskVT = BCVT.is128BitVector() ? MVT::v16i8 : MVT::v32i8;
47051 Res = DAG.getBitcast(MovmskVT, Res);
47052 Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
47053 Res = DAG.getNode(ISD::AND, DL, MVT::i32, Res,
47054 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
47055 } else {
47056 Res = getPMOVMSKB(DL, Res, DAG, Subtarget);
47057 }
47058 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Res,
47059 DAG.getConstant(0, DL, MVT::i32));
47060 }
47061 }
47062 }
47063 }
47064
47065 // TESTZ(-1,X) == TESTZ(X,X)
47066 if (ISD::isBuildVectorAllOnes(Op0.getNode()))
47067 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op1, Op1);
47068
47069 // TESTZ(X,-1) == TESTZ(X,X)
47070 if (ISD::isBuildVectorAllOnes(Op1.getNode()))
47071 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op0, Op0);
47072
47073 // TESTZ(OR(LO(X),HI(X)),OR(LO(Y),HI(Y))) -> TESTZ(X,Y)
47074 // TODO: Add COND_NE handling?
47075 if (CC == X86::COND_E && OpVT.is128BitVector() && Subtarget.hasAVX()) {
47076 SDValue Src0 = peekThroughBitcasts(Op0);
47077 SDValue Src1 = peekThroughBitcasts(Op1);
47078 if (Src0.getOpcode() == ISD::OR && Src1.getOpcode() == ISD::OR) {
47079 Src0 = getSplitVectorSrc(peekThroughBitcasts(Src0.getOperand(0)),
47080 peekThroughBitcasts(Src0.getOperand(1)), true);
47081 Src1 = getSplitVectorSrc(peekThroughBitcasts(Src1.getOperand(0)),
47082 peekThroughBitcasts(Src1.getOperand(1)), true);
47083 if (Src0 && Src1) {
47084 MVT OpVT2 = OpVT.getDoubleNumVectorElementsVT();
47085 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT,
47086 DAG.getBitcast(OpVT2, Src0),
47087 DAG.getBitcast(OpVT2, Src1));
47088 }
47089 }
47090 }
47091 }
47092
47093 return SDValue();
47094}
47095
47096// Attempt to simplify the MOVMSK input based on the comparison type.
47097static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
47098 SelectionDAG &DAG,
47099 const X86Subtarget &Subtarget) {
47100 // Handle eq/ne against zero (any_of).
47101 // Handle eq/ne against -1 (all_of).
47102 if (!(CC == X86::COND_E || CC == X86::COND_NE))
47103 return SDValue();
47104 if (EFLAGS.getValueType() != MVT::i32)
47105 return SDValue();
47106 unsigned CmpOpcode = EFLAGS.getOpcode();
47107 if (CmpOpcode != X86ISD::CMP && CmpOpcode != X86ISD::SUB)
47108 return SDValue();
47109 auto *CmpConstant = dyn_cast<ConstantSDNode>(EFLAGS.getOperand(1));
47110 if (!CmpConstant)
47111 return SDValue();
47112 const APInt &CmpVal = CmpConstant->getAPIntValue();
47113
47114 SDValue CmpOp = EFLAGS.getOperand(0);
47115 unsigned CmpBits = CmpOp.getValueSizeInBits();
47116 assert(CmpBits == CmpVal.getBitWidth() && "Value size mismatch");
47117
47118 // Peek through any truncate.
47119 if (CmpOp.getOpcode() == ISD::TRUNCATE)
47120 CmpOp = CmpOp.getOperand(0);
47121
47122 // Bail if we don't find a MOVMSK.
47123 if (CmpOp.getOpcode() != X86ISD::MOVMSK)
47124 return SDValue();
47125
47126 SDValue Vec = CmpOp.getOperand(0);
47127 MVT VecVT = Vec.getSimpleValueType();
47128 assert((VecVT.is128BitVector() || VecVT.is256BitVector()) &&
47129 "Unexpected MOVMSK operand");
47130 unsigned NumElts = VecVT.getVectorNumElements();
47131 unsigned NumEltBits = VecVT.getScalarSizeInBits();
47132
47133 bool IsAnyOf = CmpOpcode == X86ISD::CMP && CmpVal.isZero();
47134 bool IsAllOf = (CmpOpcode == X86ISD::SUB || CmpOpcode == X86ISD::CMP) &&
47135 NumElts <= CmpBits && CmpVal.isMask(NumElts);
47136 if (!IsAnyOf && !IsAllOf)
47137 return SDValue();
47138
47139 // TODO: Check more combining cases for me.
47140 // Here we check the cmp use number to decide do combining or not.
47141 // Currently we only get 2 tests about combining "MOVMSK(CONCAT(..))"
47142 // and "MOVMSK(PCMPEQ(..))" are fit to use this constraint.
47143 bool IsOneUse = CmpOp.getNode()->hasOneUse();
47144
47145 // See if we can peek through to a vector with a wider element type, if the
47146 // signbits extend down to all the sub-elements as well.
47147 // Calling MOVMSK with the wider type, avoiding the bitcast, helps expose
47148 // potential SimplifyDemandedBits/Elts cases.
47149 // If we looked through a truncate that discard bits, we can't do this
47150 // transform.
47151 // FIXME: We could do this transform for truncates that discarded bits by
47152 // inserting an AND mask between the new MOVMSK and the CMP.
47153 if (Vec.getOpcode() == ISD::BITCAST && NumElts <= CmpBits) {
47154 SDValue BC = peekThroughBitcasts(Vec);
47155 MVT BCVT = BC.getSimpleValueType();
47156 unsigned BCNumElts = BCVT.getVectorNumElements();
47157 unsigned BCNumEltBits = BCVT.getScalarSizeInBits();
47158 if ((BCNumEltBits == 32 || BCNumEltBits == 64) &&
47159 BCNumEltBits > NumEltBits &&
47160 DAG.ComputeNumSignBits(BC) > (BCNumEltBits - NumEltBits)) {
47161 SDLoc DL(EFLAGS);
47162 APInt CmpMask = APInt::getLowBitsSet(32, IsAnyOf ? 0 : BCNumElts);
47163 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
47164 DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, BC),
47165 DAG.getConstant(CmpMask, DL, MVT::i32));
47166 }
47167 }
47168
47169 // MOVMSK(CONCAT(X,Y)) == 0 -> MOVMSK(OR(X,Y)).
47170 // MOVMSK(CONCAT(X,Y)) != 0 -> MOVMSK(OR(X,Y)).
47171 // MOVMSK(CONCAT(X,Y)) == -1 -> MOVMSK(AND(X,Y)).
47172 // MOVMSK(CONCAT(X,Y)) != -1 -> MOVMSK(AND(X,Y)).
47173 if (VecVT.is256BitVector() && NumElts <= CmpBits && IsOneUse) {
47174 SmallVector<SDValue> Ops;
47175 if (collectConcatOps(peekThroughBitcasts(Vec).getNode(), Ops, DAG) &&
47176 Ops.size() == 2) {
47177 SDLoc DL(EFLAGS);
47178 EVT SubVT = Ops[0].getValueType().changeTypeToInteger();
47179 APInt CmpMask = APInt::getLowBitsSet(32, IsAnyOf ? 0 : NumElts / 2);
47180 SDValue V = DAG.getNode(IsAnyOf ? ISD::OR : ISD::AND, DL, SubVT,
47181 DAG.getBitcast(SubVT, Ops[0]),
47182 DAG.getBitcast(SubVT, Ops[1]));
47183 V = DAG.getBitcast(VecVT.getHalfNumVectorElementsVT(), V);
47184 return DAG.getNode(X86ISD::CMP, DL, MVT::i32,
47185 DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V),
47186 DAG.getConstant(CmpMask, DL, MVT::i32));
47187 }
47188 }
47189
47190 // MOVMSK(PCMPEQ(X,0)) == -1 -> PTESTZ(X,X).
47191 // MOVMSK(PCMPEQ(X,0)) != -1 -> !PTESTZ(X,X).
47192 // MOVMSK(PCMPEQ(X,Y)) == -1 -> PTESTZ(XOR(X,Y),XOR(X,Y)).
47193 // MOVMSK(PCMPEQ(X,Y)) != -1 -> !PTESTZ(XOR(X,Y),XOR(X,Y)).
47194 if (IsAllOf && Subtarget.hasSSE41() && IsOneUse) {
47195 MVT TestVT = VecVT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
47196 SDValue BC = peekThroughBitcasts(Vec);
47197 // Ensure MOVMSK was testing every signbit of BC.
47198 if (BC.getValueType().getVectorNumElements() <= NumElts) {
47199 if (BC.getOpcode() == X86ISD::PCMPEQ) {
47200 SDValue V = DAG.getNode(ISD::XOR, SDLoc(BC), BC.getValueType(),
47201 BC.getOperand(0), BC.getOperand(1));
47202 V = DAG.getBitcast(TestVT, V);
47203 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
47204 }
47205 // Check for 256-bit split vector cases.
47206 if (BC.getOpcode() == ISD::AND &&
47207 BC.getOperand(0).getOpcode() == X86ISD::PCMPEQ &&
47208 BC.getOperand(1).getOpcode() == X86ISD::PCMPEQ) {
47209 SDValue LHS = BC.getOperand(0);
47210 SDValue RHS = BC.getOperand(1);
47211 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), LHS.getValueType(),
47212 LHS.getOperand(0), LHS.getOperand(1));
47213 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), RHS.getValueType(),
47214 RHS.getOperand(0), RHS.getOperand(1));
47215 LHS = DAG.getBitcast(TestVT, LHS);
47216 RHS = DAG.getBitcast(TestVT, RHS);
47217 SDValue V = DAG.getNode(ISD::OR, SDLoc(EFLAGS), TestVT, LHS, RHS);
47218 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
47219 }
47220 }
47221 }
47222
47223 // See if we can avoid a PACKSS by calling MOVMSK on the sources.
47224 // For vXi16 cases we can use a v2Xi8 PMOVMSKB. We must mask out
47225 // sign bits prior to the comparison with zero unless we know that
47226 // the vXi16 splats the sign bit down to the lower i8 half.
47227 // TODO: Handle all_of patterns.
47228 if (Vec.getOpcode() == X86ISD::PACKSS && VecVT == MVT::v16i8) {
47229 SDValue VecOp0 = Vec.getOperand(0);
47230 SDValue VecOp1 = Vec.getOperand(1);
47231 bool SignExt0 = DAG.ComputeNumSignBits(VecOp0) > 8;
47232 bool SignExt1 = DAG.ComputeNumSignBits(VecOp1) > 8;
47233 // PMOVMSKB(PACKSSBW(X, undef)) -> PMOVMSKB(BITCAST_v16i8(X)) & 0xAAAA.
47234 if (IsAnyOf && CmpBits == 8 && VecOp1.isUndef()) {
47235 SDLoc DL(EFLAGS);
47236 SDValue Result = DAG.getBitcast(MVT::v16i8, VecOp0);
47237 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
47238 Result = DAG.getZExtOrTrunc(Result, DL, MVT::i16);
47239 if (!SignExt0) {
47240 Result = DAG.getNode(ISD::AND, DL, MVT::i16, Result,
47241 DAG.getConstant(0xAAAA, DL, MVT::i16));
47242 }
47243 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
47244 DAG.getConstant(0, DL, MVT::i16));
47245 }
47246 // PMOVMSKB(PACKSSBW(LO(X), HI(X)))
47247 // -> PMOVMSKB(BITCAST_v32i8(X)) & 0xAAAAAAAA.
47248 if (CmpBits >= 16 && Subtarget.hasInt256() &&
47249 (IsAnyOf || (SignExt0 && SignExt1))) {
47250 if (SDValue Src = getSplitVectorSrc(VecOp0, VecOp1, true)) {
47251 SDLoc DL(EFLAGS);
47252 SDValue Result = peekThroughBitcasts(Src);
47253 if (IsAllOf && Result.getOpcode() == X86ISD::PCMPEQ &&
47254 Result.getValueType().getVectorNumElements() <= NumElts) {
47255 SDValue V = DAG.getNode(ISD::XOR, DL, Result.getValueType(),
47256 Result.getOperand(0), Result.getOperand(1));
47257 V = DAG.getBitcast(MVT::v4i64, V);
47258 return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);
47259 }
47260 Result = DAG.getBitcast(MVT::v32i8, Result);
47261 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
47262 unsigned CmpMask = IsAnyOf ? 0 : 0xFFFFFFFF;
47263 if (!SignExt0 || !SignExt1) {
47264 assert(IsAnyOf &&
47265 "Only perform v16i16 signmasks for any_of patterns");
47266 Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result,
47267 DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));
47268 }
47269 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
47270 DAG.getConstant(CmpMask, DL, MVT::i32));
47271 }
47272 }
47273 }
47274
47275 // MOVMSK(SHUFFLE(X,u)) -> MOVMSK(X) iff every element is referenced.
47276 // Since we peek through a bitcast, we need to be careful if the base vector
47277 // type has smaller elements than the MOVMSK type. In that case, even if
47278 // all the elements are demanded by the shuffle mask, only the "high"
47279 // elements which have highbits that align with highbits in the MOVMSK vec
47280 // elements are actually demanded. A simplification of spurious operations
47281 // on the "low" elements take place during other simplifications.
47282 //
47283 // For example:
47284 // MOVMSK64(BITCAST(SHUF32 X, (1,0,3,2))) even though all the elements are
47285 // demanded, because we are swapping around the result can change.
47286 //
47287 // To address this, we check that we can scale the shuffle mask to MOVMSK
47288 // element width (this will ensure "high" elements match). Its slightly overly
47289 // conservative, but fine for an edge case fold.
47290 SmallVector<int, 32> ShuffleMask;
47291 SmallVector<SDValue, 2> ShuffleInputs;
47292 if (NumElts <= CmpBits &&
47293 getTargetShuffleInputs(peekThroughBitcasts(Vec), ShuffleInputs,
47294 ShuffleMask, DAG) &&
47295 ShuffleInputs.size() == 1 && isCompletePermute(ShuffleMask) &&
47296 ShuffleInputs[0].getValueSizeInBits() == VecVT.getSizeInBits() &&
47297 canScaleShuffleElements(ShuffleMask, NumElts)) {
47298 SDLoc DL(EFLAGS);
47299 SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
47300 Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
47301 Result =
47302 DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType());
47303 return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, EFLAGS.getOperand(1));
47304 }
47305
47306 // MOVMSKPS(V) !=/== 0 -> TESTPS(V,V)
47307 // MOVMSKPD(V) !=/== 0 -> TESTPD(V,V)
47308 // MOVMSKPS(V) !=/== -1 -> TESTPS(V,V)
47309 // MOVMSKPD(V) !=/== -1 -> TESTPD(V,V)
47310 // iff every element is referenced.
47311 if (NumElts <= CmpBits && Subtarget.hasAVX() &&
47312 !Subtarget.preferMovmskOverVTest() && IsOneUse &&
47313 (NumEltBits == 32 || NumEltBits == 64)) {
47314 SDLoc DL(EFLAGS);
47315 MVT FloatSVT = MVT::getFloatingPointVT(NumEltBits);
47316 MVT FloatVT = MVT::getVectorVT(FloatSVT, NumElts);
47317 MVT IntVT = FloatVT.changeVectorElementTypeToInteger();
47318 SDValue LHS = Vec;
47319 SDValue RHS = IsAnyOf ? Vec : DAG.getAllOnesConstant(DL, IntVT);
47320 CC = IsAnyOf ? CC : (CC == X86::COND_E ? X86::COND_B : X86::COND_AE);
47321 return DAG.getNode(X86ISD::TESTP, DL, MVT::i32,
47322 DAG.getBitcast(FloatVT, LHS),
47323 DAG.getBitcast(FloatVT, RHS));
47324 }
47325
47326 return SDValue();
47327}
47328
47329/// Optimize an EFLAGS definition used according to the condition code \p CC
47330/// into a simpler EFLAGS value, potentially returning a new \p CC and replacing
47331/// uses of chain values.
47332static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
47333 SelectionDAG &DAG,
47334 const X86Subtarget &Subtarget) {
47335 if (CC == X86::COND_B)
47336 if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG))
47337 return Flags;
47338
47339 if (SDValue R = checkSignTestSetCCCombine(EFLAGS, CC, DAG))
47340 return R;
47341
47342 if (SDValue R = checkBoolTestSetCCCombine(EFLAGS, CC))
47343 return R;
47344
47345 if (SDValue R = combinePTESTCC(EFLAGS, CC, DAG, Subtarget))
47346 return R;
47347
47348 if (SDValue R = combineSetCCMOVMSK(EFLAGS, CC, DAG, Subtarget))
47349 return R;
47350
47351 return combineSetCCAtomicArith(EFLAGS, CC, DAG, Subtarget);
47352}
47353
47354/// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
47355static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
47356 TargetLowering::DAGCombinerInfo &DCI,
47357 const X86Subtarget &Subtarget) {
47358 SDLoc DL(N);
47359
47360 SDValue FalseOp = N->getOperand(0);
47361 SDValue TrueOp = N->getOperand(1);
47362 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
47363 SDValue Cond = N->getOperand(3);
47364
47365 // cmov X, X, ?, ? --> X
47366 if (TrueOp == FalseOp)
47367 return TrueOp;
47368
47369 // Try to simplify the EFLAGS and condition code operands.
47370 // We can't always do this as FCMOV only supports a subset of X86 cond.
47371 if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) {
47372 if (!(FalseOp.getValueType() == MVT::f80 ||
47373 (FalseOp.getValueType() == MVT::f64 && !Subtarget.hasSSE2()) ||
47374 (FalseOp.getValueType() == MVT::f32 && !Subtarget.hasSSE1())) ||
47375 !Subtarget.canUseCMOV() || hasFPCMov(CC)) {
47376 SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
47377 Flags};
47378 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
47379 }
47380 }
47381
47382 // If this is a select between two integer constants, try to do some
47383 // optimizations. Note that the operands are ordered the opposite of SELECT
47384 // operands.
47385 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(TrueOp)) {
47386 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(FalseOp)) {
47387 // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is
47388 // larger than FalseC (the false value).
47389 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
47390 CC = X86::GetOppositeBranchCondition(CC);
47391 std::swap(TrueC, FalseC);
47392 std::swap(TrueOp, FalseOp);
47393 }
47394
47395 // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3. Likewise for any pow2/0.
47396 // This is efficient for any integer data type (including i8/i16) and
47397 // shift amount.
47398 if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
47399 Cond = getSETCC(CC, Cond, DL, DAG);
47400
47401 // Zero extend the condition if needed.
47402 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
47403
47404 unsigned ShAmt = TrueC->getAPIntValue().logBase2();
47405 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
47406 DAG.getConstant(ShAmt, DL, MVT::i8));
47407 return Cond;
47408 }
47409
47410 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient
47411 // for any integer data type, including i8/i16.
47412 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
47413 Cond = getSETCC(CC, Cond, DL, DAG);
47414
47415 // Zero extend the condition if needed.
47416 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
47417 FalseC->getValueType(0), Cond);
47418 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
47419 SDValue(FalseC, 0));
47420 return Cond;
47421 }
47422
47423 // Optimize cases that will turn into an LEA instruction. This requires
47424 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
47425 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
47426 APInt Diff = TrueC->getAPIntValue() - FalseC->getAPIntValue();
47427 assert(Diff.getBitWidth() == N->getValueType(0).getSizeInBits() &&
47428 "Implicit constant truncation");
47429
47430 bool isFastMultiplier = false;
47431 if (Diff.ult(10)) {
47432 switch (Diff.getZExtValue()) {
47433 default: break;
47434 case 1: // result = add base, cond
47435 case 2: // result = lea base( , cond*2)
47436 case 3: // result = lea base(cond, cond*2)
47437 case 4: // result = lea base( , cond*4)
47438 case 5: // result = lea base(cond, cond*4)
47439 case 8: // result = lea base( , cond*8)
47440 case 9: // result = lea base(cond, cond*8)
47441 isFastMultiplier = true;
47442 break;
47443 }
47444 }
47445
47446 if (isFastMultiplier) {
47447 Cond = getSETCC(CC, Cond, DL ,DAG);
47448 // Zero extend the condition if needed.
47449 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
47450 Cond);
47451 // Scale the condition by the difference.
47452 if (Diff != 1)
47453 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
47454 DAG.getConstant(Diff, DL, Cond.getValueType()));
47455
47456 // Add the base if non-zero.
47457 if (FalseC->getAPIntValue() != 0)
47458 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
47459 SDValue(FalseC, 0));
47460 return Cond;
47461 }
47462 }
47463 }
47464 }
47465
47466 // Handle these cases:
47467 // (select (x != c), e, c) -> select (x != c), e, x),
47468 // (select (x == c), c, e) -> select (x == c), x, e)
47469 // where the c is an integer constant, and the "select" is the combination
47470 // of CMOV and CMP.
47471 //
47472 // The rationale for this change is that the conditional-move from a constant
47473 // needs two instructions, however, conditional-move from a register needs
47474 // only one instruction.
47475 //
47476 // CAVEAT: By replacing a constant with a symbolic value, it may obscure
47477 // some instruction-combining opportunities. This opt needs to be
47478 // postponed as late as possible.
47479 //
47480 if (!DCI.isBeforeLegalize() && !DCI.isBeforeLegalizeOps()) {
47481 // the DCI.xxxx conditions are provided to postpone the optimization as
47482 // late as possible.
47483
47484 ConstantSDNode *CmpAgainst = nullptr;
47485 if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
47486 (CmpAgainst = dyn_cast<ConstantSDNode>(Cond.getOperand(1))) &&
47487 !isa<ConstantSDNode>(Cond.getOperand(0))) {
47488
47489 if (CC == X86::COND_NE &&
47490 CmpAgainst == dyn_cast<ConstantSDNode>(FalseOp)) {
47491 CC = X86::GetOppositeBranchCondition(CC);
47492 std::swap(TrueOp, FalseOp);
47493 }
47494
47495 if (CC == X86::COND_E &&
47496 CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
47497 SDValue Ops[] = {FalseOp, Cond.getOperand(0),
47498 DAG.getTargetConstant(CC, DL, MVT::i8), Cond};
47499 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
47500 }
47501 }
47502 }
47503
47504 // Transform:
47505 //
47506 // (cmov 1 T (uge T 2))
47507 //
47508 // to:
47509 //
47510 // (adc T 0 (sub T 1))
47511 if (CC == X86::COND_AE && isOneConstant(FalseOp) &&
47512 Cond.getOpcode() == X86ISD::SUB && Cond->hasOneUse()) {
47513 SDValue Cond0 = Cond.getOperand(0);
47514 if (Cond0.getOpcode() == ISD::TRUNCATE)
47515 Cond0 = Cond0.getOperand(0);
47516 auto *Sub1C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
47517 if (Cond0 == TrueOp && Sub1C && Sub1C->getZExtValue() == 2) {
47518 EVT CondVT = Cond->getValueType(0);
47519 EVT OuterVT = N->getValueType(0);
47520 // Subtract 1 and generate a carry.
47521 SDValue NewSub =
47522 DAG.getNode(X86ISD::SUB, DL, Cond->getVTList(), Cond.getOperand(0),
47523 DAG.getConstant(1, DL, CondVT));
47524 SDValue EFLAGS(NewSub.getNode(), 1);
47525 return DAG.getNode(X86ISD::ADC, DL, DAG.getVTList(OuterVT, MVT::i32),
47526 TrueOp, DAG.getConstant(0, DL, OuterVT), EFLAGS);
47527 }
47528 }
47529
47530 // Fold and/or of setcc's to double CMOV:
47531 // (CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
47532 // (CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
47533 //
47534 // This combine lets us generate:
47535 // cmovcc1 (jcc1 if we don't have CMOV)
47536 // cmovcc2 (same)
47537 // instead of:
47538 // setcc1
47539 // setcc2
47540 // and/or
47541 // cmovne (jne if we don't have CMOV)
47542 // When we can't use the CMOV instruction, it might increase branch
47543 // mispredicts.
47544 // When we can use CMOV, or when there is no mispredict, this improves
47545 // throughput and reduces register pressure.
47546 //
47547 if (CC == X86::COND_NE) {
47548 SDValue Flags;
47549 X86::CondCode CC0, CC1;
47550 bool isAndSetCC;
47551 if (checkBoolTestAndOrSetCCCombine(Cond, CC0, CC1, Flags, isAndSetCC)) {
47552 if (isAndSetCC) {
47553 std::swap(FalseOp, TrueOp);
47554 CC0 = X86::GetOppositeBranchCondition(CC0);
47555 CC1 = X86::GetOppositeBranchCondition(CC1);
47556 }
47557
47558 SDValue LOps[] = {FalseOp, TrueOp,
47559 DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};
47560 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
47561 SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
47562 Flags};
47563 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
47564 return CMOV;
47565 }
47566 }
47567
47568 // Fold (CMOV C1, (ADD (CTTZ X), C2), (X != 0)) ->
47569 // (ADD (CMOV C1-C2, (CTTZ X), (X != 0)), C2)
47570 // Or (CMOV (ADD (CTTZ X), C2), C1, (X == 0)) ->
47571 // (ADD (CMOV (CTTZ X), C1-C2, (X == 0)), C2)
47572 if ((CC == X86::COND_NE || CC == X86::COND_E) &&
47573 Cond.getOpcode() == X86ISD::CMP && isNullConstant(Cond.getOperand(1))) {
47574 SDValue Add = TrueOp;
47575 SDValue Const = FalseOp;
47576 // Canonicalize the condition code for easier matching and output.
47577 if (CC == X86::COND_E)
47578 std::swap(Add, Const);
47579
47580 // We might have replaced the constant in the cmov with the LHS of the
47581 // compare. If so change it to the RHS of the compare.
47582 if (Const == Cond.getOperand(0))
47583 Const = Cond.getOperand(1);
47584
47585 // Ok, now make sure that Add is (add (cttz X), C2) and Const is a constant.
47586 if (isa<ConstantSDNode>(Const) && Add.getOpcode() == ISD::ADD &&
47587 Add.hasOneUse() && isa<ConstantSDNode>(Add.getOperand(1)) &&
47588 (Add.getOperand(0).getOpcode() == ISD::CTTZ_ZERO_UNDEF ||
47589 Add.getOperand(0).getOpcode() == ISD::CTTZ) &&
47590 Add.getOperand(0).getOperand(0) == Cond.getOperand(0)) {
47591 EVT VT = N->getValueType(0);
47592 // This should constant fold.
47593 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
47594 SDValue CMov =
47595 DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
47596 DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);
47597 return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1));
47598 }
47599 }
47600
47601 return SDValue();
47602}
47603
47604/// Different mul shrinking modes.
47605enum class ShrinkMode { MULS8, MULU8, MULS16, MULU16 };
47606
47607static bool canReduceVMulWidth(SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode) {
47608 EVT VT = N->getOperand(0).getValueType();
47609 if (VT.getScalarSizeInBits() != 32)
47610 return false;
47611
47612 assert(N->getNumOperands() == 2 && "NumOperands of Mul are 2");
47613 unsigned SignBits[2] = {1, 1};
47614 bool IsPositive[2] = {false, false};
47615 for (unsigned i = 0; i < 2; i++) {
47616 SDValue Opd = N->getOperand(i);
47617
47618 SignBits[i] = DAG.ComputeNumSignBits(Opd);
47619 IsPositive[i] = DAG.SignBitIsZero(Opd);
47620 }
47621
47622 bool AllPositive = IsPositive[0] && IsPositive[1];
47623 unsigned MinSignBits = std::min(SignBits[0], SignBits[1]);
47624 // When ranges are from -128 ~ 127, use MULS8 mode.
47625 if (MinSignBits >= 25)
47626 Mode = ShrinkMode::MULS8;
47627 // When ranges are from 0 ~ 255, use MULU8 mode.
47628 else if (AllPositive && MinSignBits >= 24)
47629 Mode = ShrinkMode::MULU8;
47630 // When ranges are from -32768 ~ 32767, use MULS16 mode.
47631 else if (MinSignBits >= 17)
47632 Mode = ShrinkMode::MULS16;
47633 // When ranges are from 0 ~ 65535, use MULU16 mode.
47634 else if (AllPositive && MinSignBits >= 16)
47635 Mode = ShrinkMode::MULU16;
47636 else
47637 return false;
47638 return true;
47639}
47640
47641/// When the operands of vector mul are extended from smaller size values,
47642/// like i8 and i16, the type of mul may be shrinked to generate more
47643/// efficient code. Two typical patterns are handled:
47644/// Pattern1:
47645/// %2 = sext/zext <N x i8> %1 to <N x i32>
47646/// %4 = sext/zext <N x i8> %3 to <N x i32>
47647// or %4 = build_vector <N x i32> %C1, ..., %CN (%C1..%CN are constants)
47648/// %5 = mul <N x i32> %2, %4
47649///
47650/// Pattern2:
47651/// %2 = zext/sext <N x i16> %1 to <N x i32>
47652/// %4 = zext/sext <N x i16> %3 to <N x i32>
47653/// or %4 = build_vector <N x i32> %C1, ..., %CN (%C1..%CN are constants)
47654/// %5 = mul <N x i32> %2, %4
47655///
47656/// There are four mul shrinking modes:
47657/// If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is
47658/// -128 to 128, and the scalar value range of %4 is also -128 to 128,
47659/// generate pmullw+sext32 for it (MULS8 mode).
47660/// If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is
47661/// 0 to 255, and the scalar value range of %4 is also 0 to 255,
47662/// generate pmullw+zext32 for it (MULU8 mode).
47663/// If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is
47664/// -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767,
47665/// generate pmullw+pmulhw for it (MULS16 mode).
47666/// If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is
47667/// 0 to 65535, and the scalar value range of %4 is also 0 to 65535,
47668/// generate pmullw+pmulhuw for it (MULU16 mode).
47669static SDValue reduceVMULWidth(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
47670 const X86Subtarget &Subtarget) {
47671 // Check for legality
47672 // pmullw/pmulhw are not supported by SSE.
47673 if (!Subtarget.hasSSE2())
47674 return SDValue();
47675
47676 // Check for profitability
47677 // pmulld is supported since SSE41. It is better to use pmulld
47678 // instead of pmullw+pmulhw, except for subtargets where pmulld is slower than
47679 // the expansion.
47680 bool OptForMinSize = DAG.getMachineFunction().getFunction().hasMinSize();
47681 if (Subtarget.hasSSE41() && (OptForMinSize || !Subtarget.isPMULLDSlow()))
47682 return SDValue();
47683
47684 ShrinkMode Mode;
47685 if (!canReduceVMulWidth(N, DAG, Mode))
47686 return SDValue();
47687
47688 SDValue N0 = N->getOperand(0);
47689 SDValue N1 = N->getOperand(1);
47690 EVT VT = N->getOperand(0).getValueType();
47691 unsigned NumElts = VT.getVectorNumElements();
47692 if ((NumElts % 2) != 0)
47693 return SDValue();
47694
47695 EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts);
47696
47697 // Shrink the operands of mul.
47698 SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
47699 SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
47700
47701 // Generate the lower part of mul: pmullw. For MULU8/MULS8, only the
47702 // lower part is needed.
47703 SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
47704 if (Mode == ShrinkMode::MULU8 || Mode == ShrinkMode::MULS8)
47705 return DAG.getNode((Mode == ShrinkMode::MULU8) ? ISD::ZERO_EXTEND
47706 : ISD::SIGN_EXTEND,
47707 DL, VT, MulLo);
47708
47709 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts / 2);
47710 // Generate the higher part of mul: pmulhw/pmulhuw. For MULU16/MULS16,
47711 // the higher part is also needed.
47712 SDValue MulHi =
47713 DAG.getNode(Mode == ShrinkMode::MULS16 ? ISD::MULHS : ISD::MULHU, DL,
47714 ReducedVT, NewN0, NewN1);
47715
47716 // Repack the lower part and higher part result of mul into a wider
47717 // result.
47718 // Generate shuffle functioning as punpcklwd.
47719 SmallVector<int, 16> ShuffleMask(NumElts);
47720 for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
47721 ShuffleMask[2 * i] = i;
47722 ShuffleMask[2 * i + 1] = i + NumElts;
47723 }
47724 SDValue ResLo =
47725 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
47726 ResLo = DAG.getBitcast(ResVT, ResLo);
47727 // Generate shuffle functioning as punpckhwd.
47728 for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
47729 ShuffleMask[2 * i] = i + NumElts / 2;
47730 ShuffleMask[2 * i + 1] = i + NumElts * 3 / 2;
47731 }
47732 SDValue ResHi =
47733 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
47734 ResHi = DAG.getBitcast(ResVT, ResHi);
47735 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
47736}
47737
47738static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
47739 EVT VT, const SDLoc &DL) {
47740
47741 auto combineMulShlAddOrSub = [&](int Mult, int Shift, bool isAdd) {
47742 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
47743 DAG.getConstant(Mult, DL, VT));
47744 Result = DAG.getNode(ISD::SHL, DL, VT, Result,
47745 DAG.getConstant(Shift, DL, MVT::i8));
47746 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
47747 N->getOperand(0));
47748 return Result;
47749 };
47750
47751 auto combineMulMulAddOrSub = [&](int Mul1, int Mul2, bool isAdd) {
47752 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
47753 DAG.getConstant(Mul1, DL, VT));
47754 Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result,
47755 DAG.getConstant(Mul2, DL, VT));
47756 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
47757 N->getOperand(0));
47758 return Result;
47759 };
47760
47761 switch (MulAmt) {
47762 default:
47763 break;
47764 case 11:
47765 // mul x, 11 => add ((shl (mul x, 5), 1), x)
47766 return combineMulShlAddOrSub(5, 1, /*isAdd*/ true);
47767 case 21:
47768 // mul x, 21 => add ((shl (mul x, 5), 2), x)
47769 return combineMulShlAddOrSub(5, 2, /*isAdd*/ true);
47770 case 41:
47771 // mul x, 41 => add ((shl (mul x, 5), 3), x)
47772 return combineMulShlAddOrSub(5, 3, /*isAdd*/ true);
47773 case 22:
47774 // mul x, 22 => add (add ((shl (mul x, 5), 2), x), x)
47775 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
47776 combineMulShlAddOrSub(5, 2, /*isAdd*/ true));
47777 case 19:
47778 // mul x, 19 => add ((shl (mul x, 9), 1), x)
47779 return combineMulShlAddOrSub(9, 1, /*isAdd*/ true);
47780 case 37:
47781 // mul x, 37 => add ((shl (mul x, 9), 2), x)
47782 return combineMulShlAddOrSub(9, 2, /*isAdd*/ true);
47783 case 73:
47784 // mul x, 73 => add ((shl (mul x, 9), 3), x)
47785 return combineMulShlAddOrSub(9, 3, /*isAdd*/ true);
47786 case 13:
47787 // mul x, 13 => add ((shl (mul x, 3), 2), x)
47788 return combineMulShlAddOrSub(3, 2, /*isAdd*/ true);
47789 case 23:
47790 // mul x, 23 => sub ((shl (mul x, 3), 3), x)
47791 return combineMulShlAddOrSub(3, 3, /*isAdd*/ false);
47792 case 26:
47793 // mul x, 26 => add ((mul (mul x, 5), 5), x)
47794 return combineMulMulAddOrSub(5, 5, /*isAdd*/ true);
47795 case 28:
47796 // mul x, 28 => add ((mul (mul x, 9), 3), x)
47797 return combineMulMulAddOrSub(9, 3, /*isAdd*/ true);
47798 case 29:
47799 // mul x, 29 => add (add ((mul (mul x, 9), 3), x), x)
47800 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
47801 combineMulMulAddOrSub(9, 3, /*isAdd*/ true));
47802 }
47803
47804 // Another trick. If this is a power 2 + 2/4/8, we can use a shift followed
47805 // by a single LEA.
47806 // First check if this a sum of two power of 2s because that's easy. Then
47807 // count how many zeros are up to the first bit.
47808 // TODO: We can do this even without LEA at a cost of two shifts and an add.
47809 if (isPowerOf2_64(MulAmt & (MulAmt - 1))) {
47810 unsigned ScaleShift = llvm::countr_zero(MulAmt);
47811 if (ScaleShift >= 1 && ScaleShift < 4) {
47812 unsigned ShiftAmt = Log2_64((MulAmt & (MulAmt - 1)));
47813 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
47814 DAG.getConstant(ShiftAmt, DL, MVT::i8));
47815 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
47816 DAG.getConstant(ScaleShift, DL, MVT::i8));
47817 return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2);
47818 }
47819 }
47820
47821 return SDValue();
47822}
47823
47824// If the upper 17 bits of either element are zero and the other element are
47825// zero/sign bits then we can use PMADDWD, which is always at least as quick as
47826// PMULLD, except on KNL.
47827static SDValue combineMulToPMADDWD(SDNode *N, const SDLoc &DL,
47828 SelectionDAG &DAG,
47829 const X86Subtarget &Subtarget) {
47830 if (!Subtarget.hasSSE2())
47831 return SDValue();
47832
47833 if (Subtarget.isPMADDWDSlow())
47834 return SDValue();
47835
47836 EVT VT = N->getValueType(0);
47837
47838 // Only support vXi32 vectors.
47839 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32)
47840 return SDValue();
47841
47842 // Make sure the type is legal or can split/widen to a legal type.
47843 // With AVX512 but without BWI, we would need to split v32i16.
47844 unsigned NumElts = VT.getVectorNumElements();
47845 if (NumElts == 1 || !isPowerOf2_32(NumElts))
47846 return SDValue();
47847
47848 // With AVX512 but without BWI, we would need to split v32i16.
47849 if (32 <= (2 * NumElts) && Subtarget.hasAVX512() && !Subtarget.hasBWI())
47850 return SDValue();
47851
47852 SDValue N0 = N->getOperand(0);
47853 SDValue N1 = N->getOperand(1);
47854
47855 // If we are zero/sign extending two steps without SSE4.1, its better to
47856 // reduce the vmul width instead.
47857 if (!Subtarget.hasSSE41() &&
47858 (((N0.getOpcode() == ISD::ZERO_EXTEND &&
47859 N0.getOperand(0).getScalarValueSizeInBits() <= 8) &&
47860 (N1.getOpcode() == ISD::ZERO_EXTEND &&
47861 N1.getOperand(0).getScalarValueSizeInBits() <= 8)) ||
47862 ((N0.getOpcode() == ISD::SIGN_EXTEND &&
47863 N0.getOperand(0).getScalarValueSizeInBits() <= 8) &&
47864 (N1.getOpcode() == ISD::SIGN_EXTEND &&
47865 N1.getOperand(0).getScalarValueSizeInBits() <= 8))))
47866 return SDValue();
47867
47868 // If we are sign extending a wide vector without SSE4.1, its better to reduce
47869 // the vmul width instead.
47870 if (!Subtarget.hasSSE41() &&
47871 (N0.getOpcode() == ISD::SIGN_EXTEND &&
47872 N0.getOperand(0).getValueSizeInBits() > 128) &&
47873 (N1.getOpcode() == ISD::SIGN_EXTEND &&
47874 N1.getOperand(0).getValueSizeInBits() > 128))
47875 return SDValue();
47876
47877 // Sign bits must extend down to the lowest i16.
47878 if (DAG.ComputeMaxSignificantBits(N1) > 16 ||
47879 DAG.ComputeMaxSignificantBits(N0) > 16)
47880 return SDValue();
47881
47882 // At least one of the elements must be zero in the upper 17 bits, or can be
47883 // safely made zero without altering the final result.
47884 auto GetZeroableOp = [&](SDValue Op) {
47885 APInt Mask17 = APInt::getHighBitsSet(32, 17);
47886 if (DAG.MaskedValueIsZero(Op, Mask17))
47887 return Op;
47888 // Mask off upper 16-bits of sign-extended constants.
47889 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()))
47890 return DAG.getNode(ISD::AND, DL, VT, Op, DAG.getConstant(0xFFFF, DL, VT));
47891 if (Op.getOpcode() == ISD::SIGN_EXTEND && N->isOnlyUserOf(Op.getNode())) {
47892 SDValue Src = Op.getOperand(0);
47893 // Convert sext(vXi16) to zext(vXi16).
47894 if (Src.getScalarValueSizeInBits() == 16 && VT.getSizeInBits() <= 128)
47895 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Src);
47896 // Convert sext(vXi8) to zext(vXi16 sext(vXi8)) on pre-SSE41 targets
47897 // which will expand the extension.
47898 if (Src.getScalarValueSizeInBits() < 16 && !Subtarget.hasSSE41()) {
47899 EVT ExtVT = VT.changeVectorElementType(MVT::i16);
47900 Src = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, Src);
47901 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Src);
47902 }
47903 }
47904 // Convert SIGN_EXTEND_VECTOR_INREG to ZEXT_EXTEND_VECTOR_INREG.
47905 if (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG &&
47906 N->isOnlyUserOf(Op.getNode())) {
47907 SDValue Src = Op.getOperand(0);
47908 if (Src.getScalarValueSizeInBits() == 16)
47909 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Src);
47910 }
47911 // Convert VSRAI(Op, 16) to VSRLI(Op, 16).
47912 if (Op.getOpcode() == X86ISD::VSRAI && Op.getConstantOperandVal(1) == 16 &&
47913 N->isOnlyUserOf(Op.getNode())) {
47914 return DAG.getNode(X86ISD::VSRLI, DL, VT, Op.getOperand(0),
47915 Op.getOperand(1));
47916 }
47917 return SDValue();
47918 };
47919 SDValue ZeroN0 = GetZeroableOp(N0);
47920 SDValue ZeroN1 = GetZeroableOp(N1);
47921 if (!ZeroN0 && !ZeroN1)
47922 return SDValue();
47923 N0 = ZeroN0 ? ZeroN0 : N0;
47924 N1 = ZeroN1 ? ZeroN1 : N1;
47925
47926 // Use SplitOpsAndApply to handle AVX splitting.
47927 auto PMADDWDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
47928 ArrayRef<SDValue> Ops) {
47929 MVT ResVT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
47930 MVT OpVT = MVT::getVectorVT(MVT::i16, Ops[0].getValueSizeInBits() / 16);
47931 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT,
47932 DAG.getBitcast(OpVT, Ops[0]),
47933 DAG.getBitcast(OpVT, Ops[1]));
47934 };
47935 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMADDWDBuilder);
47936}
47937
47938static SDValue combineMulToPMULDQ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
47939 const X86Subtarget &Subtarget) {
47940 if (!Subtarget.hasSSE2())
47941 return SDValue();
47942
47943 EVT VT = N->getValueType(0);
47944
47945 // Only support vXi64 vectors.
47946 if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 ||
47947 VT.getVectorNumElements() < 2 ||
47948 !isPowerOf2_32(VT.getVectorNumElements()))
47949 return SDValue();
47950
47951 SDValue N0 = N->getOperand(0);
47952 SDValue N1 = N->getOperand(1);
47953
47954 // MULDQ returns the 64-bit result of the signed multiplication of the lower
47955 // 32-bits. We can lower with this if the sign bits stretch that far.
47956 if (Subtarget.hasSSE41() && DAG.ComputeNumSignBits(N0) > 32 &&
47957 DAG.ComputeNumSignBits(N1) > 32) {
47958 auto PMULDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
47959 ArrayRef<SDValue> Ops) {
47960 return DAG.getNode(X86ISD::PMULDQ, DL, Ops[0].getValueType(), Ops);
47961 };
47962 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMULDQBuilder,
47963 /*CheckBWI*/ false);
47964 }
47965
47966 // If the upper bits are zero we can use a single pmuludq.
47967 APInt Mask = APInt::getHighBitsSet(64, 32);
47968 if (DAG.MaskedValueIsZero(N0, Mask) && DAG.MaskedValueIsZero(N1, Mask)) {
47969 auto PMULUDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
47970 ArrayRef<SDValue> Ops) {
47971 return DAG.getNode(X86ISD::PMULUDQ, DL, Ops[0].getValueType(), Ops);
47972 };
47973 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMULUDQBuilder,
47974 /*CheckBWI*/ false);
47975 }
47976
47977 return SDValue();
47978}
47979
47980static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
47981 TargetLowering::DAGCombinerInfo &DCI,
47982 const X86Subtarget &Subtarget) {
47983 EVT VT = N->getValueType(0);
47984 SDLoc DL(N);
47985
47986 if (SDValue V = combineMulToPMADDWD(N, DL, DAG, Subtarget))
47987 return V;
47988
47989 if (SDValue V = combineMulToPMULDQ(N, DL, DAG, Subtarget))
47990 return V;
47991
47992 if (DCI.isBeforeLegalize() && VT.isVector())
47993 return reduceVMULWidth(N, DL, DAG, Subtarget);
47994
47995 // Optimize a single multiply with constant into two operations in order to
47996 // implement it with two cheaper instructions, e.g. LEA + SHL, LEA + LEA.
47997 if (!MulConstantOptimization)
47998 return SDValue();
47999
48000 // An imul is usually smaller than the alternative sequence.
48001 if (DAG.getMachineFunction().getFunction().hasMinSize())
48002 return SDValue();
48003
48004 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
48005 return SDValue();
48006
48007 if (VT != MVT::i64 && VT != MVT::i32 &&
48008 (!VT.isVector() || !VT.isSimple() || !VT.isInteger()))
48009 return SDValue();
48010
48011 ConstantSDNode *CNode = isConstOrConstSplat(
48012 N->getOperand(1), /*AllowUndefs*/ true, /*AllowTrunc*/ false);
48013 const APInt *C = nullptr;
48014 if (!CNode) {
48015 if (VT.isVector())
48016 if (auto *RawC = getTargetConstantFromNode(N->getOperand(1)))
48017 if (auto *SplatC = RawC->getSplatValue())
48018 C = &(SplatC->getUniqueInteger());
48019
48020 if (!C || C->getBitWidth() != VT.getScalarSizeInBits())
48021 return SDValue();
48022 } else {
48023 C = &(CNode->getAPIntValue());
48024 }
48025
48026 if (isPowerOf2_64(C->getZExtValue()))
48027 return SDValue();
48028
48029 int64_t SignMulAmt = C->getSExtValue();
48030 assert(SignMulAmt != INT64_MIN && "Int min should have been handled!");
48031 uint64_t AbsMulAmt = SignMulAmt < 0 ? -SignMulAmt : SignMulAmt;
48032
48033 SDValue NewMul = SDValue();
48034 if (VT == MVT::i64 || VT == MVT::i32) {
48035 if (AbsMulAmt == 3 || AbsMulAmt == 5 || AbsMulAmt == 9) {
48036 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
48037 DAG.getConstant(AbsMulAmt, DL, VT));
48038 if (SignMulAmt < 0)
48039 NewMul = DAG.getNegative(NewMul, DL, VT);
48040
48041 return NewMul;
48042 }
48043
48044 uint64_t MulAmt1 = 0;
48045 uint64_t MulAmt2 = 0;
48046 if ((AbsMulAmt % 9) == 0) {
48047 MulAmt1 = 9;
48048 MulAmt2 = AbsMulAmt / 9;
48049 } else if ((AbsMulAmt % 5) == 0) {
48050 MulAmt1 = 5;
48051 MulAmt2 = AbsMulAmt / 5;
48052 } else if ((AbsMulAmt % 3) == 0) {
48053 MulAmt1 = 3;
48054 MulAmt2 = AbsMulAmt / 3;
48055 }
48056
48057 // For negative multiply amounts, only allow MulAmt2 to be a power of 2.
48058 if (MulAmt2 &&
48059 (isPowerOf2_64(MulAmt2) ||
48060 (SignMulAmt >= 0 && (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)))) {
48061
48062 if (isPowerOf2_64(MulAmt2) && !(SignMulAmt >= 0 && N->hasOneUse() &&
48063 N->use_begin()->getOpcode() == ISD::ADD))
48064 // If second multiplifer is pow2, issue it first. We want the multiply
48065 // by 3, 5, or 9 to be folded into the addressing mode unless the lone
48066 // use is an add. Only do this for positive multiply amounts since the
48067 // negate would prevent it from being used as an address mode anyway.
48068 std::swap(MulAmt1, MulAmt2);
48069
48070 if (isPowerOf2_64(MulAmt1))
48071 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48072 DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8));
48073 else
48074 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
48075 DAG.getConstant(MulAmt1, DL, VT));
48076
48077 if (isPowerOf2_64(MulAmt2))
48078 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
48079 DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8));
48080 else
48081 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
48082 DAG.getConstant(MulAmt2, DL, VT));
48083
48084 // Negate the result.
48085 if (SignMulAmt < 0)
48086 NewMul = DAG.getNegative(NewMul, DL, VT);
48087 } else if (!Subtarget.slowLEA())
48088 NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL);
48089 }
48090 if (!NewMul) {
48091 EVT ShiftVT = VT.isVector() ? VT : MVT::i8;
48092 assert(C->getZExtValue() != 0 &&
48093 C->getZExtValue() != maxUIntN(VT.getScalarSizeInBits()) &&
48094 "Both cases that could cause potential overflows should have "
48095 "already been handled.");
48096 if (isPowerOf2_64(AbsMulAmt - 1)) {
48097 // (mul x, 2^N + 1) => (add (shl x, N), x)
48098 NewMul = DAG.getNode(
48099 ISD::ADD, DL, VT, N->getOperand(0),
48100 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48101 DAG.getConstant(Log2_64(AbsMulAmt - 1), DL, ShiftVT)));
48102 if (SignMulAmt < 0)
48103 NewMul = DAG.getNegative(NewMul, DL, VT);
48104 } else if (isPowerOf2_64(AbsMulAmt + 1)) {
48105 // (mul x, 2^N - 1) => (sub (shl x, N), x)
48106 NewMul =
48107 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48108 DAG.getConstant(Log2_64(AbsMulAmt + 1), DL, ShiftVT));
48109 // To negate, reverse the operands of the subtract.
48110 if (SignMulAmt < 0)
48111 NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul);
48112 else
48113 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
48114 } else if (SignMulAmt >= 0 && isPowerOf2_64(AbsMulAmt - 2) &&
48115 (!VT.isVector() || Subtarget.fastImmVectorShift())) {
48116 // (mul x, 2^N + 2) => (add (shl x, N), (add x, x))
48117 NewMul =
48118 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48119 DAG.getConstant(Log2_64(AbsMulAmt - 2), DL, ShiftVT));
48120 NewMul = DAG.getNode(
48121 ISD::ADD, DL, VT, NewMul,
48122 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0)));
48123 } else if (SignMulAmt >= 0 && isPowerOf2_64(AbsMulAmt + 2) &&
48124 (!VT.isVector() || Subtarget.fastImmVectorShift())) {
48125 // (mul x, 2^N - 2) => (sub (shl x, N), (add x, x))
48126 NewMul =
48127 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48128 DAG.getConstant(Log2_64(AbsMulAmt + 2), DL, ShiftVT));
48129 NewMul = DAG.getNode(
48130 ISD::SUB, DL, VT, NewMul,
48131 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0)));
48132 } else if (SignMulAmt >= 0 && VT.isVector() &&
48133 Subtarget.fastImmVectorShift()) {
48134 uint64_t AbsMulAmtLowBit = AbsMulAmt & (-AbsMulAmt);
48135 uint64_t ShiftAmt1;
48136 std::optional<unsigned> Opc;
48137 if (isPowerOf2_64(AbsMulAmt - AbsMulAmtLowBit)) {
48138 ShiftAmt1 = AbsMulAmt - AbsMulAmtLowBit;
48139 Opc = ISD::ADD;
48140 } else if (isPowerOf2_64(AbsMulAmt + AbsMulAmtLowBit)) {
48141 ShiftAmt1 = AbsMulAmt + AbsMulAmtLowBit;
48142 Opc = ISD::SUB;
48143 }
48144
48145 if (Opc) {
48146 SDValue Shift1 =
48147 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48148 DAG.getConstant(Log2_64(ShiftAmt1), DL, ShiftVT));
48149 SDValue Shift2 =
48150 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
48151 DAG.getConstant(Log2_64(AbsMulAmtLowBit), DL, ShiftVT));
48152 NewMul = DAG.getNode(*Opc, DL, VT, Shift1, Shift2);
48153 }
48154 }
48155 }
48156
48157 return NewMul;
48158}
48159
48160// Try to form a MULHU or MULHS node by looking for
48161// (srl (mul ext, ext), 16)
48162// TODO: This is X86 specific because we want to be able to handle wide types
48163// before type legalization. But we can only do it if the vector will be
48164// legalized via widening/splitting. Type legalization can't handle promotion
48165// of a MULHU/MULHS. There isn't a way to convey this to the generic DAG
48166// combiner.
48167static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG,
48168 const SDLoc &DL,
48169 const X86Subtarget &Subtarget) {
48170 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
48171 "SRL or SRA node is required here!");
48172
48173 if (!Subtarget.hasSSE2())
48174 return SDValue();
48175
48176 // The operation feeding into the shift must be a multiply.
48177 SDValue ShiftOperand = N->getOperand(0);
48178 if (ShiftOperand.getOpcode() != ISD::MUL || !ShiftOperand.hasOneUse())
48179 return SDValue();
48180
48181 // Input type should be at least vXi32.
48182 EVT VT = N->getValueType(0);
48183 if (!VT.isVector() || VT.getVectorElementType().getSizeInBits() < 32)
48184 return SDValue();
48185
48186 // Need a shift by 16.
48187 APInt ShiftAmt;
48188 if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), ShiftAmt) ||
48189 ShiftAmt != 16)
48190 return SDValue();
48191
48192 SDValue LHS = ShiftOperand.getOperand(0);
48193 SDValue RHS = ShiftOperand.getOperand(1);
48194
48195 unsigned ExtOpc = LHS.getOpcode();
48196 if ((ExtOpc != ISD::SIGN_EXTEND && ExtOpc != ISD::ZERO_EXTEND) ||
48197 RHS.getOpcode() != ExtOpc)
48198 return SDValue();
48199
48200 // Peek through the extends.
48201 LHS = LHS.getOperand(0);
48202 RHS = RHS.getOperand(0);
48203
48204 // Ensure the input types match.
48205 EVT MulVT = LHS.getValueType();
48206 if (MulVT.getVectorElementType() != MVT::i16 || RHS.getValueType() != MulVT)
48207 return SDValue();
48208
48209 unsigned Opc = ExtOpc == ISD::SIGN_EXTEND ? ISD::MULHS : ISD::MULHU;
48210 SDValue Mulh = DAG.getNode(Opc, DL, MulVT, LHS, RHS);
48211
48212 ExtOpc = N->getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
48213 return DAG.getNode(ExtOpc, DL, VT, Mulh);
48214}
48215
48216static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG,
48217 const X86Subtarget &Subtarget) {
48218 using namespace llvm::SDPatternMatch;
48219 SDValue N0 = N->getOperand(0);
48220 SDValue N1 = N->getOperand(1);
48221 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
48222 EVT VT = N0.getValueType();
48223 unsigned EltSizeInBits = VT.getScalarSizeInBits();
48224 SDLoc DL(N);
48225
48226 // Exploits AVX2 VSHLV/VSRLV instructions for efficient unsigned vector shifts
48227 // with out-of-bounds clamping.
48228 if (N0.getOpcode() == ISD::VSELECT &&
48229 supportedVectorVarShift(VT, Subtarget, ISD::SHL)) {
48230 SDValue Cond = N0.getOperand(0);
48231 SDValue N00 = N0.getOperand(1);
48232 SDValue N01 = N0.getOperand(2);
48233 // fold shl(select(icmp_ult(amt,BW),x,0),amt) -> avx2 psllv(x,amt)
48234 if (ISD::isConstantSplatVectorAllZeros(N01.getNode()) &&
48235 sd_match(Cond, m_SetCC(m_Specific(N1), m_SpecificInt(EltSizeInBits),
48236 m_SpecificCondCode(ISD::SETULT)))) {
48237 return DAG.getNode(X86ISD::VSHLV, DL, VT, N00, N1);
48238 }
48239 // fold shl(select(icmp_uge(amt,BW),0,x),amt) -> avx2 psllv(x,amt)
48240 if (ISD::isConstantSplatVectorAllZeros(N00.getNode()) &&
48241 sd_match(Cond, m_SetCC(m_Specific(N1), m_SpecificInt(EltSizeInBits),
48242 m_SpecificCondCode(ISD::SETUGE)))) {
48243 return DAG.getNode(X86ISD::VSHLV, DL, VT, N01, N1);
48244 }
48245 }
48246
48247 // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
48248 // since the result of setcc_c is all zero's or all ones.
48249 if (VT.isInteger() && !VT.isVector() &&
48250 N1C && N0.getOpcode() == ISD::AND &&
48251 N0.getOperand(1).getOpcode() == ISD::Constant) {
48252 SDValue N00 = N0.getOperand(0);
48253 APInt Mask = N0.getConstantOperandAPInt(1);
48254 Mask <<= N1C->getAPIntValue();
48255 bool MaskOK = false;
48256 // We can handle cases concerning bit-widening nodes containing setcc_c if
48257 // we carefully interrogate the mask to make sure we are semantics
48258 // preserving.
48259 // The transform is not safe if the result of C1 << C2 exceeds the bitwidth
48260 // of the underlying setcc_c operation if the setcc_c was zero extended.
48261 // Consider the following example:
48262 // zext(setcc_c) -> i32 0x0000FFFF
48263 // c1 -> i32 0x0000FFFF
48264 // c2 -> i32 0x00000001
48265 // (shl (and (setcc_c), c1), c2) -> i32 0x0001FFFE
48266 // (and setcc_c, (c1 << c2)) -> i32 0x0000FFFE
48267 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
48268 MaskOK = true;
48269 } else if (N00.getOpcode() == ISD::SIGN_EXTEND &&
48270 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
48271 MaskOK = true;
48272 } else if ((N00.getOpcode() == ISD::ZERO_EXTEND ||
48273 N00.getOpcode() == ISD::ANY_EXTEND) &&
48274 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
48275 MaskOK = Mask.isIntN(N00.getOperand(0).getValueSizeInBits());
48276 }
48277 if (MaskOK && Mask != 0)
48278 return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT));
48279 }
48280
48281 return SDValue();
48282}
48283
48284static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
48285 const X86Subtarget &Subtarget) {
48286 using namespace llvm::SDPatternMatch;
48287 SDValue N0 = N->getOperand(0);
48288 SDValue N1 = N->getOperand(1);
48289 EVT VT = N0.getValueType();
48290 unsigned Size = VT.getSizeInBits();
48291 SDLoc DL(N);
48292
48293 if (SDValue V = combineShiftToPMULH(N, DAG, DL, Subtarget))
48294 return V;
48295
48296 // fold sra(x,umin(amt,bw-1)) -> avx2 psrav(x,amt)
48297 if (supportedVectorVarShift(VT, Subtarget, ISD::SRA)) {
48298 SDValue ShrAmtVal;
48299 if (sd_match(N1, m_UMin(m_Value(ShrAmtVal),
48300 m_SpecificInt(VT.getScalarSizeInBits() - 1))))
48301 return DAG.getNode(X86ISD::VSRAV, DL, VT, N0, ShrAmtVal);
48302 }
48303
48304 // fold (SRA (SHL X, ShlConst), SraConst)
48305 // into (SHL (sext_in_reg X), ShlConst - SraConst)
48306 // or (sext_in_reg X)
48307 // or (SRA (sext_in_reg X), SraConst - ShlConst)
48308 // depending on relation between SraConst and ShlConst.
48309 // We only do this if (Size - ShlConst) is equal to 8, 16 or 32. That allows
48310 // us to do the sext_in_reg from corresponding bit.
48311
48312 // sexts in X86 are MOVs. The MOVs have the same code size
48313 // as above SHIFTs (only SHIFT on 1 has lower code size).
48314 // However the MOVs have 2 advantages to a SHIFT:
48315 // 1. MOVs can write to a register that differs from source
48316 // 2. MOVs accept memory operands
48317
48318 if (VT.isVector() || N1.getOpcode() != ISD::Constant ||
48319 N0.getOpcode() != ISD::SHL || !N0.hasOneUse() ||
48320 N0.getOperand(1).getOpcode() != ISD::Constant)
48321 return SDValue();
48322
48323 SDValue N00 = N0.getOperand(0);
48324 SDValue N01 = N0.getOperand(1);
48325 APInt ShlConst = N01->getAsAPIntVal();
48326 APInt SraConst = N1->getAsAPIntVal();
48327 EVT CVT = N1.getValueType();
48328
48329 if (CVT != N01.getValueType())
48330 return SDValue();
48331 if (SraConst.isNegative())
48332 return SDValue();
48333
48334 for (MVT SVT : { MVT::i8, MVT::i16, MVT::i32 }) {
48335 unsigned ShiftSize = SVT.getSizeInBits();
48336 // Only deal with (Size - ShlConst) being equal to 8, 16 or 32.
48337 if (ShiftSize >= Size || ShlConst != Size - ShiftSize)
48338 continue;
48339 SDValue NN =
48340 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT));
48341 if (SraConst.eq(ShlConst))
48342 return NN;
48343 if (SraConst.ult(ShlConst))
48344 return DAG.getNode(ISD::SHL, DL, VT, NN,
48345 DAG.getConstant(ShlConst - SraConst, DL, CVT));
48346 return DAG.getNode(ISD::SRA, DL, VT, NN,
48347 DAG.getConstant(SraConst - ShlConst, DL, CVT));
48348 }
48349 return SDValue();
48350}
48351
48352static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG,
48353 TargetLowering::DAGCombinerInfo &DCI,
48354 const X86Subtarget &Subtarget) {
48355 using namespace llvm::SDPatternMatch;
48356 SDValue N0 = N->getOperand(0);
48357 SDValue N1 = N->getOperand(1);
48358 EVT VT = N0.getValueType();
48359 unsigned EltSizeInBits = VT.getScalarSizeInBits();
48360 SDLoc DL(N);
48361
48362 if (SDValue V = combineShiftToPMULH(N, DAG, DL, Subtarget))
48363 return V;
48364
48365 // Exploits AVX2 VSHLV/VSRLV instructions for efficient unsigned vector shifts
48366 // with out-of-bounds clamping.
48367 if (N0.getOpcode() == ISD::VSELECT &&
48368 supportedVectorVarShift(VT, Subtarget, ISD::SRL)) {
48369 SDValue Cond = N0.getOperand(0);
48370 SDValue N00 = N0.getOperand(1);
48371 SDValue N01 = N0.getOperand(2);
48372 // fold srl(select(icmp_ult(amt,BW),x,0),amt) -> avx2 psrlv(x,amt)
48373 if (ISD::isConstantSplatVectorAllZeros(N01.getNode()) &&
48374 sd_match(Cond, m_SetCC(m_Specific(N1), m_SpecificInt(EltSizeInBits),
48375 m_SpecificCondCode(ISD::SETULT)))) {
48376 return DAG.getNode(X86ISD::VSRLV, DL, VT, N00, N1);
48377 }
48378 // fold srl(select(icmp_uge(amt,BW),0,x),amt) -> avx2 psrlv(x,amt)
48379 if (ISD::isConstantSplatVectorAllZeros(N00.getNode()) &&
48380 sd_match(Cond, m_SetCC(m_Specific(N1), m_SpecificInt(EltSizeInBits),
48381 m_SpecificCondCode(ISD::SETUGE)))) {
48382 return DAG.getNode(X86ISD::VSRLV, DL, VT, N01, N1);
48383 }
48384 }
48385
48386 // Only do this on the last DAG combine as it can interfere with other
48387 // combines.
48388 if (!DCI.isAfterLegalizeDAG())
48389 return SDValue();
48390
48391 // Try to improve a sequence of srl (and X, C1), C2 by inverting the order.
48392 // TODO: This is a generic DAG combine that became an x86-only combine to
48393 // avoid shortcomings in other folds such as bswap, bit-test ('bt'), and
48394 // and-not ('andn').
48395 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
48396 return SDValue();
48397
48398 auto *ShiftC = dyn_cast<ConstantSDNode>(N1);
48399 auto *AndC = dyn_cast<ConstantSDNode>(N0.getOperand(1));
48400 if (!ShiftC || !AndC)
48401 return SDValue();
48402
48403 // If we can shrink the constant mask below 8-bits or 32-bits, then this
48404 // transform should reduce code size. It may also enable secondary transforms
48405 // from improved known-bits analysis or instruction selection.
48406 APInt MaskVal = AndC->getAPIntValue();
48407
48408 // If this can be matched by a zero extend, don't optimize.
48409 if (MaskVal.isMask()) {
48410 unsigned TO = MaskVal.countr_one();
48411 if (TO >= 8 && isPowerOf2_32(TO))
48412 return SDValue();
48413 }
48414
48415 APInt NewMaskVal = MaskVal.lshr(ShiftC->getAPIntValue());
48416 unsigned OldMaskSize = MaskVal.getSignificantBits();
48417 unsigned NewMaskSize = NewMaskVal.getSignificantBits();
48418 if ((OldMaskSize > 8 && NewMaskSize <= 8) ||
48419 (OldMaskSize > 32 && NewMaskSize <= 32)) {
48420 // srl (and X, AndC), ShiftC --> and (srl X, ShiftC), (AndC >> ShiftC)
48421 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);
48422 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
48423 return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask);
48424 }
48425 return SDValue();
48426}
48427
48428static SDValue combineHorizOpWithShuffle(SDNode *N, SelectionDAG &DAG,
48429 const X86Subtarget &Subtarget) {
48430 unsigned Opcode = N->getOpcode();
48431 assert(isHorizOp(Opcode) && "Unexpected hadd/hsub/pack opcode");
48432
48433 SDLoc DL(N);
48434 EVT VT = N->getValueType(0);
48435 SDValue N0 = N->getOperand(0);
48436 SDValue N1 = N->getOperand(1);
48437 EVT SrcVT = N0.getValueType();
48438
48439 SDValue BC0 =
48440 N->isOnlyUserOf(N0.getNode()) ? peekThroughOneUseBitcasts(N0) : N0;
48441 SDValue BC1 =
48442 N->isOnlyUserOf(N1.getNode()) ? peekThroughOneUseBitcasts(N1) : N1;
48443
48444 // Attempt to fold HOP(LOSUBVECTOR(SHUFFLE(X)),HISUBVECTOR(SHUFFLE(X)))
48445 // to SHUFFLE(HOP(LOSUBVECTOR(X),HISUBVECTOR(X))), this is mainly for
48446 // truncation trees that help us avoid lane crossing shuffles.
48447 // TODO: There's a lot more we can do for PACK/HADD style shuffle combines.
48448 // TODO: We don't handle vXf64 shuffles yet.
48449 if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) {
48450 if (SDValue BCSrc = getSplitVectorSrc(BC0, BC1, false)) {
48451 SmallVector<SDValue> ShuffleOps;
48452 SmallVector<int> ShuffleMask, ScaledMask;
48453 SDValue Vec = peekThroughBitcasts(BCSrc);
48454 if (getTargetShuffleInputs(Vec, ShuffleOps, ShuffleMask, DAG)) {
48455 resolveTargetShuffleInputsAndMask(ShuffleOps, ShuffleMask);
48456 // To keep the HOP LHS/RHS coherency, we must be able to scale the unary
48457 // shuffle to a v4X64 width - we can probably relax this in the future.
48458 if (!isAnyZero(ShuffleMask) && ShuffleOps.size() == 1 &&
48459 ShuffleOps[0].getValueType().is256BitVector() &&
48460 scaleShuffleElements(ShuffleMask, 4, ScaledMask)) {
48461 SDValue Lo, Hi;
48462 MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32;
48463 std::tie(Lo, Hi) = DAG.SplitVector(ShuffleOps[0], DL);
48464 Lo = DAG.getBitcast(SrcVT, Lo);
48465 Hi = DAG.getBitcast(SrcVT, Hi);
48466 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi);
48467 Res = DAG.getBitcast(ShufVT, Res);
48468 Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ScaledMask);
48469 return DAG.getBitcast(VT, Res);
48470 }
48471 }
48472 }
48473 }
48474
48475 // Attempt to fold HOP(SHUFFLE(X,Y),SHUFFLE(Z,W)) -> SHUFFLE(HOP()).
48476 if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) {
48477 // If either/both ops are a shuffle that can scale to v2x64,
48478 // then see if we can perform this as a v4x32 post shuffle.
48479 SmallVector<SDValue> Ops0, Ops1;
48480 SmallVector<int> Mask0, Mask1, ScaledMask0, ScaledMask1;
48481 bool IsShuf0 =
48482 getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) &&
48483 scaleShuffleElements(Mask0, 2, ScaledMask0) &&
48484 all_of(Ops0, [](SDValue Op) { return Op.getValueSizeInBits() == 128; });
48485 bool IsShuf1 =
48486 getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) &&
48487 scaleShuffleElements(Mask1, 2, ScaledMask1) &&
48488 all_of(Ops1, [](SDValue Op) { return Op.getValueSizeInBits() == 128; });
48489 if (IsShuf0 || IsShuf1) {
48490 if (!IsShuf0) {
48491 Ops0.assign({BC0});
48492 ScaledMask0.assign({0, 1});
48493 }
48494 if (!IsShuf1) {
48495 Ops1.assign({BC1});
48496 ScaledMask1.assign({0, 1});
48497 }
48498
48499 SDValue LHS, RHS;
48500 int PostShuffle[4] = {-1, -1, -1, -1};
48501 auto FindShuffleOpAndIdx = [&](int M, int &Idx, ArrayRef<SDValue> Ops) {
48502 if (M < 0)
48503 return true;
48504 Idx = M % 2;
48505 SDValue Src = Ops[M / 2];
48506 if (!LHS || LHS == Src) {
48507 LHS = Src;
48508 return true;
48509 }
48510 if (!RHS || RHS == Src) {
48511 Idx += 2;
48512 RHS = Src;
48513 return true;
48514 }
48515 return false;
48516 };
48517 if (FindShuffleOpAndIdx(ScaledMask0[0], PostShuffle[0], Ops0) &&
48518 FindShuffleOpAndIdx(ScaledMask0[1], PostShuffle[1], Ops0) &&
48519 FindShuffleOpAndIdx(ScaledMask1[0], PostShuffle[2], Ops1) &&
48520 FindShuffleOpAndIdx(ScaledMask1[1], PostShuffle[3], Ops1)) {
48521 LHS = DAG.getBitcast(SrcVT, LHS);
48522 RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS);
48523 MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32;
48524 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS);
48525 Res = DAG.getBitcast(ShufVT, Res);
48526 Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, PostShuffle);
48527 return DAG.getBitcast(VT, Res);
48528 }
48529 }
48530 }
48531
48532 // Attempt to fold HOP(SHUFFLE(X,Y),SHUFFLE(X,Y)) -> SHUFFLE(HOP(X,Y)).
48533 if (VT.is256BitVector() && Subtarget.hasInt256()) {
48534 SmallVector<int> Mask0, Mask1;
48535 SmallVector<SDValue> Ops0, Ops1;
48536 SmallVector<int, 2> ScaledMask0, ScaledMask1;
48537 if (getTargetShuffleInputs(BC0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) &&
48538 getTargetShuffleInputs(BC1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) &&
48539 !Ops0.empty() && !Ops1.empty() &&
48540 all_of(Ops0,
48541 [](SDValue Op) { return Op.getValueType().is256BitVector(); }) &&
48542 all_of(Ops1,
48543 [](SDValue Op) { return Op.getValueType().is256BitVector(); }) &&
48544 scaleShuffleElements(Mask0, 2, ScaledMask0) &&
48545 scaleShuffleElements(Mask1, 2, ScaledMask1)) {
48546 SDValue Op00 = peekThroughBitcasts(Ops0.front());
48547 SDValue Op10 = peekThroughBitcasts(Ops1.front());
48548 SDValue Op01 = peekThroughBitcasts(Ops0.back());
48549 SDValue Op11 = peekThroughBitcasts(Ops1.back());
48550 if ((Op00 == Op11) && (Op01 == Op10)) {
48551 std::swap(Op10, Op11);
48552 ShuffleVectorSDNode::commuteMask(ScaledMask1);
48553 }
48554 if ((Op00 == Op10) && (Op01 == Op11)) {
48555 const int Map[4] = {0, 2, 1, 3};
48556 SmallVector<int, 4> ShuffleMask(
48557 {Map[ScaledMask0[0]], Map[ScaledMask1[0]], Map[ScaledMask0[1]],
48558 Map[ScaledMask1[1]]});
48559 MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;
48560 SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00),
48561 DAG.getBitcast(SrcVT, Op01));
48562 Res = DAG.getBitcast(ShufVT, Res);
48563 Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ShuffleMask);
48564 return DAG.getBitcast(VT, Res);
48565 }
48566 }
48567 }
48568
48569 return SDValue();
48570}
48571
48572static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
48573 TargetLowering::DAGCombinerInfo &DCI,
48574 const X86Subtarget &Subtarget) {
48575 unsigned Opcode = N->getOpcode();
48576 assert((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) &&
48577 "Unexpected pack opcode");
48578
48579 EVT VT = N->getValueType(0);
48580 SDValue N0 = N->getOperand(0);
48581 SDValue N1 = N->getOperand(1);
48582 unsigned NumDstElts = VT.getVectorNumElements();
48583 unsigned DstBitsPerElt = VT.getScalarSizeInBits();
48584 unsigned SrcBitsPerElt = 2 * DstBitsPerElt;
48585 assert(N0.getScalarValueSizeInBits() == SrcBitsPerElt &&
48586 N1.getScalarValueSizeInBits() == SrcBitsPerElt &&
48587 "Unexpected PACKSS/PACKUS input type");
48588
48589 bool IsSigned = (X86ISD::PACKSS == Opcode);
48590
48591 // Constant Folding.
48592 APInt UndefElts0, UndefElts1;
48593 SmallVector<APInt, 32> EltBits0, EltBits1;
48594 if ((N0.isUndef() || N->isOnlyUserOf(N0.getNode())) &&
48595 (N1.isUndef() || N->isOnlyUserOf(N1.getNode())) &&
48596 getTargetConstantBitsFromNode(N0, SrcBitsPerElt, UndefElts0, EltBits0,
48597 /*AllowWholeUndefs*/ true,
48598 /*AllowPartialUndefs*/ true) &&
48599 getTargetConstantBitsFromNode(N1, SrcBitsPerElt, UndefElts1, EltBits1,
48600 /*AllowWholeUndefs*/ true,
48601 /*AllowPartialUndefs*/ true)) {
48602 unsigned NumLanes = VT.getSizeInBits() / 128;
48603 unsigned NumSrcElts = NumDstElts / 2;
48604 unsigned NumDstEltsPerLane = NumDstElts / NumLanes;
48605 unsigned NumSrcEltsPerLane = NumSrcElts / NumLanes;
48606
48607 APInt Undefs(NumDstElts, 0);
48608 SmallVector<APInt, 32> Bits(NumDstElts, APInt::getZero(DstBitsPerElt));
48609 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
48610 for (unsigned Elt = 0; Elt != NumDstEltsPerLane; ++Elt) {
48611 unsigned SrcIdx = Lane * NumSrcEltsPerLane + Elt % NumSrcEltsPerLane;
48612 auto &UndefElts = (Elt >= NumSrcEltsPerLane ? UndefElts1 : UndefElts0);
48613 auto &EltBits = (Elt >= NumSrcEltsPerLane ? EltBits1 : EltBits0);
48614
48615 if (UndefElts[SrcIdx]) {
48616 Undefs.setBit(Lane * NumDstEltsPerLane + Elt);
48617 continue;
48618 }
48619
48620 APInt &Val = EltBits[SrcIdx];
48621 if (IsSigned) {
48622 // PACKSS: Truncate signed value with signed saturation.
48623 // Source values less than dst minint are saturated to minint.
48624 // Source values greater than dst maxint are saturated to maxint.
48625 Val = Val.truncSSat(DstBitsPerElt);
48626 } else {
48627 // PACKUS: Truncate signed value with unsigned saturation.
48628 // Source values less than zero are saturated to zero.
48629 // Source values greater than dst maxuint are saturated to maxuint.
48630 // NOTE: This is different from APInt::truncUSat.
48631 if (Val.isIntN(DstBitsPerElt))
48632 Val = Val.trunc(DstBitsPerElt);
48633 else if (Val.isNegative())
48634 Val = APInt::getZero(DstBitsPerElt);
48635 else
48636 Val = APInt::getAllOnes(DstBitsPerElt);
48637 }
48638 Bits[Lane * NumDstEltsPerLane + Elt] = Val;
48639 }
48640 }
48641
48642 return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N));
48643 }
48644
48645 // Try to fold PACK(SHUFFLE(),SHUFFLE()) -> SHUFFLE(PACK()).
48646 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
48647 return V;
48648
48649 // Try to fold PACKSS(NOT(X),NOT(Y)) -> NOT(PACKSS(X,Y)).
48650 // Currently limit this to allsignbits cases only.
48651 if (IsSigned &&
48652 (N0.isUndef() || DAG.ComputeNumSignBits(N0) == SrcBitsPerElt) &&
48653 (N1.isUndef() || DAG.ComputeNumSignBits(N1) == SrcBitsPerElt)) {
48654 SDValue Not0 = N0.isUndef() ? N0 : IsNOT(N0, DAG);
48655 SDValue Not1 = N1.isUndef() ? N1 : IsNOT(N1, DAG);
48656 if (Not0 && Not1) {
48657 SDLoc DL(N);
48658 MVT SrcVT = N0.getSimpleValueType();
48659 SDValue Pack =
48660 DAG.getNode(X86ISD::PACKSS, DL, VT, DAG.getBitcast(SrcVT, Not0),
48661 DAG.getBitcast(SrcVT, Not1));
48662 return DAG.getNOT(DL, Pack, VT);
48663 }
48664 }
48665
48666 // Try to combine a PACKUSWB/PACKSSWB implemented truncate with a regular
48667 // truncate to create a larger truncate.
48668 if (Subtarget.hasAVX512() &&
48669 N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 &&
48670 N0.getOperand(0).getValueType() == MVT::v8i32) {
48671 if ((IsSigned && DAG.ComputeNumSignBits(N0) > 8) ||
48672 (!IsSigned &&
48673 DAG.MaskedValueIsZero(N0, APInt::getHighBitsSet(16, 8)))) {
48674 if (Subtarget.hasVLX())
48675 return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0));
48676
48677 // Widen input to v16i32 so we can truncate that.
48678 SDLoc dl(N);
48679 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32,
48680 N0.getOperand(0), DAG.getUNDEF(MVT::v8i32));
48681 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat);
48682 }
48683 }
48684
48685 // Try to fold PACK(EXTEND(X),EXTEND(Y)) -> CONCAT(X,Y) subvectors.
48686 if (VT.is128BitVector()) {
48687 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
48688 SDValue Src0, Src1;
48689 if (N0.getOpcode() == ExtOpc &&
48690 N0.getOperand(0).getValueType().is64BitVector() &&
48691 N0.getOperand(0).getScalarValueSizeInBits() == DstBitsPerElt) {
48692 Src0 = N0.getOperand(0);
48693 }
48694 if (N1.getOpcode() == ExtOpc &&
48695 N1.getOperand(0).getValueType().is64BitVector() &&
48696 N1.getOperand(0).getScalarValueSizeInBits() == DstBitsPerElt) {
48697 Src1 = N1.getOperand(0);
48698 }
48699 if ((Src0 || N0.isUndef()) && (Src1 || N1.isUndef())) {
48700 assert((Src0 || Src1) && "Found PACK(UNDEF,UNDEF)");
48701 Src0 = Src0 ? Src0 : DAG.getUNDEF(Src1.getValueType());
48702 Src1 = Src1 ? Src1 : DAG.getUNDEF(Src0.getValueType());
48703 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Src0, Src1);
48704 }
48705
48706 // Try again with pack(*_extend_vector_inreg, undef).
48707 unsigned VecInRegOpc = IsSigned ? ISD::SIGN_EXTEND_VECTOR_INREG
48708 : ISD::ZERO_EXTEND_VECTOR_INREG;
48709 if (N0.getOpcode() == VecInRegOpc && N1.isUndef() &&
48710 N0.getOperand(0).getScalarValueSizeInBits() < DstBitsPerElt)
48711 return getEXTEND_VECTOR_INREG(ExtOpc, SDLoc(N), VT, N0.getOperand(0),
48712 DAG);
48713 }
48714
48715 // Attempt to combine as shuffle.
48716 SDValue Op(N, 0);
48717 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
48718 return Res;
48719
48720 return SDValue();
48721}
48722
48723static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG,
48724 TargetLowering::DAGCombinerInfo &DCI,
48725 const X86Subtarget &Subtarget) {
48726 assert((X86ISD::HADD == N->getOpcode() || X86ISD::FHADD == N->getOpcode() ||
48727 X86ISD::HSUB == N->getOpcode() || X86ISD::FHSUB == N->getOpcode()) &&
48728 "Unexpected horizontal add/sub opcode");
48729
48730 if (!shouldUseHorizontalOp(true, DAG, Subtarget)) {
48731 MVT VT = N->getSimpleValueType(0);
48732 SDValue LHS = N->getOperand(0);
48733 SDValue RHS = N->getOperand(1);
48734
48735 // HOP(HOP'(X,X),HOP'(Y,Y)) -> HOP(PERMUTE(HOP'(X,Y)),PERMUTE(HOP'(X,Y)).
48736 if (LHS != RHS && LHS.getOpcode() == N->getOpcode() &&
48737 LHS.getOpcode() == RHS.getOpcode() &&
48738 LHS.getValueType() == RHS.getValueType() &&
48739 N->isOnlyUserOf(LHS.getNode()) && N->isOnlyUserOf(RHS.getNode())) {
48740 SDValue LHS0 = LHS.getOperand(0);
48741 SDValue LHS1 = LHS.getOperand(1);
48742 SDValue RHS0 = RHS.getOperand(0);
48743 SDValue RHS1 = RHS.getOperand(1);
48744 if ((LHS0 == LHS1 || LHS0.isUndef() || LHS1.isUndef()) &&
48745 (RHS0 == RHS1 || RHS0.isUndef() || RHS1.isUndef())) {
48746 SDLoc DL(N);
48747 SDValue Res = DAG.getNode(LHS.getOpcode(), DL, LHS.getValueType(),
48748 LHS0.isUndef() ? LHS1 : LHS0,
48749 RHS0.isUndef() ? RHS1 : RHS0);
48750 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);
48751 Res = DAG.getBitcast(ShufVT, Res);
48752 SDValue NewLHS =
48753 DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
48754 getV4X86ShuffleImm8ForMask({0, 1, 0, 1}, DL, DAG));
48755 SDValue NewRHS =
48756 DAG.getNode(X86ISD::PSHUFD, DL, ShufVT, Res,
48757 getV4X86ShuffleImm8ForMask({2, 3, 2, 3}, DL, DAG));
48758 return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(VT, NewLHS),
48759 DAG.getBitcast(VT, NewRHS));
48760 }
48761 }
48762 }
48763
48764 // Try to fold HOP(SHUFFLE(),SHUFFLE()) -> SHUFFLE(HOP()).
48765 if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget))
48766 return V;
48767
48768 return SDValue();
48769}
48770
48771static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG,
48772 TargetLowering::DAGCombinerInfo &DCI,
48773 const X86Subtarget &Subtarget) {
48774 assert((X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->getOpcode() ||
48775 X86ISD::VSRL == N->getOpcode()) &&
48776 "Unexpected shift opcode");
48777 EVT VT = N->getValueType(0);
48778 SDValue N0 = N->getOperand(0);
48779 SDValue N1 = N->getOperand(1);
48780
48781 // Shift zero -> zero.
48782 if (ISD::isBuildVectorAllZeros(N0.getNode()))
48783 return DAG.getConstant(0, SDLoc(N), VT);
48784
48785 // Detect constant shift amounts.
48786 APInt UndefElts;
48787 SmallVector<APInt, 32> EltBits;
48788 if (getTargetConstantBitsFromNode(N1, 64, UndefElts, EltBits,
48789 /*AllowWholeUndefs*/ true,
48790 /*AllowPartialUndefs*/ false)) {
48791 unsigned X86Opc = getTargetVShiftUniformOpcode(N->getOpcode(), false);
48792 return getTargetVShiftByConstNode(X86Opc, SDLoc(N), VT.getSimpleVT(), N0,
48793 EltBits[0].getZExtValue(), DAG);
48794 }
48795
48796 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
48797 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
48798 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
48799 return SDValue(N, 0);
48800
48801 return SDValue();
48802}
48803
48804static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
48805 TargetLowering::DAGCombinerInfo &DCI,
48806 const X86Subtarget &Subtarget) {
48807 unsigned Opcode = N->getOpcode();
48808 assert((X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode ||
48809 X86ISD::VSRLI == Opcode) &&
48810 "Unexpected shift opcode");
48811 bool LogicalShift = X86ISD::VSHLI == Opcode || X86ISD::VSRLI == Opcode;
48812 EVT VT = N->getValueType(0);
48813 SDValue N0 = N->getOperand(0);
48814 SDValue N1 = N->getOperand(1);
48815 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
48816 assert(VT == N0.getValueType() && (NumBitsPerElt % 8) == 0 &&
48817 "Unexpected value type");
48818 assert(N1.getValueType() == MVT::i8 && "Unexpected shift amount type");
48819
48820 // (shift undef, X) -> 0
48821 if (N0.isUndef())
48822 return DAG.getConstant(0, SDLoc(N), VT);
48823
48824 // Out of range logical bit shifts are guaranteed to be zero.
48825 // Out of range arithmetic bit shifts splat the sign bit.
48826 unsigned ShiftVal = N->getConstantOperandVal(1);
48827 if (ShiftVal >= NumBitsPerElt) {
48828 if (LogicalShift)
48829 return DAG.getConstant(0, SDLoc(N), VT);
48830 ShiftVal = NumBitsPerElt - 1;
48831 }
48832
48833 // (shift X, 0) -> X
48834 if (!ShiftVal)
48835 return N0;
48836
48837 // (shift 0, C) -> 0
48838 if (ISD::isBuildVectorAllZeros(N0.getNode()))
48839 // N0 is all zeros or undef. We guarantee that the bits shifted into the
48840 // result are all zeros, not undef.
48841 return DAG.getConstant(0, SDLoc(N), VT);
48842
48843 // (VSRAI -1, C) -> -1
48844 if (!LogicalShift && ISD::isBuildVectorAllOnes(N0.getNode()))
48845 // N0 is all ones or undef. We guarantee that the bits shifted into the
48846 // result are all ones, not undef.
48847 return DAG.getConstant(-1, SDLoc(N), VT);
48848
48849 auto MergeShifts = [&](SDValue X, uint64_t Amt0, uint64_t Amt1) {
48850 unsigned NewShiftVal = Amt0 + Amt1;
48851 if (NewShiftVal >= NumBitsPerElt) {
48852 // Out of range logical bit shifts are guaranteed to be zero.
48853 // Out of range arithmetic bit shifts splat the sign bit.
48854 if (LogicalShift)
48855 return DAG.getConstant(0, SDLoc(N), VT);
48856 NewShiftVal = NumBitsPerElt - 1;
48857 }
48858 return DAG.getNode(Opcode, SDLoc(N), VT, N0.getOperand(0),
48859 DAG.getTargetConstant(NewShiftVal, SDLoc(N), MVT::i8));
48860 };
48861
48862 // (shift (shift X, C2), C1) -> (shift X, (C1 + C2))
48863 if (Opcode == N0.getOpcode())
48864 return MergeShifts(N0.getOperand(0), ShiftVal, N0.getConstantOperandVal(1));
48865
48866 // (shl (add X, X), C) -> (shl X, (C + 1))
48867 if (Opcode == X86ISD::VSHLI && N0.getOpcode() == ISD::ADD &&
48868 N0.getOperand(0) == N0.getOperand(1))
48869 return MergeShifts(N0.getOperand(0), ShiftVal, 1);
48870
48871 // We can decode 'whole byte' logical bit shifts as shuffles.
48872 if (LogicalShift && (ShiftVal % 8) == 0) {
48873 SDValue Op(N, 0);
48874 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
48875 return Res;
48876 }
48877
48878 // Attempt to detect an expanded vXi64 SIGN_EXTEND_INREG vXi1 pattern, and
48879 // convert to a splatted v2Xi32 SIGN_EXTEND_INREG pattern:
48880 // psrad(pshufd(psllq(X,63),1,1,3,3),31) ->
48881 // pshufd(psrad(pslld(X,31),31),0,0,2,2).
48882 if (Opcode == X86ISD::VSRAI && NumBitsPerElt == 32 && ShiftVal == 31 &&
48883 N0.getOpcode() == X86ISD::PSHUFD &&
48884 N0.getConstantOperandVal(1) == getV4X86ShuffleImm({1, 1, 3, 3}) &&
48885 N0->hasOneUse()) {
48886 SDValue BC = peekThroughOneUseBitcasts(N0.getOperand(0));
48887 if (BC.getOpcode() == X86ISD::VSHLI &&
48888 BC.getScalarValueSizeInBits() == 64 &&
48889 BC.getConstantOperandVal(1) == 63) {
48890 SDLoc DL(N);
48891 SDValue Src = BC.getOperand(0);
48892 Src = DAG.getBitcast(VT, Src);
48893 Src = DAG.getNode(X86ISD::PSHUFD, DL, VT, Src,
48894 getV4X86ShuffleImm8ForMask({0, 0, 2, 2}, DL, DAG));
48895 Src = DAG.getNode(X86ISD::VSHLI, DL, VT, Src, N1);
48896 Src = DAG.getNode(X86ISD::VSRAI, DL, VT, Src, N1);
48897 return Src;
48898 }
48899 }
48900
48901 auto TryConstantFold = [&](SDValue V) {
48902 APInt UndefElts;
48903 SmallVector<APInt, 32> EltBits;
48904 if (!getTargetConstantBitsFromNode(V, NumBitsPerElt, UndefElts, EltBits,
48905 /*AllowWholeUndefs*/ true,
48906 /*AllowPartialUndefs*/ true))
48907 return SDValue();
48908 assert(EltBits.size() == VT.getVectorNumElements() &&
48909 "Unexpected shift value type");
48910 // Undef elements need to fold to 0. It's possible SimplifyDemandedBits
48911 // created an undef input due to no input bits being demanded, but user
48912 // still expects 0 in other bits.
48913 for (unsigned i = 0, e = EltBits.size(); i != e; ++i) {
48914 APInt &Elt = EltBits[i];
48915 if (UndefElts[i])
48916 Elt = 0;
48917 else if (X86ISD::VSHLI == Opcode)
48918 Elt <<= ShiftVal;
48919 else if (X86ISD::VSRAI == Opcode)
48920 Elt.ashrInPlace(ShiftVal);
48921 else
48922 Elt.lshrInPlace(ShiftVal);
48923 }
48924 // Reset undef elements since they were zeroed above.
48925 UndefElts = 0;
48926 return getConstVector(EltBits, UndefElts, VT.getSimpleVT(), DAG, SDLoc(N));
48927 };
48928
48929 // Constant Folding.
48930 if (N->isOnlyUserOf(N0.getNode())) {
48931 if (SDValue C = TryConstantFold(N0))
48932 return C;
48933
48934 // Fold (shift (logic X, C2), C1) -> (logic (shift X, C1), (shift C2, C1))
48935 // Don't break NOT patterns.
48936 SDValue BC = peekThroughOneUseBitcasts(N0);
48937 if (ISD::isBitwiseLogicOp(BC.getOpcode()) &&
48938 BC->isOnlyUserOf(BC.getOperand(1).getNode()) &&
48939 !ISD::isBuildVectorAllOnes(BC.getOperand(1).getNode())) {
48940 if (SDValue RHS = TryConstantFold(BC.getOperand(1))) {
48941 SDLoc DL(N);
48942 SDValue LHS = DAG.getNode(Opcode, DL, VT,
48943 DAG.getBitcast(VT, BC.getOperand(0)), N1);
48944 return DAG.getNode(BC.getOpcode(), DL, VT, LHS, RHS);
48945 }
48946 }
48947 }
48948
48949 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
48950 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumBitsPerElt),
48951 DCI))
48952 return SDValue(N, 0);
48953
48954 return SDValue();
48955}
48956
48957static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG,
48958 TargetLowering::DAGCombinerInfo &DCI,
48959 const X86Subtarget &Subtarget) {
48960 EVT VT = N->getValueType(0);
48961 unsigned Opcode = N->getOpcode();
48962 assert(((Opcode == X86ISD::PINSRB && VT == MVT::v16i8) ||
48963 (Opcode == X86ISD::PINSRW && VT == MVT::v8i16) ||
48964 Opcode == ISD::INSERT_VECTOR_ELT) &&
48965 "Unexpected vector insertion");
48966
48967 SDValue Vec = N->getOperand(0);
48968 SDValue Scl = N->getOperand(1);
48969 SDValue Idx = N->getOperand(2);
48970
48971 // Fold insert_vector_elt(undef, elt, 0) --> scalar_to_vector(elt).
48972 if (Opcode == ISD::INSERT_VECTOR_ELT && Vec.isUndef() && isNullConstant(Idx))
48973 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Scl);
48974
48975 if (Opcode == X86ISD::PINSRB || Opcode == X86ISD::PINSRW) {
48976 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
48977 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
48978 if (TLI.SimplifyDemandedBits(SDValue(N, 0),
48979 APInt::getAllOnes(NumBitsPerElt), DCI))
48980 return SDValue(N, 0);
48981 }
48982
48983 // Attempt to combine insertion patterns to a shuffle.
48984 if (VT.isSimple() && DCI.isAfterLegalizeDAG()) {
48985 SDValue Op(N, 0);
48986 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
48987 return Res;
48988 }
48989
48990 return SDValue();
48991}
48992
48993/// Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs
48994/// reference the same FP CMP, and rewrite for CMPEQSS and friends. Likewise for
48995/// OR -> CMPNEQSS.
48996static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
48997 TargetLowering::DAGCombinerInfo &DCI,
48998 const X86Subtarget &Subtarget) {
48999 unsigned opcode;
49000
49001 // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but
49002 // we're requiring SSE2 for both.
49003 if (Subtarget.hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
49004 SDValue N0 = N->getOperand(0);
49005 SDValue N1 = N->getOperand(1);
49006 SDValue CMP0 = N0.getOperand(1);
49007 SDValue CMP1 = N1.getOperand(1);
49008 SDLoc DL(N);
49009
49010 // The SETCCs should both refer to the same CMP.
49011 if (CMP0.getOpcode() != X86ISD::FCMP || CMP0 != CMP1)
49012 return SDValue();
49013
49014 SDValue CMP00 = CMP0->getOperand(0);
49015 SDValue CMP01 = CMP0->getOperand(1);
49016 EVT VT = CMP00.getValueType();
49017
49018 if (VT == MVT::f32 || VT == MVT::f64 ||
49019 (VT == MVT::f16 && Subtarget.hasFP16())) {
49020 bool ExpectingFlags = false;
49021 // Check for any users that want flags:
49022 for (const SDNode *U : N->uses()) {
49023 if (ExpectingFlags)
49024 break;
49025
49026 switch (U->getOpcode()) {
49027 default:
49028 case ISD::BR_CC:
49029 case ISD::BRCOND:
49030 case ISD::SELECT:
49031 ExpectingFlags = true;
49032 break;
49033 case ISD::CopyToReg:
49034 case ISD::SIGN_EXTEND:
49035 case ISD::ZERO_EXTEND:
49036 case ISD::ANY_EXTEND:
49037 break;
49038 }
49039 }
49040
49041 if (!ExpectingFlags) {
49042 enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
49043 enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
49044
49045 if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
49046 X86::CondCode tmp = cc0;
49047 cc0 = cc1;
49048 cc1 = tmp;
49049 }
49050
49051 if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) ||
49052 (cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
49053 // FIXME: need symbolic constants for these magic numbers.
49054 // See X86ATTInstPrinter.cpp:printSSECC().
49055 unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4;
49056 if (Subtarget.hasAVX512()) {
49057 SDValue FSetCC =
49058 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
49059 DAG.getTargetConstant(x86cc, DL, MVT::i8));
49060 // Need to fill with zeros to ensure the bitcast will produce zeroes
49061 // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
49062 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
49063 DAG.getConstant(0, DL, MVT::v16i1),
49064 FSetCC, DAG.getIntPtrConstant(0, DL));
49065 return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL,
49066 N->getSimpleValueType(0));
49067 }
49068 SDValue OnesOrZeroesF =
49069 DAG.getNode(X86ISD::FSETCC, DL, CMP00.getValueType(), CMP00,
49070 CMP01, DAG.getTargetConstant(x86cc, DL, MVT::i8));
49071
49072 bool is64BitFP = (CMP00.getValueType() == MVT::f64);
49073 MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;
49074
49075 if (is64BitFP && !Subtarget.is64Bit()) {
49076 // On a 32-bit target, we cannot bitcast the 64-bit float to a
49077 // 64-bit integer, since that's not a legal type. Since
49078 // OnesOrZeroesF is all ones or all zeroes, we don't need all the
49079 // bits, but can do this little dance to extract the lowest 32 bits
49080 // and work with those going forward.
49081 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
49082 OnesOrZeroesF);
49083 SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);
49084 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,
49085 Vector32, DAG.getIntPtrConstant(0, DL));
49086 IntVT = MVT::i32;
49087 }
49088
49089 SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
49090 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
49091 DAG.getConstant(1, DL, IntVT));
49092 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
49093 ANDed);
49094 return OneBitOfTruth;
49095 }
49096 }
49097 }
49098 }
49099 return SDValue();
49100}
49101
49102/// Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
49103static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG) {
49104 assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP");
49105
49106 MVT VT = N->getSimpleValueType(0);
49107 if (!VT.is128BitVector() && !VT.is256BitVector() && !VT.is512BitVector())
49108 return SDValue();
49109
49110 SDValue X, Y;
49111 SDValue N0 = N->getOperand(0);
49112 SDValue N1 = N->getOperand(1);
49113
49114 if (SDValue Not = IsNOT(N0, DAG)) {
49115 X = Not;
49116 Y = N1;
49117 } else if (SDValue Not = IsNOT(N1, DAG)) {
49118 X = Not;
49119 Y = N0;
49120 } else
49121 return SDValue();
49122
49123 X = DAG.getBitcast(VT, X);
49124 Y = DAG.getBitcast(VT, Y);
49125 return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);
49126}
49127
49128/// Try to fold:
49129/// and (vector_shuffle<Z,...,Z>
49130/// (insert_vector_elt undef, (xor X, -1), Z), undef), Y
49131/// ->
49132/// andnp (vector_shuffle<Z,...,Z>
49133/// (insert_vector_elt undef, X, Z), undef), Y
49134static SDValue combineAndShuffleNot(SDNode *N, SelectionDAG &DAG,
49135 const X86Subtarget &Subtarget) {
49136 assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP");
49137
49138 EVT VT = N->getValueType(0);
49139 // Do not split 256 and 512 bit vectors with SSE2 as they overwrite original
49140 // value and require extra moves.
49141 if (!((VT.is128BitVector() && Subtarget.hasSSE2()) ||
49142 ((VT.is256BitVector() || VT.is512BitVector()) && Subtarget.hasAVX())))
49143 return SDValue();
49144
49145 auto GetNot = [&DAG](SDValue V) {
49146 auto *SVN = dyn_cast<ShuffleVectorSDNode>(peekThroughOneUseBitcasts(V));
49147 // TODO: SVN->hasOneUse() is a strong condition. It can be relaxed if all
49148 // end-users are ISD::AND including cases
49149 // (and(extract_vector_element(SVN), Y)).
49150 if (!SVN || !SVN->hasOneUse() || !SVN->isSplat() ||
49151 !SVN->getOperand(1).isUndef()) {
49152 return SDValue();
49153 }
49154 SDValue IVEN = SVN->getOperand(0);
49155 if (IVEN.getOpcode() != ISD::INSERT_VECTOR_ELT ||
49156 !IVEN.getOperand(0).isUndef() || !IVEN.hasOneUse())
49157 return SDValue();
49158 if (!isa<ConstantSDNode>(IVEN.getOperand(2)) ||
49159 IVEN.getConstantOperandAPInt(2) != SVN->getSplatIndex())
49160 return SDValue();
49161 SDValue Src = IVEN.getOperand(1);
49162 if (SDValue Not = IsNOT(Src, DAG)) {
49163 SDValue NotSrc = DAG.getBitcast(Src.getValueType(), Not);
49164 SDValue NotIVEN =
49165 DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(IVEN), IVEN.getValueType(),
49166 IVEN.getOperand(0), NotSrc, IVEN.getOperand(2));
49167 return DAG.getVectorShuffle(SVN->getValueType(0), SDLoc(SVN), NotIVEN,
49168 SVN->getOperand(1), SVN->getMask());
49169 }
49170 return SDValue();
49171 };
49172
49173 SDValue X, Y;
49174 SDValue N0 = N->getOperand(0);
49175 SDValue N1 = N->getOperand(1);
49176 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49177
49178 if (SDValue Not = GetNot(N0)) {
49179 X = Not;
49180 Y = N1;
49181 } else if (SDValue Not = GetNot(N1)) {
49182 X = Not;
49183 Y = N0;
49184 } else
49185 return SDValue();
49186
49187 X = DAG.getBitcast(VT, X);
49188 Y = DAG.getBitcast(VT, Y);
49189 SDLoc DL(N);
49190
49191 // We do not split for SSE at all, but we need to split vectors for AVX1 and
49192 // AVX2.
49193 if (!Subtarget.useAVX512Regs() && VT.is512BitVector() &&
49194 TLI.isTypeLegal(VT.getHalfNumVectorElementsVT(*DAG.getContext()))) {
49195 SDValue LoX, HiX;
49196 std::tie(LoX, HiX) = splitVector(X, DAG, DL);
49197 SDValue LoY, HiY;
49198 std::tie(LoY, HiY) = splitVector(Y, DAG, DL);
49199 EVT SplitVT = LoX.getValueType();
49200 SDValue LoV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {LoX, LoY});
49201 SDValue HiV = DAG.getNode(X86ISD::ANDNP, DL, SplitVT, {HiX, HiY});
49202 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, {LoV, HiV});
49203 }
49204
49205 if (TLI.isTypeLegal(VT))
49206 return DAG.getNode(X86ISD::ANDNP, DL, VT, {X, Y});
49207
49208 return SDValue();
49209}
49210
49211// Try to widen AND, OR and XOR nodes to VT in order to remove casts around
49212// logical operations, like in the example below.
49213// or (and (truncate x, truncate y)),
49214// (xor (truncate z, build_vector (constants)))
49215// Given a target type \p VT, we generate
49216// or (and x, y), (xor z, zext(build_vector (constants)))
49217// given x, y and z are of type \p VT. We can do so, if operands are either
49218// truncates from VT types, the second operand is a vector of constants or can
49219// be recursively promoted.
49220static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT,
49221 SelectionDAG &DAG, unsigned Depth) {
49222 // Limit recursion to avoid excessive compile times.
49223 if (Depth >= SelectionDAG::MaxRecursionDepth)
49224 return SDValue();
49225
49226 if (!ISD::isBitwiseLogicOp(N.getOpcode()))
49227 return SDValue();
49228
49229 SDValue N0 = N.getOperand(0);
49230 SDValue N1 = N.getOperand(1);
49231
49232 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49233 if (!TLI.isOperationLegalOrPromote(N.getOpcode(), VT))
49234 return SDValue();
49235
49236 if (SDValue NN0 = PromoteMaskArithmetic(N0, DL, VT, DAG, Depth + 1))
49237 N0 = NN0;
49238 else {
49239 // The left side has to be a trunc.
49240 if (N0.getOpcode() != ISD::TRUNCATE)
49241 return SDValue();
49242
49243 // The type of the truncated inputs.
49244 if (N0.getOperand(0).getValueType() != VT)
49245 return SDValue();
49246
49247 N0 = N0.getOperand(0);
49248 }
49249
49250 if (SDValue NN1 = PromoteMaskArithmetic(N1, DL, VT, DAG, Depth + 1))
49251 N1 = NN1;
49252 else {
49253 // The right side has to be a 'trunc' or a (foldable) constant.
49254 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
49255 N1.getOperand(0).getValueType() == VT;
49256 if (RHSTrunc)
49257 N1 = N1.getOperand(0);
49258 else if (SDValue Cst =
49259 DAG.FoldConstantArithmetic(ISD::ZERO_EXTEND, DL, VT, {N1}))
49260 N1 = Cst;
49261 else
49262 return SDValue();
49263 }
49264
49265 return DAG.getNode(N.getOpcode(), DL, VT, N0, N1);
49266}
49267
49268// On AVX/AVX2 the type v8i1 is legalized to v8i16, which is an XMM sized
49269// register. In most cases we actually compare or select YMM-sized registers
49270// and mixing the two types creates horrible code. This method optimizes
49271// some of the transition sequences.
49272// Even with AVX-512 this is still useful for removing casts around logical
49273// operations on vXi1 mask types.
49274static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL,
49275 SelectionDAG &DAG,
49276 const X86Subtarget &Subtarget) {
49277 EVT VT = N.getValueType();
49278 assert(VT.isVector() && "Expected vector type");
49279 assert((N.getOpcode() == ISD::ANY_EXTEND ||
49280 N.getOpcode() == ISD::ZERO_EXTEND ||
49281 N.getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node");
49282
49283 SDValue Narrow = N.getOperand(0);
49284 EVT NarrowVT = Narrow.getValueType();
49285
49286 // Generate the wide operation.
49287 SDValue Op = PromoteMaskArithmetic(Narrow, DL, VT, DAG, 0);
49288 if (!Op)
49289 return SDValue();
49290 switch (N.getOpcode()) {
49291 default: llvm_unreachable("Unexpected opcode");
49292 case ISD::ANY_EXTEND:
49293 return Op;
49294 case ISD::ZERO_EXTEND:
49295 return DAG.getZeroExtendInReg(Op, DL, NarrowVT);
49296 case ISD::SIGN_EXTEND:
49297 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
49298 Op, DAG.getValueType(NarrowVT));
49299 }
49300}
49301
49302static unsigned convertIntLogicToFPLogicOpcode(unsigned Opcode) {
49303 unsigned FPOpcode;
49304 switch (Opcode) {
49305 // clang-format off
49306 default: llvm_unreachable("Unexpected input node for FP logic conversion");
49307 case ISD::AND: FPOpcode = X86ISD::FAND; break;
49308 case ISD::OR: FPOpcode = X86ISD::FOR; break;
49309 case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
49310 // clang-format on
49311 }
49312 return FPOpcode;
49313}
49314
49315/// If both input operands of a logic op are being cast from floating-point
49316/// types or FP compares, try to convert this into a floating-point logic node
49317/// to avoid unnecessary moves from SSE to integer registers.
49318static SDValue convertIntLogicToFPLogic(SDNode *N, SelectionDAG &DAG,
49319 TargetLowering::DAGCombinerInfo &DCI,
49320 const X86Subtarget &Subtarget) {
49321 EVT VT = N->getValueType(0);
49322 SDValue N0 = N->getOperand(0);
49323 SDValue N1 = N->getOperand(1);
49324 SDLoc DL(N);
49325
49326 if (!((N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) ||
49327 (N0.getOpcode() == ISD::SETCC && N1.getOpcode() == ISD::SETCC)))
49328 return SDValue();
49329
49330 SDValue N00 = N0.getOperand(0);
49331 SDValue N10 = N1.getOperand(0);
49332 EVT N00Type = N00.getValueType();
49333 EVT N10Type = N10.getValueType();
49334
49335 // Ensure that both types are the same and are legal scalar fp types.
49336 if (N00Type != N10Type || !((Subtarget.hasSSE1() && N00Type == MVT::f32) ||
49337 (Subtarget.hasSSE2() && N00Type == MVT::f64) ||
49338 (Subtarget.hasFP16() && N00Type == MVT::f16)))
49339 return SDValue();
49340
49341 if (N0.getOpcode() == ISD::BITCAST && !DCI.isBeforeLegalizeOps()) {
49342 unsigned FPOpcode = convertIntLogicToFPLogicOpcode(N->getOpcode());
49343 SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
49344 return DAG.getBitcast(VT, FPLogic);
49345 }
49346
49347 if (VT != MVT::i1 || N0.getOpcode() != ISD::SETCC || !N0.hasOneUse() ||
49348 !N1.hasOneUse())
49349 return SDValue();
49350
49351 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0.getOperand(2))->get();
49352 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1.getOperand(2))->get();
49353
49354 // The vector ISA for FP predicates is incomplete before AVX, so converting
49355 // COMIS* to CMPS* may not be a win before AVX.
49356 if (!Subtarget.hasAVX() &&
49357 !(cheapX86FSETCC_SSE(CC0) && cheapX86FSETCC_SSE(CC1)))
49358 return SDValue();
49359
49360 // Convert scalar FP compares and logic to vector compares (COMIS* to CMPS*)
49361 // and vector logic:
49362 // logic (setcc N00, N01), (setcc N10, N11) -->
49363 // extelt (logic (setcc (s2v N00), (s2v N01)), setcc (s2v N10), (s2v N11))), 0
49364 unsigned NumElts = 128 / N00Type.getSizeInBits();
49365 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), N00Type, NumElts);
49366 EVT BoolVecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
49367 SDValue ZeroIndex = DAG.getVectorIdxConstant(0, DL);
49368 SDValue N01 = N0.getOperand(1);
49369 SDValue N11 = N1.getOperand(1);
49370 SDValue Vec00 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N00);
49371 SDValue Vec01 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N01);
49372 SDValue Vec10 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N10);
49373 SDValue Vec11 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, N11);
49374 SDValue Setcc0 = DAG.getSetCC(DL, BoolVecVT, Vec00, Vec01, CC0);
49375 SDValue Setcc1 = DAG.getSetCC(DL, BoolVecVT, Vec10, Vec11, CC1);
49376 SDValue Logic = DAG.getNode(N->getOpcode(), DL, BoolVecVT, Setcc0, Setcc1);
49377 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Logic, ZeroIndex);
49378}
49379
49380// Attempt to fold BITOP(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(BITOP(X,Y))
49381// to reduce XMM->GPR traffic.
49382static SDValue combineBitOpWithMOVMSK(SDNode *N, SelectionDAG &DAG) {
49383 unsigned Opc = N->getOpcode();
49384 assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
49385 "Unexpected bit opcode");
49386
49387 SDValue N0 = N->getOperand(0);
49388 SDValue N1 = N->getOperand(1);
49389
49390 // Both operands must be single use MOVMSK.
49391 if (N0.getOpcode() != X86ISD::MOVMSK || !N0.hasOneUse() ||
49392 N1.getOpcode() != X86ISD::MOVMSK || !N1.hasOneUse())
49393 return SDValue();
49394
49395 SDValue Vec0 = N0.getOperand(0);
49396 SDValue Vec1 = N1.getOperand(0);
49397 EVT VecVT0 = Vec0.getValueType();
49398 EVT VecVT1 = Vec1.getValueType();
49399
49400 // Both MOVMSK operands must be from vectors of the same size and same element
49401 // size, but its OK for a fp/int diff.
49402 if (VecVT0.getSizeInBits() != VecVT1.getSizeInBits() ||
49403 VecVT0.getScalarSizeInBits() != VecVT1.getScalarSizeInBits())
49404 return SDValue();
49405
49406 SDLoc DL(N);
49407 unsigned VecOpc =
49408 VecVT0.isFloatingPoint() ? convertIntLogicToFPLogicOpcode(Opc) : Opc;
49409 SDValue Result =
49410 DAG.getNode(VecOpc, DL, VecVT0, Vec0, DAG.getBitcast(VecVT0, Vec1));
49411 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
49412}
49413
49414// Attempt to fold BITOP(SHIFT(X,Z),SHIFT(Y,Z)) -> SHIFT(BITOP(X,Y),Z).
49415// NOTE: This is a very limited case of what SimplifyUsingDistributiveLaws
49416// handles in InstCombine.
49417static SDValue combineBitOpWithShift(SDNode *N, SelectionDAG &DAG) {
49418 unsigned Opc = N->getOpcode();
49419 assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
49420 "Unexpected bit opcode");
49421
49422 SDValue N0 = N->getOperand(0);
49423 SDValue N1 = N->getOperand(1);
49424 EVT VT = N->getValueType(0);
49425
49426 // Both operands must be single use.
49427 if (!N0.hasOneUse() || !N1.hasOneUse())
49428 return SDValue();
49429
49430 // Search for matching shifts.
49431 SDValue BC0 = peekThroughOneUseBitcasts(N0);
49432 SDValue BC1 = peekThroughOneUseBitcasts(N1);
49433
49434 unsigned BCOpc = BC0.getOpcode();
49435 EVT BCVT = BC0.getValueType();
49436 if (BCOpc != BC1->getOpcode() || BCVT != BC1.getValueType())
49437 return SDValue();
49438
49439 switch (BCOpc) {
49440 case X86ISD::VSHLI:
49441 case X86ISD::VSRLI:
49442 case X86ISD::VSRAI: {
49443 if (BC0.getOperand(1) != BC1.getOperand(1))
49444 return SDValue();
49445
49446 SDLoc DL(N);
49447 SDValue BitOp =
49448 DAG.getNode(Opc, DL, BCVT, BC0.getOperand(0), BC1.getOperand(0));
49449 SDValue Shift = DAG.getNode(BCOpc, DL, BCVT, BitOp, BC0.getOperand(1));
49450 return DAG.getBitcast(VT, Shift);
49451 }
49452 }
49453
49454 return SDValue();
49455}
49456
49457// Attempt to fold:
49458// BITOP(PACKSS(X,Z),PACKSS(Y,W)) --> PACKSS(BITOP(X,Y),BITOP(Z,W)).
49459// TODO: Handle PACKUS handling.
49460static SDValue combineBitOpWithPACK(SDNode *N, SelectionDAG &DAG) {
49461 unsigned Opc = N->getOpcode();
49462 assert((Opc == ISD::OR || Opc == ISD::AND || Opc == ISD::XOR) &&
49463 "Unexpected bit opcode");
49464
49465 SDValue N0 = N->getOperand(0);
49466 SDValue N1 = N->getOperand(1);
49467 EVT VT = N->getValueType(0);
49468
49469 // Both operands must be single use.
49470 if (!N0.hasOneUse() || !N1.hasOneUse())
49471 return SDValue();
49472
49473 // Search for matching packs.
49474 N0 = peekThroughOneUseBitcasts(N0);
49475 N1 = peekThroughOneUseBitcasts(N1);
49476
49477 if (N0.getOpcode() != X86ISD::PACKSS || N1.getOpcode() != X86ISD::PACKSS)
49478 return SDValue();
49479
49480 MVT DstVT = N0.getSimpleValueType();
49481 if (DstVT != N1.getSimpleValueType())
49482 return SDValue();
49483
49484 MVT SrcVT = N0.getOperand(0).getSimpleValueType();
49485 unsigned NumSrcBits = SrcVT.getScalarSizeInBits();
49486
49487 // Limit to allsignbits packing.
49488 if (DAG.ComputeNumSignBits(N0.getOperand(0)) != NumSrcBits ||
49489 DAG.ComputeNumSignBits(N0.getOperand(1)) != NumSrcBits ||
49490 DAG.ComputeNumSignBits(N1.getOperand(0)) != NumSrcBits ||
49491 DAG.ComputeNumSignBits(N1.getOperand(1)) != NumSrcBits)
49492 return SDValue();
49493
49494 SDLoc DL(N);
49495 SDValue LHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(0), N1.getOperand(0));
49496 SDValue RHS = DAG.getNode(Opc, DL, SrcVT, N0.getOperand(1), N1.getOperand(1));
49497 return DAG.getBitcast(VT, DAG.getNode(X86ISD::PACKSS, DL, DstVT, LHS, RHS));
49498}
49499
49500/// If this is a zero/all-bits result that is bitwise-anded with a low bits
49501/// mask. (Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and'
49502/// with a shift-right to eliminate loading the vector constant mask value.
49503static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
49504 const X86Subtarget &Subtarget) {
49505 SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
49506 SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
49507 EVT VT = Op0.getValueType();
49508 if (VT != Op1.getValueType() || !VT.isSimple() || !VT.isInteger())
49509 return SDValue();
49510
49511 // Try to convert an "is positive" signbit masking operation into arithmetic
49512 // shift and "andn". This saves a materialization of a -1 vector constant.
49513 // The "is negative" variant should be handled more generally because it only
49514 // requires "and" rather than "andn":
49515 // and (pcmpgt X, -1), Y --> pandn (vsrai X, BitWidth - 1), Y
49516 //
49517 // This is limited to the original type to avoid producing even more bitcasts.
49518 // If the bitcasts can't be eliminated, then it is unlikely that this fold
49519 // will be profitable.
49520 if (N->getValueType(0) == VT &&
49521 supportedVectorShiftWithImm(VT, Subtarget, ISD::SRA)) {
49522 SDValue X, Y;
49523 if (Op1.getOpcode() == X86ISD::PCMPGT &&
49524 isAllOnesOrAllOnesSplat(Op1.getOperand(1)) && Op1.hasOneUse()) {
49525 X = Op1.getOperand(0);
49526 Y = Op0;
49527 } else if (Op0.getOpcode() == X86ISD::PCMPGT &&
49528 isAllOnesOrAllOnesSplat(Op0.getOperand(1)) && Op0.hasOneUse()) {
49529 X = Op0.getOperand(0);
49530 Y = Op1;
49531 }
49532 if (X && Y) {
49533 SDLoc DL(N);
49534 SDValue Sra =
49535 getTargetVShiftByConstNode(X86ISD::VSRAI, DL, VT.getSimpleVT(), X,
49536 VT.getScalarSizeInBits() - 1, DAG);
49537 return DAG.getNode(X86ISD::ANDNP, DL, VT, Sra, Y);
49538 }
49539 }
49540
49541 APInt SplatVal;
49542 if (!X86::isConstantSplat(Op1, SplatVal, false) || !SplatVal.isMask())
49543 return SDValue();
49544
49545 // Don't prevent creation of ANDN.
49546 if (isBitwiseNot(Op0))
49547 return SDValue();
49548
49549 if (!supportedVectorShiftWithImm(VT, Subtarget, ISD::SRL))
49550 return SDValue();
49551
49552 unsigned EltBitWidth = VT.getScalarSizeInBits();
49553 if (EltBitWidth != DAG.ComputeNumSignBits(Op0))
49554 return SDValue();
49555
49556 SDLoc DL(N);
49557 unsigned ShiftVal = SplatVal.countr_one();
49558 SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
49559 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt);
49560 return DAG.getBitcast(N->getValueType(0), Shift);
49561}
49562
49563// Get the index node from the lowered DAG of a GEP IR instruction with one
49564// indexing dimension.
49565static SDValue getIndexFromUnindexedLoad(LoadSDNode *Ld) {
49566 if (Ld->isIndexed())
49567 return SDValue();
49568
49569 SDValue Base = Ld->getBasePtr();
49570
49571 if (Base.getOpcode() != ISD::ADD)
49572 return SDValue();
49573
49574 SDValue ShiftedIndex = Base.getOperand(0);
49575
49576 if (ShiftedIndex.getOpcode() != ISD::SHL)
49577 return SDValue();
49578
49579 return ShiftedIndex.getOperand(0);
49580
49581}
49582
49583static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) {
49584 return Subtarget.hasBMI2() &&
49585 (VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit()));
49586}
49587
49588// This function recognizes cases where X86 bzhi instruction can replace and
49589// 'and-load' sequence.
49590// In case of loading integer value from an array of constants which is defined
49591// as follows:
49592//
49593// int array[SIZE] = {0x0, 0x1, 0x3, 0x7, 0xF ..., 2^(SIZE-1) - 1}
49594//
49595// then applying a bitwise and on the result with another input.
49596// It's equivalent to performing bzhi (zero high bits) on the input, with the
49597// same index of the load.
49598static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG,
49599 const X86Subtarget &Subtarget) {
49600 MVT VT = Node->getSimpleValueType(0);
49601 SDLoc dl(Node);
49602
49603 // Check if subtarget has BZHI instruction for the node's type
49604 if (!hasBZHI(Subtarget, VT))
49605 return SDValue();
49606
49607 // Try matching the pattern for both operands.
49608 for (unsigned i = 0; i < 2; i++) {
49609 SDValue N = Node->getOperand(i);
49610 LoadSDNode *Ld = dyn_cast<LoadSDNode>(N.getNode());
49611
49612 // continue if the operand is not a load instruction
49613 if (!Ld)
49614 return SDValue();
49615
49616 const Value *MemOp = Ld->getMemOperand()->getValue();
49617
49618 if (!MemOp)
49619 return SDValue();
49620
49621 if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(MemOp)) {
49622 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) {
49623 if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
49624
49625 Constant *Init = GV->getInitializer();
49626 Type *Ty = Init->getType();
49627 if (!isa<ConstantDataArray>(Init) ||
49628 !Ty->getArrayElementType()->isIntegerTy() ||
49629 Ty->getArrayElementType()->getScalarSizeInBits() !=
49630 VT.getSizeInBits() ||
49631 Ty->getArrayNumElements() >
49632 Ty->getArrayElementType()->getScalarSizeInBits())
49633 continue;
49634
49635 // Check if the array's constant elements are suitable to our case.
49636 uint64_t ArrayElementCount = Init->getType()->getArrayNumElements();
49637 bool ConstantsMatch = true;
49638 for (uint64_t j = 0; j < ArrayElementCount; j++) {
49639 auto *Elem = cast<ConstantInt>(Init->getAggregateElement(j));
49640 if (Elem->getZExtValue() != (((uint64_t)1 << j) - 1)) {
49641 ConstantsMatch = false;
49642 break;
49643 }
49644 }
49645 if (!ConstantsMatch)
49646 continue;
49647
49648 // Do the transformation (For 32-bit type):
49649 // -> (and (load arr[idx]), inp)
49650 // <- (and (srl 0xFFFFFFFF, (sub 32, idx)))
49651 // that will be replaced with one bzhi instruction.
49652 SDValue Inp = (i == 0) ? Node->getOperand(1) : Node->getOperand(0);
49653 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);
49654
49655 // Get the Node which indexes into the array.
49656 SDValue Index = getIndexFromUnindexedLoad(Ld);
49657 if (!Index)
49658 return SDValue();
49659 Index = DAG.getZExtOrTrunc(Index, dl, MVT::i32);
49660
49661 SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index);
49662 Sub = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Sub);
49663
49664 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
49665 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub);
49666
49667 return DAG.getNode(ISD::AND, dl, VT, Inp, LShr);
49668 }
49669 }
49670 }
49671 }
49672 return SDValue();
49673}
49674
49675// Look for (and (bitcast (vXi1 (concat_vectors (vYi1 setcc), undef,))), C)
49676// Where C is a mask containing the same number of bits as the setcc and
49677// where the setcc will freely 0 upper bits of k-register. We can replace the
49678// undef in the concat with 0s and remove the AND. This mainly helps with
49679// v2i1/v4i1 setcc being casted to scalar.
49680static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
49681 const X86Subtarget &Subtarget) {
49682 assert(N->getOpcode() == ISD::AND && "Unexpected opcode!");
49683
49684 EVT VT = N->getValueType(0);
49685
49686 // Make sure this is an AND with constant. We will check the value of the
49687 // constant later.
49688 auto *C1 = dyn_cast<ConstantSDNode>(N->getOperand(1));
49689 if (!C1)
49690 return SDValue();
49691
49692 // This is implied by the ConstantSDNode.
49693 assert(!VT.isVector() && "Expected scalar VT!");
49694
49695 SDValue Src = N->getOperand(0);
49696 if (!Src.hasOneUse())
49697 return SDValue();
49698
49699 // (Optionally) peek through any_extend().
49700 if (Src.getOpcode() == ISD::ANY_EXTEND) {
49701 if (!Src.getOperand(0).hasOneUse())
49702 return SDValue();
49703 Src = Src.getOperand(0);
49704 }
49705
49706 if (Src.getOpcode() != ISD::BITCAST || !Src.getOperand(0).hasOneUse())
49707 return SDValue();
49708
49709 Src = Src.getOperand(0);
49710 EVT SrcVT = Src.getValueType();
49711
49712 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49713 if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::i1 ||
49714 !TLI.isTypeLegal(SrcVT))
49715 return SDValue();
49716
49717 if (Src.getOpcode() != ISD::CONCAT_VECTORS)
49718 return SDValue();
49719
49720 // We only care about the first subvector of the concat, we expect the
49721 // other subvectors to be ignored due to the AND if we make the change.
49722 SDValue SubVec = Src.getOperand(0);
49723 EVT SubVecVT = SubVec.getValueType();
49724
49725 // The RHS of the AND should be a mask with as many bits as SubVec.
49726 if (!TLI.isTypeLegal(SubVecVT) ||
49727 !C1->getAPIntValue().isMask(SubVecVT.getVectorNumElements()))
49728 return SDValue();
49729
49730 // First subvector should be a setcc with a legal result type or a
49731 // AND containing at least one setcc with a legal result type.
49732 auto IsLegalSetCC = [&](SDValue V) {
49733 if (V.getOpcode() != ISD::SETCC)
49734 return false;
49735 EVT SetccVT = V.getOperand(0).getValueType();
49736 if (!TLI.isTypeLegal(SetccVT) ||
49737 !(Subtarget.hasVLX() || SetccVT.is512BitVector()))
49738 return false;
49739 if (!(Subtarget.hasBWI() || SetccVT.getScalarSizeInBits() >= 32))
49740 return false;
49741 return true;
49742 };
49743 if (!(IsLegalSetCC(SubVec) || (SubVec.getOpcode() == ISD::AND &&
49744 (IsLegalSetCC(SubVec.getOperand(0)) ||
49745 IsLegalSetCC(SubVec.getOperand(1))))))
49746 return SDValue();
49747
49748 // We passed all the checks. Rebuild the concat_vectors with zeroes
49749 // and cast it back to VT.
49750 SDLoc dl(N);
49751 SmallVector<SDValue, 4> Ops(Src.getNumOperands(),
49752 DAG.getConstant(0, dl, SubVecVT));
49753 Ops[0] = SubVec;
49754 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
49755 Ops);
49756 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcVT.getSizeInBits());
49757 return DAG.getZExtOrTrunc(DAG.getBitcast(IntVT, Concat), dl, VT);
49758}
49759
49760static SDValue getBMIMatchingOp(unsigned Opc, SelectionDAG &DAG,
49761 SDValue OpMustEq, SDValue Op, unsigned Depth) {
49762 // We don't want to go crazy with the recursion here. This isn't a super
49763 // important optimization.
49764 static constexpr unsigned kMaxDepth = 2;
49765
49766 // Only do this re-ordering if op has one use.
49767 if (!Op.hasOneUse())
49768 return SDValue();
49769
49770 SDLoc DL(Op);
49771 // If we hit another assosiative op, recurse further.
49772 if (Op.getOpcode() == Opc) {
49773 // Done recursing.
49774 if (Depth++ >= kMaxDepth)
49775 return SDValue();
49776
49777 for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx)
49778 if (SDValue R =
49779 getBMIMatchingOp(Opc, DAG, OpMustEq, Op.getOperand(OpIdx), Depth))
49780 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), R,
49781 Op.getOperand(1 - OpIdx));
49782
49783 } else if (Op.getOpcode() == ISD::SUB) {
49784 if (Opc == ISD::AND) {
49785 // BLSI: (and x, (sub 0, x))
49786 if (isNullConstant(Op.getOperand(0)) && Op.getOperand(1) == OpMustEq)
49787 return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
49788 }
49789 // Opc must be ISD::AND or ISD::XOR
49790 // BLSR: (and x, (sub x, 1))
49791 // BLSMSK: (xor x, (sub x, 1))
49792 if (isOneConstant(Op.getOperand(1)) && Op.getOperand(0) == OpMustEq)
49793 return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
49794
49795 } else if (Op.getOpcode() == ISD::ADD) {
49796 // Opc must be ISD::AND or ISD::XOR
49797 // BLSR: (and x, (add x, -1))
49798 // BLSMSK: (xor x, (add x, -1))
49799 if (isAllOnesConstant(Op.getOperand(1)) && Op.getOperand(0) == OpMustEq)
49800 return DAG.getNode(Opc, DL, Op.getValueType(), OpMustEq, Op);
49801 }
49802 return SDValue();
49803}
49804
49805static SDValue combineBMILogicOp(SDNode *N, SelectionDAG &DAG,
49806 const X86Subtarget &Subtarget) {
49807 EVT VT = N->getValueType(0);
49808 // Make sure this node is a candidate for BMI instructions.
49809 if (!Subtarget.hasBMI() || !VT.isScalarInteger() ||
49810 (VT != MVT::i32 && VT != MVT::i64))
49811 return SDValue();
49812
49813 assert(N->getOpcode() == ISD::AND || N->getOpcode() == ISD::XOR);
49814
49815 // Try and match LHS and RHS.
49816 for (unsigned OpIdx = 0; OpIdx < 2; ++OpIdx)
49817 if (SDValue OpMatch =
49818 getBMIMatchingOp(N->getOpcode(), DAG, N->getOperand(OpIdx),
49819 N->getOperand(1 - OpIdx), 0))
49820 return OpMatch;
49821 return SDValue();
49822}
49823
49824static SDValue combineX86SubCmpForFlags(SDNode *N, SDValue Flag,
49825 SelectionDAG &DAG,
49826 TargetLowering::DAGCombinerInfo &DCI,
49827 const X86Subtarget &ST) {
49828 // cmp(setcc(cc, X), 0)
49829 // brcond ne
49830 // ->
49831 // X
49832 // brcond cc
49833
49834 // sub(setcc(cc, X), 1)
49835 // brcond ne
49836 // ->
49837 // X
49838 // brcond ~cc
49839 //
49840 // if only flag has users
49841
49842 SDValue SetCC = N->getOperand(0);
49843
49844 if (SetCC.getOpcode() != X86ISD::SETCC || !Flag.hasOneUse())
49845 return SDValue();
49846
49847 // Check the only user of flag is `brcond ne`.
49848 SDNode *BrCond = *Flag->uses().begin();
49849 if (BrCond->getOpcode() != X86ISD::BRCOND)
49850 return SDValue();
49851 unsigned CondNo = 2;
49852 if (static_cast<X86::CondCode>(BrCond->getConstantOperandVal(CondNo)) !=
49853 X86::COND_NE)
49854 return SDValue();
49855
49856 SDValue X = SetCC.getOperand(1);
49857 // sub has two results while X only have one. DAG combine assumes the value
49858 // type matches.
49859 if (N->getOpcode() == X86ISD::SUB)
49860 X = DAG.getMergeValues({N->getOperand(0), X}, SDLoc(N));
49861
49862 SDValue CCN = SetCC.getOperand(0);
49863 X86::CondCode CC =
49864 static_cast<X86::CondCode>(CCN->getAsAPIntVal().getSExtValue());
49865 X86::CondCode OppositeCC = X86::GetOppositeBranchCondition(CC);
49866 // Update CC for the consumer of the flag.
49867 // The old CC is `ne`. Hence, when comparing the result with 0, we are
49868 // checking if the second condition evaluates to true. When comparing the
49869 // result with 1, we are checking uf the second condition evaluates to false.
49870 SmallVector<SDValue> Ops(BrCond->op_values());
49871 if (isNullConstant(N->getOperand(1)))
49872 Ops[CondNo] = CCN;
49873 else if (isOneConstant(N->getOperand(1)))
49874 Ops[CondNo] = DAG.getTargetConstant(OppositeCC, SDLoc(BrCond), MVT::i8);
49875 else
49876 llvm_unreachable("expect constant 0 or 1");
49877
49878 SDValue NewBrCond =
49879 DAG.getNode(X86ISD::BRCOND, SDLoc(BrCond), BrCond->getValueType(0), Ops);
49880 // Avoid self-assign error b/c CC1 can be `e/ne`.
49881 if (BrCond != NewBrCond.getNode())
49882 DCI.CombineTo(BrCond, NewBrCond);
49883 return X;
49884}
49885
49886static SDValue combineAndOrForCcmpCtest(SDNode *N, SelectionDAG &DAG,
49887 TargetLowering::DAGCombinerInfo &DCI,
49888 const X86Subtarget &ST) {
49889 // and/or(setcc(cc0, flag0), setcc(cc1, sub (X, Y)))
49890 // ->
49891 // setcc(cc1, ccmp(X, Y, ~cflags/cflags, cc0/~cc0, flag0))
49892
49893 // and/or(setcc(cc0, flag0), setcc(cc1, cmp (X, 0)))
49894 // ->
49895 // setcc(cc1, ctest(X, X, ~cflags/cflags, cc0/~cc0, flag0))
49896 //
49897 // where cflags is determined by cc1.
49898
49899 if (!ST.hasCCMP())
49900 return SDValue();
49901
49902 SDValue SetCC0 = N->getOperand(0);
49903 SDValue SetCC1 = N->getOperand(1);
49904 if (SetCC0.getOpcode() != X86ISD::SETCC ||
49905 SetCC1.getOpcode() != X86ISD::SETCC)
49906 return SDValue();
49907
49908 auto GetCombineToOpc = [&](SDValue V) -> unsigned {
49909 SDValue Op = V.getOperand(1);
49910 unsigned Opc = Op.getOpcode();
49911 if (Opc == X86ISD::SUB)
49912 return X86ISD::CCMP;
49913 if (Opc == X86ISD::CMP && isNullConstant(Op.getOperand(1)))
49914 return X86ISD::CTEST;
49915 return 0U;
49916 };
49917
49918 unsigned NewOpc = 0;
49919
49920 // AND/OR is commutable. Canonicalize the operands to make SETCC with SUB/CMP
49921 // appear on the right.
49922 if (!(NewOpc = GetCombineToOpc(SetCC1))) {
49923 std::swap(SetCC0, SetCC1);
49924 if (!(NewOpc = GetCombineToOpc(SetCC1)))
49925 return SDValue();
49926 }
49927
49928 X86::CondCode CC0 =
49929 static_cast<X86::CondCode>(SetCC0.getConstantOperandVal(0));
49930 // CCMP/CTEST is not conditional when the source condition is COND_P/COND_NP.
49931 if (CC0 == X86::COND_P || CC0 == X86::COND_NP)
49932 return SDValue();
49933
49934 bool IsOR = N->getOpcode() == ISD::OR;
49935
49936 // CMP/TEST is executed and updates the EFLAGS normally only when SrcCC
49937 // evaluates to true. So we need to inverse CC0 as SrcCC when the logic
49938 // operator is OR. Similar for CC1.
49939 SDValue SrcCC =
49940 IsOR ? DAG.getTargetConstant(X86::GetOppositeBranchCondition(CC0),
49941 SDLoc(SetCC0.getOperand(0)), MVT::i8)
49942 : SetCC0.getOperand(0);
49943 SDValue CC1N = SetCC1.getOperand(0);
49944 X86::CondCode CC1 =
49945 static_cast<X86::CondCode>(CC1N->getAsAPIntVal().getSExtValue());
49946 X86::CondCode OppositeCC1 = X86::GetOppositeBranchCondition(CC1);
49947 X86::CondCode CFlagsCC = IsOR ? CC1 : OppositeCC1;
49948 SDLoc DL(N);
49949 SDValue CFlags = DAG.getTargetConstant(
49950 X86::getCCMPCondFlagsFromCondCode(CFlagsCC), DL, MVT::i8);
49951 SDValue Sub = SetCC1.getOperand(1);
49952
49953 // Replace any uses of the old flag produced by SUB/CMP with the new one
49954 // produced by CCMP/CTEST.
49955 SDValue CCMP = (NewOpc == X86ISD::CCMP)
49956 ? DAG.getNode(X86ISD::CCMP, DL, MVT::i32,
49957 {Sub.getOperand(0), Sub.getOperand(1),
49958 CFlags, SrcCC, SetCC0.getOperand(1)})
49959 : DAG.getNode(X86ISD::CTEST, DL, MVT::i32,
49960 {Sub.getOperand(0), Sub.getOperand(0),
49961 CFlags, SrcCC, SetCC0.getOperand(1)});
49962
49963 return DAG.getNode(X86ISD::SETCC, DL, MVT::i8, {CC1N, CCMP});
49964}
49965
49966static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
49967 TargetLowering::DAGCombinerInfo &DCI,
49968 const X86Subtarget &Subtarget) {
49969 SDValue N0 = N->getOperand(0);
49970 SDValue N1 = N->getOperand(1);
49971 EVT VT = N->getValueType(0);
49972 SDLoc dl(N);
49973 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
49974
49975 // If this is SSE1 only convert to FAND to avoid scalarization.
49976 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
49977 return DAG.getBitcast(MVT::v4i32,
49978 DAG.getNode(X86ISD::FAND, dl, MVT::v4f32,
49979 DAG.getBitcast(MVT::v4f32, N0),
49980 DAG.getBitcast(MVT::v4f32, N1)));
49981 }
49982
49983 // Use a 32-bit and+zext if upper bits known zero.
49984 if (VT == MVT::i64 && Subtarget.is64Bit() && !isa<ConstantSDNode>(N1)) {
49985 APInt HiMask = APInt::getHighBitsSet(64, 32);
49986 if (DAG.MaskedValueIsZero(N1, HiMask) ||
49987 DAG.MaskedValueIsZero(N0, HiMask)) {
49988 SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0);
49989 SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1);
49990 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64,
49991 DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS));
49992 }
49993 }
49994
49995 // Match all-of bool scalar reductions into a bitcast/movmsk + cmp.
49996 // TODO: Support multiple SrcOps.
49997 if (VT == MVT::i1) {
49998 SmallVector<SDValue, 2> SrcOps;
49999 SmallVector<APInt, 2> SrcPartials;
50000 if (matchScalarReduction(SDValue(N, 0), ISD::AND, SrcOps, &SrcPartials) &&
50001 SrcOps.size() == 1) {
50002 unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
50003 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
50004 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
50005 if (!Mask && TLI.isTypeLegal(SrcOps[0].getValueType()))
50006 Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
50007 if (Mask) {
50008 assert(SrcPartials[0].getBitWidth() == NumElts &&
50009 "Unexpected partial reduction mask");
50010 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
50011 Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits);
50012 return DAG.getSetCC(dl, MVT::i1, Mask, PartialBits, ISD::SETEQ);
50013 }
50014 }
50015 }
50016
50017 // InstCombine converts:
50018 // `(-x << C0) & C1`
50019 // to
50020 // `(x * (Pow2_Ceil(C1) - (1 << C0))) & C1`
50021 // This saves an IR instruction but on x86 the neg/shift version is preferable
50022 // so undo the transform.
50023
50024 if (N0.getOpcode() == ISD::MUL && N0.hasOneUse()) {
50025 // TODO: We don't actually need a splat for this, we just need the checks to
50026 // hold for each element.
50027 ConstantSDNode *N1C = isConstOrConstSplat(N1, /*AllowUndefs*/ true,
50028 /*AllowTruncation*/ false);
50029 ConstantSDNode *N01C =
50030 isConstOrConstSplat(N0.getOperand(1), /*AllowUndefs*/ true,
50031 /*AllowTruncation*/ false);
50032 if (N1C && N01C) {
50033 const APInt &MulC = N01C->getAPIntValue();
50034 const APInt &AndC = N1C->getAPIntValue();
50035 APInt MulCLowBit = MulC & (-MulC);
50036 if (MulC.uge(AndC) && !MulC.isPowerOf2() &&
50037 (MulCLowBit + MulC).isPowerOf2()) {
50038 SDValue Neg = DAG.getNegative(N0.getOperand(0), dl, VT);
50039 int32_t MulCLowBitLog = MulCLowBit.exactLogBase2();
50040 assert(MulCLowBitLog != -1 &&
50041 "Isolated lowbit is somehow not a power of 2!");
50042 SDValue Shift = DAG.getNode(ISD::SHL, dl, VT, Neg,
50043 DAG.getConstant(MulCLowBitLog, dl, VT));
50044 return DAG.getNode(ISD::AND, dl, VT, Shift, N1);
50045 }
50046 }
50047 }
50048
50049 if (SDValue SetCC = combineAndOrForCcmpCtest(N, DAG, DCI, Subtarget))
50050 return SetCC;
50051
50052 if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget))
50053 return V;
50054
50055 if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
50056 return R;
50057
50058 if (SDValue R = combineBitOpWithShift(N, DAG))
50059 return R;
50060
50061 if (SDValue R = combineBitOpWithPACK(N, DAG))
50062 return R;
50063
50064 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
50065 return FPLogic;
50066
50067 if (SDValue R = combineAndShuffleNot(N, DAG, Subtarget))
50068 return R;
50069
50070 if (DCI.isBeforeLegalizeOps())
50071 return SDValue();
50072
50073 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
50074 return R;
50075
50076 if (SDValue R = combineAndNotIntoANDNP(N, DAG))
50077 return R;
50078
50079 if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget))
50080 return ShiftRight;
50081
50082 if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
50083 return R;
50084
50085 // fold (and (mul x, c1), c2) -> (mul x, (and c1, c2))
50086 // iff c2 is all/no bits mask - i.e. a select-with-zero mask.
50087 // TODO: Handle PMULDQ/PMULUDQ/VPMADDWD/VPMADDUBSW?
50088 if (VT.isVector() && getTargetConstantFromNode(N1)) {
50089 unsigned Opc0 = N0.getOpcode();
50090 if ((Opc0 == ISD::MUL || Opc0 == ISD::MULHU || Opc0 == ISD::MULHS) &&
50091 getTargetConstantFromNode(N0.getOperand(1)) &&
50092 DAG.ComputeNumSignBits(N1) == VT.getScalarSizeInBits() &&
50093 N0->hasOneUse() && N0.getOperand(1)->hasOneUse()) {
50094 SDValue MaskMul = DAG.getNode(ISD::AND, dl, VT, N0.getOperand(1), N1);
50095 return DAG.getNode(Opc0, dl, VT, N0.getOperand(0), MaskMul);
50096 }
50097 }
50098
50099 // Fold AND(SRL(X,Y),1) -> SETCC(BT(X,Y), COND_B) iff Y is not a constant
50100 // avoids slow variable shift (moving shift amount to ECX etc.)
50101 if (isOneConstant(N1) && N0->hasOneUse()) {
50102 SDValue Src = N0;
50103 while ((Src.getOpcode() == ISD::ZERO_EXTEND ||
50104 Src.getOpcode() == ISD::TRUNCATE) &&
50105 Src.getOperand(0)->hasOneUse())
50106 Src = Src.getOperand(0);
50107 bool ContainsNOT = false;
50108 X86::CondCode X86CC = X86::COND_B;
50109 // Peek through AND(NOT(SRL(X,Y)),1).
50110 if (isBitwiseNot(Src)) {
50111 Src = Src.getOperand(0);
50112 X86CC = X86::COND_AE;
50113 ContainsNOT = true;
50114 }
50115 if (Src.getOpcode() == ISD::SRL &&
50116 !isa<ConstantSDNode>(Src.getOperand(1))) {
50117 SDValue BitNo = Src.getOperand(1);
50118 Src = Src.getOperand(0);
50119 // Peek through AND(SRL(NOT(X),Y),1).
50120 if (isBitwiseNot(Src)) {
50121 Src = Src.getOperand(0);
50122 X86CC = X86CC == X86::COND_AE ? X86::COND_B : X86::COND_AE;
50123 ContainsNOT = true;
50124 }
50125 // If we have BMI2 then SHRX should be faster for i32/i64 cases.
50126 if (!(Subtarget.hasBMI2() && !ContainsNOT && VT.getSizeInBits() >= 32))
50127 if (SDValue BT = getBT(Src, BitNo, dl, DAG))
50128 return DAG.getZExtOrTrunc(getSETCC(X86CC, BT, dl, DAG), dl, VT);
50129 }
50130 }
50131
50132 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
50133 // Attempt to recursively combine a bitmask AND with shuffles.
50134 SDValue Op(N, 0);
50135 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
50136 return Res;
50137
50138 // If either operand is a constant mask, then only the elements that aren't
50139 // zero are actually demanded by the other operand.
50140 auto GetDemandedMasks = [&](SDValue Op) {
50141 APInt UndefElts;
50142 SmallVector<APInt> EltBits;
50143 int NumElts = VT.getVectorNumElements();
50144 int EltSizeInBits = VT.getScalarSizeInBits();
50145 APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
50146 APInt DemandedElts = APInt::getAllOnes(NumElts);
50147 if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
50148 EltBits)) {
50149 DemandedBits.clearAllBits();
50150 DemandedElts.clearAllBits();
50151 for (int I = 0; I != NumElts; ++I) {
50152 if (UndefElts[I]) {
50153 // We can't assume an undef src element gives an undef dst - the
50154 // other src might be zero.
50155 DemandedBits.setAllBits();
50156 DemandedElts.setBit(I);
50157 } else if (!EltBits[I].isZero()) {
50158 DemandedBits |= EltBits[I];
50159 DemandedElts.setBit(I);
50160 }
50161 }
50162 }
50163 return std::make_pair(DemandedBits, DemandedElts);
50164 };
50165 APInt Bits0, Elts0;
50166 APInt Bits1, Elts1;
50167 std::tie(Bits0, Elts0) = GetDemandedMasks(N1);
50168 std::tie(Bits1, Elts1) = GetDemandedMasks(N0);
50169
50170 if (TLI.SimplifyDemandedVectorElts(N0, Elts0, DCI) ||
50171 TLI.SimplifyDemandedVectorElts(N1, Elts1, DCI) ||
50172 TLI.SimplifyDemandedBits(N0, Bits0, Elts0, DCI) ||
50173 TLI.SimplifyDemandedBits(N1, Bits1, Elts1, DCI)) {
50174 if (N->getOpcode() != ISD::DELETED_NODE)
50175 DCI.AddToWorklist(N);
50176 return SDValue(N, 0);
50177 }
50178
50179 SDValue NewN0 = TLI.SimplifyMultipleUseDemandedBits(N0, Bits0, Elts0, DAG);
50180 SDValue NewN1 = TLI.SimplifyMultipleUseDemandedBits(N1, Bits1, Elts1, DAG);
50181 if (NewN0 || NewN1)
50182 return DAG.getNode(ISD::AND, dl, VT, NewN0 ? NewN0 : N0,
50183 NewN1 ? NewN1 : N1);
50184 }
50185
50186 // Attempt to combine a scalar bitmask AND with an extracted shuffle.
50187 if ((VT.getScalarSizeInBits() % 8) == 0 &&
50188 N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
50189 isa<ConstantSDNode>(N0.getOperand(1)) && N0->hasOneUse()) {
50190 SDValue BitMask = N1;
50191 SDValue SrcVec = N0.getOperand(0);
50192 EVT SrcVecVT = SrcVec.getValueType();
50193
50194 // Check that the constant bitmask masks whole bytes.
50195 APInt UndefElts;
50196 SmallVector<APInt, 64> EltBits;
50197 if (VT == SrcVecVT.getScalarType() && N0->isOnlyUserOf(SrcVec.getNode()) &&
50198 getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) &&
50199 llvm::all_of(EltBits, [](const APInt &M) {
50200 return M.isZero() || M.isAllOnes();
50201 })) {
50202 unsigned NumElts = SrcVecVT.getVectorNumElements();
50203 unsigned Scale = SrcVecVT.getScalarSizeInBits() / 8;
50204 unsigned Idx = N0.getConstantOperandVal(1);
50205
50206 // Create a root shuffle mask from the byte mask and the extracted index.
50207 SmallVector<int, 16> ShuffleMask(NumElts * Scale, SM_SentinelUndef);
50208 for (unsigned i = 0; i != Scale; ++i) {
50209 if (UndefElts[i])
50210 continue;
50211 int VecIdx = Scale * Idx + i;
50212 ShuffleMask[VecIdx] = EltBits[i].isZero() ? SM_SentinelZero : VecIdx;
50213 }
50214
50215 if (SDValue Shuffle = combineX86ShufflesRecursively(
50216 {SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1,
50217 X86::MaxShuffleCombineDepth,
50218 /*HasVarMask*/ false, /*AllowVarCrossLaneMask*/ true,
50219 /*AllowVarPerLaneMask*/ true, DAG, Subtarget))
50220 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle,
50221 N0.getOperand(1));
50222 }
50223 }
50224
50225 if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
50226 return R;
50227
50228 return SDValue();
50229}
50230
50231// Canonicalize OR(AND(X,C),AND(Y,~C)) -> OR(AND(X,C),ANDNP(C,Y))
50232static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
50233 const X86Subtarget &Subtarget) {
50234 assert(N->getOpcode() == ISD::OR && "Unexpected Opcode");
50235
50236 MVT VT = N->getSimpleValueType(0);
50237 unsigned EltSizeInBits = VT.getScalarSizeInBits();
50238 if (!VT.isVector() || (EltSizeInBits % 8) != 0)
50239 return SDValue();
50240
50241 SDValue N0 = peekThroughBitcasts(N->getOperand(0));
50242 SDValue N1 = peekThroughBitcasts(N->getOperand(1));
50243 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
50244 return SDValue();
50245
50246 // On XOP we'll lower to PCMOV so accept one use. With AVX512, we can use
50247 // VPTERNLOG. Otherwise only do this if either mask has multiple uses already.
50248 if (!(Subtarget.hasXOP() || useVPTERNLOG(Subtarget, VT) ||
50249 !N0.getOperand(1).hasOneUse() || !N1.getOperand(1).hasOneUse()))
50250 return SDValue();
50251
50252 // Attempt to extract constant byte masks.
50253 APInt UndefElts0, UndefElts1;
50254 SmallVector<APInt, 32> EltBits0, EltBits1;
50255 if (!getTargetConstantBitsFromNode(N0.getOperand(1), 8, UndefElts0, EltBits0,
50256 /*AllowWholeUndefs*/ false,
50257 /*AllowPartialUndefs*/ false))
50258 return SDValue();
50259 if (!getTargetConstantBitsFromNode(N1.getOperand(1), 8, UndefElts1, EltBits1,
50260 /*AllowWholeUndefs*/ false,
50261 /*AllowPartialUndefs*/ false))
50262 return SDValue();
50263
50264 for (unsigned i = 0, e = EltBits0.size(); i != e; ++i) {
50265 // TODO - add UNDEF elts support.
50266 if (UndefElts0[i] || UndefElts1[i])
50267 return SDValue();
50268 if (EltBits0[i] != ~EltBits1[i])
50269 return SDValue();
50270 }
50271
50272 SDLoc DL(N);
50273
50274 if (useVPTERNLOG(Subtarget, VT)) {
50275 // Emit a VPTERNLOG node directly - 0xCA is the imm code for A?B:C.
50276 // VPTERNLOG is only available as vXi32/64-bit types.
50277 MVT OpSVT = EltSizeInBits <= 32 ? MVT::i32 : MVT::i64;
50278 MVT OpVT =
50279 MVT::getVectorVT(OpSVT, VT.getSizeInBits() / OpSVT.getSizeInBits());
50280 SDValue A = DAG.getBitcast(OpVT, N0.getOperand(1));
50281 SDValue B = DAG.getBitcast(OpVT, N0.getOperand(0));
50282 SDValue C = DAG.getBitcast(OpVT, N1.getOperand(0));
50283 SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8);
50284 SDValue Res = getAVX512Node(X86ISD::VPTERNLOG, DL, OpVT, {A, B, C, Imm},
50285 DAG, Subtarget);
50286 return DAG.getBitcast(VT, Res);
50287 }
50288
50289 SDValue X = N->getOperand(0);
50290 SDValue Y =
50291 DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)),
50292 DAG.getBitcast(VT, N1.getOperand(0)));
50293 return DAG.getNode(ISD::OR, DL, VT, X, Y);
50294}
50295
50296// Try to match OR(AND(~MASK,X),AND(MASK,Y)) logic pattern.
50297static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
50298 if (N->getOpcode() != ISD::OR)
50299 return false;
50300
50301 SDValue N0 = N->getOperand(0);
50302 SDValue N1 = N->getOperand(1);
50303
50304 // Canonicalize AND to LHS.
50305 if (N1.getOpcode() == ISD::AND)
50306 std::swap(N0, N1);
50307
50308 // Attempt to match OR(AND(M,Y),ANDNP(M,X)).
50309 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != X86ISD::ANDNP)
50310 return false;
50311
50312 Mask = N1.getOperand(0);
50313 X = N1.getOperand(1);
50314
50315 // Check to see if the mask appeared in both the AND and ANDNP.
50316 if (N0.getOperand(0) == Mask)
50317 Y = N0.getOperand(1);
50318 else if (N0.getOperand(1) == Mask)
50319 Y = N0.getOperand(0);
50320 else
50321 return false;
50322
50323 // TODO: Attempt to match against AND(XOR(-1,M),Y) as well, waiting for
50324 // ANDNP combine allows other combines to happen that prevent matching.
50325 return true;
50326}
50327
50328// Try to fold:
50329// (or (and (m, y), (pandn m, x)))
50330// into:
50331// (vselect m, x, y)
50332// As a special case, try to fold:
50333// (or (and (m, (sub 0, x)), (pandn m, x)))
50334// into:
50335// (sub (xor X, M), M)
50336static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG,
50337 const X86Subtarget &Subtarget) {
50338 assert(N->getOpcode() == ISD::OR && "Unexpected Opcode");
50339
50340 EVT VT = N->getValueType(0);
50341 if (!((VT.is128BitVector() && Subtarget.hasSSE2()) ||
50342 (VT.is256BitVector() && Subtarget.hasInt256())))
50343 return SDValue();
50344
50345 SDValue X, Y, Mask;
50346 if (!matchLogicBlend(N, X, Y, Mask))
50347 return SDValue();
50348
50349 // Validate that X, Y, and Mask are bitcasts, and see through them.
50350 Mask = peekThroughBitcasts(Mask);
50351 X = peekThroughBitcasts(X);
50352 Y = peekThroughBitcasts(Y);
50353
50354 EVT MaskVT = Mask.getValueType();
50355 unsigned EltBits = MaskVT.getScalarSizeInBits();
50356
50357 // TODO: Attempt to handle floating point cases as well?
50358 if (!MaskVT.isInteger() || DAG.ComputeNumSignBits(Mask) != EltBits)
50359 return SDValue();
50360
50361 SDLoc DL(N);
50362
50363 // Attempt to combine to conditional negate: (sub (xor X, M), M)
50364 if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL,
50365 DAG, Subtarget))
50366 return Res;
50367
50368 // PBLENDVB is only available on SSE 4.1.
50369 if (!Subtarget.hasSSE41())
50370 return SDValue();
50371
50372 // If we have VPTERNLOG we should prefer that since PBLENDVB is multiple uops.
50373 if (Subtarget.hasVLX())
50374 return SDValue();
50375
50376 MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8;
50377
50378 X = DAG.getBitcast(BlendVT, X);
50379 Y = DAG.getBitcast(BlendVT, Y);
50380 Mask = DAG.getBitcast(BlendVT, Mask);
50381 Mask = DAG.getSelect(DL, BlendVT, Mask, Y, X);
50382 return DAG.getBitcast(VT, Mask);
50383}
50384
50385// Helper function for combineOrCmpEqZeroToCtlzSrl
50386// Transforms:
50387// seteq(cmp x, 0)
50388// into:
50389// srl(ctlz x), log2(bitsize(x))
50390// Input pattern is checked by caller.
50391static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) {
50392 SDValue Cmp = Op.getOperand(1);
50393 EVT VT = Cmp.getOperand(0).getValueType();
50394 unsigned Log2b = Log2_32(VT.getSizeInBits());
50395 SDLoc dl(Op);
50396 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0));
50397 // The result of the shift is true or false, and on X86, the 32-bit
50398 // encoding of shr and lzcnt is more desirable.
50399 SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);
50400 SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,
50401 DAG.getConstant(Log2b, dl, MVT::i8));
50402 return Scc;
50403}
50404
50405// Try to transform:
50406// zext(or(setcc(eq, (cmp x, 0)), setcc(eq, (cmp y, 0))))
50407// into:
50408// srl(or(ctlz(x), ctlz(y)), log2(bitsize(x))
50409// Will also attempt to match more generic cases, eg:
50410// zext(or(or(setcc(eq, cmp 0), setcc(eq, cmp 0)), setcc(eq, cmp 0)))
50411// Only applies if the target supports the FastLZCNT feature.
50412static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
50413 TargetLowering::DAGCombinerInfo &DCI,
50414 const X86Subtarget &Subtarget) {
50415 if (DCI.isBeforeLegalize() || !Subtarget.getTargetLowering()->isCtlzFast())
50416 return SDValue();
50417
50418 auto isORCandidate = [](SDValue N) {
50419 return (N->getOpcode() == ISD::OR && N->hasOneUse());
50420 };
50421
50422 // Check the zero extend is extending to 32-bit or more. The code generated by
50423 // srl(ctlz) for 16-bit or less variants of the pattern would require extra
50424 // instructions to clear the upper bits.
50425 if (!N->hasOneUse() || !N->getSimpleValueType(0).bitsGE(MVT::i32) ||
50426 !isORCandidate(N->getOperand(0)))
50427 return SDValue();
50428
50429 // Check the node matches: setcc(eq, cmp 0)
50430 auto isSetCCCandidate = [](SDValue N) {
50431 return N->getOpcode() == X86ISD::SETCC && N->hasOneUse() &&
50432 X86::CondCode(N->getConstantOperandVal(0)) == X86::COND_E &&
50433 N->getOperand(1).getOpcode() == X86ISD::CMP &&
50434 isNullConstant(N->getOperand(1).getOperand(1)) &&
50435 N->getOperand(1).getValueType().bitsGE(MVT::i32);
50436 };
50437
50438 SDNode *OR = N->getOperand(0).getNode();
50439 SDValue LHS = OR->getOperand(0);
50440 SDValue RHS = OR->getOperand(1);
50441
50442 // Save nodes matching or(or, setcc(eq, cmp 0)).
50443 SmallVector<SDNode *, 2> ORNodes;
50444 while (((isORCandidate(LHS) && isSetCCCandidate(RHS)) ||
50445 (isORCandidate(RHS) && isSetCCCandidate(LHS)))) {
50446 ORNodes.push_back(OR);
50447 OR = (LHS->getOpcode() == ISD::OR) ? LHS.getNode() : RHS.getNode();
50448 LHS = OR->getOperand(0);
50449 RHS = OR->getOperand(1);
50450 }
50451
50452 // The last OR node should match or(setcc(eq, cmp 0), setcc(eq, cmp 0)).
50453 if (!(isSetCCCandidate(LHS) && isSetCCCandidate(RHS)) ||
50454 !isORCandidate(SDValue(OR, 0)))
50455 return SDValue();
50456
50457 // We have a or(setcc(eq, cmp 0), setcc(eq, cmp 0)) pattern, try to lower it
50458 // to
50459 // or(srl(ctlz),srl(ctlz)).
50460 // The dag combiner can then fold it into:
50461 // srl(or(ctlz, ctlz)).
50462 SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, DAG);
50463 SDValue Ret, NewRHS;
50464 if (NewLHS && (NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG)))
50465 Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, NewLHS, NewRHS);
50466
50467 if (!Ret)
50468 return SDValue();
50469
50470 // Try to lower nodes matching the or(or, setcc(eq, cmp 0)) pattern.
50471 while (!ORNodes.empty()) {
50472 OR = ORNodes.pop_back_val();
50473 LHS = OR->getOperand(0);
50474 RHS = OR->getOperand(1);
50475 // Swap rhs with lhs to match or(setcc(eq, cmp, 0), or).
50476 if (RHS->getOpcode() == ISD::OR)
50477 std::swap(LHS, RHS);
50478 NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, DAG);
50479 if (!NewRHS)
50480 return SDValue();
50481 Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, Ret, NewRHS);
50482 }
50483
50484 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret);
50485}
50486
50487static SDValue foldMaskedMergeImpl(SDValue And0_L, SDValue And0_R,
50488 SDValue And1_L, SDValue And1_R,
50489 const SDLoc &DL, SelectionDAG &DAG) {
50490 if (!isBitwiseNot(And0_L, true) || !And0_L->hasOneUse())
50491 return SDValue();
50492 SDValue NotOp = And0_L->getOperand(0);
50493 if (NotOp == And1_R)
50494 std::swap(And1_R, And1_L);
50495 if (NotOp != And1_L)
50496 return SDValue();
50497
50498 // (~(NotOp) & And0_R) | (NotOp & And1_R)
50499 // --> ((And0_R ^ And1_R) & NotOp) ^ And1_R
50500 EVT VT = And1_L->getValueType(0);
50501 SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R);
50502 SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R);
50503 SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp);
50504 SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R);
50505 return Xor1;
50506}
50507
50508/// Fold "masked merge" expressions like `(m & x) | (~m & y)` into the
50509/// equivalent `((x ^ y) & m) ^ y)` pattern.
50510/// This is typically a better representation for targets without a fused
50511/// "and-not" operation. This function is intended to be called from a
50512/// `TargetLowering::PerformDAGCombine` callback on `ISD::OR` nodes.
50513static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG) {
50514 // Note that masked-merge variants using XOR or ADD expressions are
50515 // normalized to OR by InstCombine so we only check for OR.
50516 assert(Node->getOpcode() == ISD::OR && "Must be called with ISD::OR node");
50517 SDValue N0 = Node->getOperand(0);
50518 if (N0->getOpcode() != ISD::AND || !N0->hasOneUse())
50519 return SDValue();
50520 SDValue N1 = Node->getOperand(1);
50521 if (N1->getOpcode() != ISD::AND || !N1->hasOneUse())
50522 return SDValue();
50523
50524 SDLoc DL(Node);
50525 SDValue N00 = N0->getOperand(0);
50526 SDValue N01 = N0->getOperand(1);
50527 SDValue N10 = N1->getOperand(0);
50528 SDValue N11 = N1->getOperand(1);
50529 if (SDValue Result = foldMaskedMergeImpl(N00, N01, N10, N11, DL, DAG))
50530 return Result;
50531 if (SDValue Result = foldMaskedMergeImpl(N01, N00, N10, N11, DL, DAG))
50532 return Result;
50533 if (SDValue Result = foldMaskedMergeImpl(N10, N11, N00, N01, DL, DAG))
50534 return Result;
50535 if (SDValue Result = foldMaskedMergeImpl(N11, N10, N00, N01, DL, DAG))
50536 return Result;
50537 return SDValue();
50538}
50539
50540/// If this is an add or subtract where one operand is produced by a cmp+setcc,
50541/// then try to convert it to an ADC or SBB. This replaces TEST+SET+{ADD/SUB}
50542/// with CMP+{ADC, SBB}.
50543/// Also try (ADD/SUB)+(AND(SRL,1)) bit extraction pattern with BT+{ADC, SBB}.
50544static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
50545 SDValue X, SDValue Y,
50546 SelectionDAG &DAG,
50547 bool ZeroSecondOpOnly = false) {
50548 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
50549 return SDValue();
50550
50551 // Look through a one-use zext.
50552 if (Y.getOpcode() == ISD::ZERO_EXTEND && Y.hasOneUse())
50553 Y = Y.getOperand(0);
50554
50555 X86::CondCode CC;
50556 SDValue EFLAGS;
50557 if (Y.getOpcode() == X86ISD::SETCC && Y.hasOneUse()) {
50558 CC = (X86::CondCode)Y.getConstantOperandVal(0);
50559 EFLAGS = Y.getOperand(1);
50560 } else if (Y.getOpcode() == ISD::AND && isOneConstant(Y.getOperand(1)) &&
50561 Y.hasOneUse()) {
50562 EFLAGS = LowerAndToBT(Y, ISD::SETNE, DL, DAG, CC);
50563 }
50564
50565 if (!EFLAGS)
50566 return SDValue();
50567
50568 // If X is -1 or 0, then we have an opportunity to avoid constants required in
50569 // the general case below.
50570 auto *ConstantX = dyn_cast<ConstantSDNode>(X);
50571 if (ConstantX && !ZeroSecondOpOnly) {
50572 if ((!IsSub && CC == X86::COND_AE && ConstantX->isAllOnes()) ||
50573 (IsSub && CC == X86::COND_B && ConstantX->isZero())) {
50574 // This is a complicated way to get -1 or 0 from the carry flag:
50575 // -1 + SETAE --> -1 + (!CF) --> CF ? -1 : 0 --> SBB %eax, %eax
50576 // 0 - SETB --> 0 - (CF) --> CF ? -1 : 0 --> SBB %eax, %eax
50577 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
50578 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
50579 EFLAGS);
50580 }
50581
50582 if ((!IsSub && CC == X86::COND_BE && ConstantX->isAllOnes()) ||
50583 (IsSub && CC == X86::COND_A && ConstantX->isZero())) {
50584 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
50585 EFLAGS.getValueType().isInteger() &&
50586 !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
50587 // Swap the operands of a SUB, and we have the same pattern as above.
50588 // -1 + SETBE (SUB A, B) --> -1 + SETAE (SUB B, A) --> SUB + SBB
50589 // 0 - SETA (SUB A, B) --> 0 - SETB (SUB B, A) --> SUB + SBB
50590 SDValue NewSub = DAG.getNode(
50591 X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
50592 EFLAGS.getOperand(1), EFLAGS.getOperand(0));
50593 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
50594 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
50595 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
50596 NewEFLAGS);
50597 }
50598 }
50599 }
50600
50601 if (CC == X86::COND_B) {
50602 // X + SETB Z --> adc X, 0
50603 // X - SETB Z --> sbb X, 0
50604 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
50605 DAG.getVTList(VT, MVT::i32), X,
50606 DAG.getConstant(0, DL, VT), EFLAGS);
50607 }
50608
50609 if (ZeroSecondOpOnly)
50610 return SDValue();
50611
50612 if (CC == X86::COND_A) {
50613 // Try to convert COND_A into COND_B in an attempt to facilitate
50614 // materializing "setb reg".
50615 //
50616 // Do not flip "e > c", where "c" is a constant, because Cmp instruction
50617 // cannot take an immediate as its first operand.
50618 //
50619 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() &&
50620 EFLAGS.getValueType().isInteger() &&
50621 !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
50622 SDValue NewSub =
50623 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
50624 EFLAGS.getOperand(1), EFLAGS.getOperand(0));
50625 SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
50626 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
50627 DAG.getVTList(VT, MVT::i32), X,
50628 DAG.getConstant(0, DL, VT), NewEFLAGS);
50629 }
50630 }
50631
50632 if (CC == X86::COND_AE) {
50633 // X + SETAE --> sbb X, -1
50634 // X - SETAE --> adc X, -1
50635 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
50636 DAG.getVTList(VT, MVT::i32), X,
50637 DAG.getAllOnesConstant(DL, VT), EFLAGS);
50638 }
50639
50640 if (CC == X86::COND_BE) {
50641 // X + SETBE --> sbb X, -1
50642 // X - SETBE --> adc X, -1
50643 // Try to convert COND_BE into COND_AE in an attempt to facilitate
50644 // materializing "setae reg".
50645 //
50646 // Do not flip "e <= c", where "c" is a constant, because Cmp instruction
50647 // cannot take an immediate as its first operand.
50648 //
50649 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() &&
50650 EFLAGS.getValueType().isInteger() &&
50651 !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
50652 SDValue NewSub =
50653 DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
50654 EFLAGS.getOperand(1), EFLAGS.getOperand(0));
50655 SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
50656 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
50657 DAG.getVTList(VT, MVT::i32), X,
50658 DAG.getAllOnesConstant(DL, VT), NewEFLAGS);
50659 }
50660 }
50661
50662 if (CC != X86::COND_E && CC != X86::COND_NE)
50663 return SDValue();
50664
50665 if (EFLAGS.getOpcode() != X86ISD::CMP || !EFLAGS.hasOneUse() ||
50666 !X86::isZeroNode(EFLAGS.getOperand(1)) ||
50667 !EFLAGS.getOperand(0).getValueType().isInteger())
50668 return SDValue();
50669
50670 SDValue Z = EFLAGS.getOperand(0);
50671 EVT ZVT = Z.getValueType();
50672
50673 // If X is -1 or 0, then we have an opportunity to avoid constants required in
50674 // the general case below.
50675 if (ConstantX) {
50676 // 'neg' sets the carry flag when Z != 0, so create 0 or -1 using 'sbb' with
50677 // fake operands:
50678 // 0 - (Z != 0) --> sbb %eax, %eax, (neg Z)
50679 // -1 + (Z == 0) --> sbb %eax, %eax, (neg Z)
50680 if ((IsSub && CC == X86::COND_NE && ConstantX->isZero()) ||
50681 (!IsSub && CC == X86::COND_E && ConstantX->isAllOnes())) {
50682 SDValue Zero = DAG.getConstant(0, DL, ZVT);
50683 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
50684 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
50685 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
50686 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
50687 SDValue(Neg.getNode(), 1));
50688 }
50689
50690 // cmp with 1 sets the carry flag when Z == 0, so create 0 or -1 using 'sbb'
50691 // with fake operands:
50692 // 0 - (Z == 0) --> sbb %eax, %eax, (cmp Z, 1)
50693 // -1 + (Z != 0) --> sbb %eax, %eax, (cmp Z, 1)
50694 if ((IsSub && CC == X86::COND_E && ConstantX->isZero()) ||
50695 (!IsSub && CC == X86::COND_NE && ConstantX->isAllOnes())) {
50696 SDValue One = DAG.getConstant(1, DL, ZVT);
50697 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
50698 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
50699 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
50700 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
50701 Cmp1.getValue(1));
50702 }
50703 }
50704
50705 // (cmp Z, 1) sets the carry flag if Z is 0.
50706 SDValue One = DAG.getConstant(1, DL, ZVT);
50707 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
50708 SDValue Cmp1 = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Z, One);
50709
50710 // Add the flags type for ADC/SBB nodes.
50711 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
50712
50713 // X - (Z != 0) --> sub X, (zext(setne Z, 0)) --> adc X, -1, (cmp Z, 1)
50714 // X + (Z != 0) --> add X, (zext(setne Z, 0)) --> sbb X, -1, (cmp Z, 1)
50715 if (CC == X86::COND_NE)
50716 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X,
50717 DAG.getAllOnesConstant(DL, VT), Cmp1.getValue(1));
50718
50719 // X - (Z == 0) --> sub X, (zext(sete Z, 0)) --> sbb X, 0, (cmp Z, 1)
50720 // X + (Z == 0) --> add X, (zext(sete Z, 0)) --> adc X, 0, (cmp Z, 1)
50721 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, VTs, X,
50722 DAG.getConstant(0, DL, VT), Cmp1.getValue(1));
50723}
50724
50725/// If this is an add or subtract where one operand is produced by a cmp+setcc,
50726/// then try to convert it to an ADC or SBB. This replaces TEST+SET+{ADD/SUB}
50727/// with CMP+{ADC, SBB}.
50728static SDValue combineAddOrSubToADCOrSBB(SDNode *N, const SDLoc &DL,
50729 SelectionDAG &DAG) {
50730 bool IsSub = N->getOpcode() == ISD::SUB;
50731 SDValue X = N->getOperand(0);
50732 SDValue Y = N->getOperand(1);
50733 EVT VT = N->getValueType(0);
50734
50735 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, X, Y, DAG))
50736 return ADCOrSBB;
50737
50738 // Commute and try again (negate the result for subtracts).
50739 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, Y, X, DAG)) {
50740 if (IsSub)
50741 ADCOrSBB = DAG.getNegative(ADCOrSBB, DL, VT);
50742 return ADCOrSBB;
50743 }
50744
50745 return SDValue();
50746}
50747
50748static SDValue combineOrXorWithSETCC(SDNode *N, SDValue N0, SDValue N1,
50749 SelectionDAG &DAG) {
50750 assert((N->getOpcode() == ISD::XOR || N->getOpcode() == ISD::OR) &&
50751 "Unexpected opcode");
50752
50753 // Delegate to combineAddOrSubToADCOrSBB if we have:
50754 //
50755 // (xor/or (zero_extend (setcc)) imm)
50756 //
50757 // where imm is odd if and only if we have xor, in which case the XOR/OR are
50758 // equivalent to a SUB/ADD, respectively.
50759 if (N0.getOpcode() == ISD::ZERO_EXTEND &&
50760 N0.getOperand(0).getOpcode() == X86ISD::SETCC && N0.hasOneUse()) {
50761 if (auto *N1C = dyn_cast<ConstantSDNode>(N1)) {
50762 bool IsSub = N->getOpcode() == ISD::XOR;
50763 bool N1COdd = N1C->getZExtValue() & 1;
50764 if (IsSub ? N1COdd : !N1COdd) {
50765 SDLoc DL(N);
50766 EVT VT = N->getValueType(0);
50767 if (SDValue R = combineAddOrSubToADCOrSBB(IsSub, DL, VT, N1, N0, DAG))
50768 return R;
50769 }
50770 }
50771 }
50772
50773 // not(pcmpeq(and(X,CstPow2),0)) -> pcmpeq(and(X,CstPow2),CstPow2)
50774 if (N->getOpcode() == ISD::XOR && N0.getOpcode() == X86ISD::PCMPEQ &&
50775 N0.getOperand(0).getOpcode() == ISD::AND &&
50776 ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode()) &&
50777 ISD::isBuildVectorAllOnes(N1.getNode())) {
50778 MVT VT = N->getSimpleValueType(0);
50779 APInt UndefElts;
50780 SmallVector<APInt> EltBits;
50781 if (getTargetConstantBitsFromNode(N0.getOperand(0).getOperand(1),
50782 VT.getScalarSizeInBits(), UndefElts,
50783 EltBits)) {
50784 bool IsPow2OrUndef = true;
50785 for (unsigned I = 0, E = EltBits.size(); I != E; ++I)
50786 IsPow2OrUndef &= UndefElts[I] || EltBits[I].isPowerOf2();
50787
50788 if (IsPow2OrUndef)
50789 return DAG.getNode(X86ISD::PCMPEQ, SDLoc(N), VT, N0.getOperand(0),
50790 N0.getOperand(0).getOperand(1));
50791 }
50792 }
50793
50794 return SDValue();
50795}
50796
50797static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
50798 TargetLowering::DAGCombinerInfo &DCI,
50799 const X86Subtarget &Subtarget) {
50800 SDValue N0 = N->getOperand(0);
50801 SDValue N1 = N->getOperand(1);
50802 EVT VT = N->getValueType(0);
50803 SDLoc dl(N);
50804 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50805
50806 // If this is SSE1 only convert to FOR to avoid scalarization.
50807 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
50808 return DAG.getBitcast(MVT::v4i32,
50809 DAG.getNode(X86ISD::FOR, dl, MVT::v4f32,
50810 DAG.getBitcast(MVT::v4f32, N0),
50811 DAG.getBitcast(MVT::v4f32, N1)));
50812 }
50813
50814 // Match any-of bool scalar reductions into a bitcast/movmsk + cmp.
50815 // TODO: Support multiple SrcOps.
50816 if (VT == MVT::i1) {
50817 SmallVector<SDValue, 2> SrcOps;
50818 SmallVector<APInt, 2> SrcPartials;
50819 if (matchScalarReduction(SDValue(N, 0), ISD::OR, SrcOps, &SrcPartials) &&
50820 SrcOps.size() == 1) {
50821 unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
50822 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
50823 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
50824 if (!Mask && TLI.isTypeLegal(SrcOps[0].getValueType()))
50825 Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
50826 if (Mask) {
50827 assert(SrcPartials[0].getBitWidth() == NumElts &&
50828 "Unexpected partial reduction mask");
50829 SDValue ZeroBits = DAG.getConstant(0, dl, MaskVT);
50830 SDValue PartialBits = DAG.getConstant(SrcPartials[0], dl, MaskVT);
50831 Mask = DAG.getNode(ISD::AND, dl, MaskVT, Mask, PartialBits);
50832 return DAG.getSetCC(dl, MVT::i1, Mask, ZeroBits, ISD::SETNE);
50833 }
50834 }
50835 }
50836
50837 if (SDValue SetCC = combineAndOrForCcmpCtest(N, DAG, DCI, Subtarget))
50838 return SetCC;
50839
50840 if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
50841 return R;
50842
50843 if (SDValue R = combineBitOpWithShift(N, DAG))
50844 return R;
50845
50846 if (SDValue R = combineBitOpWithPACK(N, DAG))
50847 return R;
50848
50849 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
50850 return FPLogic;
50851
50852 if (DCI.isBeforeLegalizeOps())
50853 return SDValue();
50854
50855 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
50856 return R;
50857
50858 if (SDValue R = canonicalizeBitSelect(N, DAG, Subtarget))
50859 return R;
50860
50861 if (SDValue R = combineLogicBlendIntoPBLENDV(N, DAG, Subtarget))
50862 return R;
50863
50864 // (0 - SetCC) | C -> (zext (not SetCC)) * (C + 1) - 1 if we can get a LEA out of it.
50865 if ((VT == MVT::i32 || VT == MVT::i64) &&
50866 N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
50867 isNullConstant(N0.getOperand(0))) {
50868 SDValue Cond = N0.getOperand(1);
50869 if (Cond.getOpcode() == ISD::ZERO_EXTEND && Cond.hasOneUse())
50870 Cond = Cond.getOperand(0);
50871
50872 if (Cond.getOpcode() == X86ISD::SETCC && Cond.hasOneUse()) {
50873 if (auto *CN = dyn_cast<ConstantSDNode>(N1)) {
50874 uint64_t Val = CN->getZExtValue();
50875 if (Val == 1 || Val == 2 || Val == 3 || Val == 4 || Val == 7 || Val == 8) {
50876 X86::CondCode CCode = (X86::CondCode)Cond.getConstantOperandVal(0);
50877 CCode = X86::GetOppositeBranchCondition(CCode);
50878 SDValue NotCond = getSETCC(CCode, Cond.getOperand(1), SDLoc(Cond), DAG);
50879
50880 SDValue R = DAG.getZExtOrTrunc(NotCond, dl, VT);
50881 R = DAG.getNode(ISD::MUL, dl, VT, R, DAG.getConstant(Val + 1, dl, VT));
50882 R = DAG.getNode(ISD::SUB, dl, VT, R, DAG.getConstant(1, dl, VT));
50883 return R;
50884 }
50885 }
50886 }
50887 }
50888
50889 // Combine OR(X,KSHIFTL(Y,Elts/2)) -> CONCAT_VECTORS(X,Y) == KUNPCK(X,Y).
50890 // Combine OR(KSHIFTL(X,Elts/2),Y) -> CONCAT_VECTORS(Y,X) == KUNPCK(Y,X).
50891 // iff the upper elements of the non-shifted arg are zero.
50892 // KUNPCK require 16+ bool vector elements.
50893 if (N0.getOpcode() == X86ISD::KSHIFTL || N1.getOpcode() == X86ISD::KSHIFTL) {
50894 unsigned NumElts = VT.getVectorNumElements();
50895 unsigned HalfElts = NumElts / 2;
50896 APInt UpperElts = APInt::getHighBitsSet(NumElts, HalfElts);
50897 if (NumElts >= 16 && N1.getOpcode() == X86ISD::KSHIFTL &&
50898 N1.getConstantOperandAPInt(1) == HalfElts &&
50899 DAG.MaskedVectorIsZero(N0, UpperElts)) {
50900 return DAG.getNode(
50901 ISD::CONCAT_VECTORS, dl, VT,
50902 extractSubVector(N0, 0, DAG, dl, HalfElts),
50903 extractSubVector(N1.getOperand(0), 0, DAG, dl, HalfElts));
50904 }
50905 if (NumElts >= 16 && N0.getOpcode() == X86ISD::KSHIFTL &&
50906 N0.getConstantOperandAPInt(1) == HalfElts &&
50907 DAG.MaskedVectorIsZero(N1, UpperElts)) {
50908 return DAG.getNode(
50909 ISD::CONCAT_VECTORS, dl, VT,
50910 extractSubVector(N1, 0, DAG, dl, HalfElts),
50911 extractSubVector(N0.getOperand(0), 0, DAG, dl, HalfElts));
50912 }
50913 }
50914
50915 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
50916 // Attempt to recursively combine an OR of shuffles.
50917 SDValue Op(N, 0);
50918 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
50919 return Res;
50920
50921 // If either operand is a constant mask, then only the elements that aren't
50922 // allones are actually demanded by the other operand.
50923 auto SimplifyUndemandedElts = [&](SDValue Op, SDValue OtherOp) {
50924 APInt UndefElts;
50925 SmallVector<APInt> EltBits;
50926 int NumElts = VT.getVectorNumElements();
50927 int EltSizeInBits = VT.getScalarSizeInBits();
50928 if (!getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts, EltBits))
50929 return false;
50930
50931 APInt DemandedElts = APInt::getZero(NumElts);
50932 for (int I = 0; I != NumElts; ++I)
50933 if (!EltBits[I].isAllOnes())
50934 DemandedElts.setBit(I);
50935
50936 return TLI.SimplifyDemandedVectorElts(OtherOp, DemandedElts, DCI);
50937 };
50938 if (SimplifyUndemandedElts(N0, N1) || SimplifyUndemandedElts(N1, N0)) {
50939 if (N->getOpcode() != ISD::DELETED_NODE)
50940 DCI.AddToWorklist(N);
50941 return SDValue(N, 0);
50942 }
50943 }
50944
50945 // We should fold "masked merge" patterns when `andn` is not available.
50946 if (!Subtarget.hasBMI() && VT.isScalarInteger() && VT != MVT::i1)
50947 if (SDValue R = foldMaskedMerge(N, DAG))
50948 return R;
50949
50950 if (SDValue R = combineOrXorWithSETCC(N, N0, N1, DAG))
50951 return R;
50952
50953 return SDValue();
50954}
50955
50956/// Try to turn tests against the signbit in the form of:
50957/// XOR(TRUNCATE(SRL(X, size(X)-1)), 1)
50958/// into:
50959/// SETGT(X, -1)
50960static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
50961 // This is only worth doing if the output type is i8 or i1.
50962 EVT ResultType = N->getValueType(0);
50963 if (ResultType != MVT::i8 && ResultType != MVT::i1)
50964 return SDValue();
50965
50966 SDValue N0 = N->getOperand(0);
50967 SDValue N1 = N->getOperand(1);
50968
50969 // We should be performing an xor against a truncated shift.
50970 if (N0.getOpcode() != ISD::TRUNCATE || !N0.hasOneUse())
50971 return SDValue();
50972
50973 // Make sure we are performing an xor against one.
50974 if (!isOneConstant(N1))
50975 return SDValue();
50976
50977 // SetCC on x86 zero extends so only act on this if it's a logical shift.
50978 SDValue Shift = N0.getOperand(0);
50979 if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse())
50980 return SDValue();
50981
50982 // Make sure we are truncating from one of i16, i32 or i64.
50983 EVT ShiftTy = Shift.getValueType();
50984 if (ShiftTy != MVT::i16 && ShiftTy != MVT::i32 && ShiftTy != MVT::i64)
50985 return SDValue();
50986
50987 // Make sure the shift amount extracts the sign bit.
50988 if (!isa<ConstantSDNode>(Shift.getOperand(1)) ||
50989 Shift.getConstantOperandAPInt(1) != (ShiftTy.getSizeInBits() - 1))
50990 return SDValue();
50991
50992 // Create a greater-than comparison against -1.
50993 // N.B. Using SETGE against 0 works but we want a canonical looking
50994 // comparison, using SETGT matches up with what TranslateX86CC.
50995 SDLoc DL(N);
50996 SDValue ShiftOp = Shift.getOperand(0);
50997 EVT ShiftOpTy = ShiftOp.getValueType();
50998 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
50999 EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(),
51000 *DAG.getContext(), ResultType);
51001 SDValue Cond =
51002 DAG.getSetCC(DL, SetCCResultType, ShiftOp,
51003 DAG.getAllOnesConstant(DL, ShiftOpTy), ISD::SETGT);
51004 if (SetCCResultType != ResultType)
51005 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond);
51006 return Cond;
51007}
51008
51009/// Turn vector tests of the signbit in the form of:
51010/// xor (sra X, elt_size(X)-1), -1
51011/// into:
51012/// pcmpgt X, -1
51013///
51014/// This should be called before type legalization because the pattern may not
51015/// persist after that.
51016static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
51017 const X86Subtarget &Subtarget) {
51018 EVT VT = N->getValueType(0);
51019 if (!VT.isSimple())
51020 return SDValue();
51021
51022 switch (VT.getSimpleVT().SimpleTy) {
51023 // clang-format off
51024 default: return SDValue();
51025 case MVT::v16i8:
51026 case MVT::v8i16:
51027 case MVT::v4i32:
51028 case MVT::v2i64: if (!Subtarget.hasSSE2()) return SDValue(); break;
51029 case MVT::v32i8:
51030 case MVT::v16i16:
51031 case MVT::v8i32:
51032 case MVT::v4i64: if (!Subtarget.hasAVX2()) return SDValue(); break;
51033 // clang-format on
51034 }
51035
51036 // There must be a shift right algebraic before the xor, and the xor must be a
51037 // 'not' operation.
51038 SDValue Shift = N->getOperand(0);
51039 SDValue Ones = N->getOperand(1);
51040 if (Shift.getOpcode() != ISD::SRA || !Shift.hasOneUse() ||
51041 !ISD::isBuildVectorAllOnes(Ones.getNode()))
51042 return SDValue();
51043
51044 // The shift should be smearing the sign bit across each vector element.
51045 auto *ShiftAmt =
51046 isConstOrConstSplat(Shift.getOperand(1), /*AllowUndefs*/ true);
51047 if (!ShiftAmt ||
51048 ShiftAmt->getAPIntValue() != (Shift.getScalarValueSizeInBits() - 1))
51049 return SDValue();
51050
51051 // Create a greater-than comparison against -1. We don't use the more obvious
51052 // greater-than-or-equal-to-zero because SSE/AVX don't have that instruction.
51053 return DAG.getSetCC(SDLoc(N), VT, Shift.getOperand(0), Ones, ISD::SETGT);
51054}
51055
51056/// Detect patterns of truncation with unsigned saturation:
51057///
51058/// 1. (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type).
51059/// Return the source value x to be truncated or SDValue() if the pattern was
51060/// not matched.
51061///
51062/// 2. (truncate (smin (smax (x, C1), C2)) to dest_type),
51063/// where C1 >= 0 and C2 is unsigned max of destination type.
51064///
51065/// (truncate (smax (smin (x, C2), C1)) to dest_type)
51066/// where C1 >= 0, C2 is unsigned max of destination type and C1 <= C2.
51067///
51068/// These two patterns are equivalent to:
51069/// (truncate (umin (smax(x, C1), unsigned_max_of_dest_type)) to dest_type)
51070/// So return the smax(x, C1) value to be truncated or SDValue() if the
51071/// pattern was not matched.
51072static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,
51073 const SDLoc &DL) {
51074 using namespace llvm::SDPatternMatch;
51075 EVT InVT = In.getValueType();
51076
51077 // Saturation with truncation. We truncate from InVT to VT.
51078 assert(InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() &&
51079 "Unexpected types for truncate operation");
51080
51081 APInt C1, C2;
51082 SDValue UMin, SMin, SMax;
51083
51084 // C2 should be equal to UINT32_MAX / UINT16_MAX / UINT8_MAX according
51085 // the element size of the destination type.
51086 if (sd_match(In, m_UMin(m_Value(UMin), m_ConstInt(C2))) &&
51087 C2.isMask(VT.getScalarSizeInBits()))
51088 return UMin;
51089
51090 if (sd_match(In, m_SMin(m_Value(SMin), m_ConstInt(C2))) &&
51091 sd_match(SMin, m_SMax(m_Value(SMax), m_ConstInt(C1))) &&
51092 C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()))
51093 return SMin;
51094
51095 if (sd_match(In, m_SMax(m_Value(SMax), m_ConstInt(C1))) &&
51096 sd_match(SMax, m_SMin(m_Value(SMin), m_ConstInt(C2))) &&
51097 C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()) && C2.uge(C1))
51098 return DAG.getNode(ISD::SMAX, DL, InVT, SMin, In.getOperand(1));
51099
51100 return SDValue();
51101}
51102
51103/// Detect patterns of truncation with signed saturation:
51104/// (truncate (smin ((smax (x, signed_min_of_dest_type)),
51105/// signed_max_of_dest_type)) to dest_type)
51106/// or:
51107/// (truncate (smax ((smin (x, signed_max_of_dest_type)),
51108/// signed_min_of_dest_type)) to dest_type).
51109/// With MatchPackUS, the smax/smin range is [0, unsigned_max_of_dest_type].
51110/// Return the source value to be truncated or SDValue() if the pattern was not
51111/// matched.
51112static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) {
51113 using namespace llvm::SDPatternMatch;
51114 unsigned NumDstBits = VT.getScalarSizeInBits();
51115 unsigned NumSrcBits = In.getScalarValueSizeInBits();
51116 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
51117
51118 APInt SignedMax, SignedMin;
51119 if (MatchPackUS) {
51120 SignedMax = APInt::getAllOnes(NumDstBits).zext(NumSrcBits);
51121 SignedMin = APInt::getZero(NumSrcBits);
51122 } else {
51123 SignedMax = APInt::getSignedMaxValue(NumDstBits).sext(NumSrcBits);
51124 SignedMin = APInt::getSignedMinValue(NumDstBits).sext(NumSrcBits);
51125 }
51126
51127 SDValue SMin, SMax;
51128 if (sd_match(In, m_SMin(m_Value(SMin), m_SpecificInt(SignedMax))) &&
51129 sd_match(SMin, m_SMax(m_Value(SMax), m_SpecificInt(SignedMin))))
51130 return SMax;
51131
51132 if (sd_match(In, m_SMax(m_Value(SMax), m_SpecificInt(SignedMin))) &&
51133 sd_match(SMax, m_SMin(m_Value(SMin), m_SpecificInt(SignedMax))))
51134 return SMin;
51135
51136 return SDValue();
51137}
51138
51139static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
51140 SelectionDAG &DAG,
51141 const X86Subtarget &Subtarget) {
51142 if (!Subtarget.hasSSE2() || !VT.isVector())
51143 return SDValue();
51144
51145 EVT SVT = VT.getVectorElementType();
51146 EVT InVT = In.getValueType();
51147 EVT InSVT = InVT.getVectorElementType();
51148
51149 // If we're clamping a signed 32-bit vector to 0-255 and the 32-bit vector is
51150 // split across two registers. We can use a packusdw+perm to clamp to 0-65535
51151 // and concatenate at the same time. Then we can use a final vpmovuswb to
51152 // clip to 0-255.
51153 if (Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
51154 InVT == MVT::v16i32 && VT == MVT::v16i8) {
51155 if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
51156 // Emit a VPACKUSDW+VPERMQ followed by a VPMOVUSWB.
51157 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,
51158 DL, DAG, Subtarget);
51159 assert(Mid && "Failed to pack!");
51160 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid);
51161 }
51162 }
51163
51164 // vXi32 truncate instructions are available with AVX512F.
51165 // vXi16 truncate instructions are only available with AVX512BW.
51166 // For 256-bit or smaller vectors, we require VLX.
51167 // FIXME: We could widen truncates to 512 to remove the VLX restriction.
51168 // If the result type is 256-bits or larger and we have disable 512-bit
51169 // registers, we should go ahead and use the pack instructions if possible.
51170 bool PreferAVX512 = ((Subtarget.hasAVX512() && InSVT == MVT::i32) ||
51171 (Subtarget.hasBWI() && InSVT == MVT::i16)) &&
51172 (InVT.getSizeInBits() > 128) &&
51173 (Subtarget.hasVLX() || InVT.getSizeInBits() > 256) &&
51174 !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256);
51175
51176 if (!PreferAVX512 && VT.getVectorNumElements() > 1 &&
51177 isPowerOf2_32(VT.getVectorNumElements()) &&
51178 (SVT == MVT::i8 || SVT == MVT::i16) &&
51179 (InSVT == MVT::i16 || InSVT == MVT::i32)) {
51180 if (SDValue USatVal = detectSSatPattern(In, VT, true)) {
51181 // vXi32 -> vXi8 must be performed as PACKUSWB(PACKSSDW,PACKSSDW).
51182 if (SVT == MVT::i8 && InSVT == MVT::i32) {
51183 EVT MidVT = VT.changeVectorElementType(MVT::i16);
51184 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
51185 DAG, Subtarget);
51186 assert(Mid && "Failed to pack!");
51187 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
51188 Subtarget);
51189 assert(V && "Failed to pack!");
51190 return V;
51191 } else if (SVT == MVT::i8 || Subtarget.hasSSE41())
51192 return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG,
51193 Subtarget);
51194 }
51195 if (SDValue SSatVal = detectSSatPattern(In, VT))
51196 return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG,
51197 Subtarget);
51198 }
51199
51200 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51201 if (TLI.isTypeLegal(InVT) && InVT.isVector() && SVT != MVT::i1 &&
51202 Subtarget.hasAVX512() && (InSVT != MVT::i16 || Subtarget.hasBWI()) &&
51203 (SVT == MVT::i32 || SVT == MVT::i16 || SVT == MVT::i8)) {
51204 unsigned TruncOpc = 0;
51205 SDValue SatVal;
51206 if (SDValue SSatVal = detectSSatPattern(In, VT)) {
51207 SatVal = SSatVal;
51208 TruncOpc = X86ISD::VTRUNCS;
51209 } else if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) {
51210 SatVal = USatVal;
51211 TruncOpc = X86ISD::VTRUNCUS;
51212 }
51213 if (SatVal) {
51214 unsigned ResElts = VT.getVectorNumElements();
51215 // If the input type is less than 512 bits and we don't have VLX, we need
51216 // to widen to 512 bits.
51217 if (!Subtarget.hasVLX() && !InVT.is512BitVector()) {
51218 unsigned NumConcats = 512 / InVT.getSizeInBits();
51219 ResElts *= NumConcats;
51220 SmallVector<SDValue, 4> ConcatOps(NumConcats, DAG.getUNDEF(InVT));
51221 ConcatOps[0] = SatVal;
51222 InVT = EVT::getVectorVT(*DAG.getContext(), InSVT,
51223 NumConcats * InVT.getVectorNumElements());
51224 SatVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, InVT, ConcatOps);
51225 }
51226 // Widen the result if its narrower than 128 bits.
51227 if (ResElts * SVT.getSizeInBits() < 128)
51228 ResElts = 128 / SVT.getSizeInBits();
51229 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), SVT, ResElts);
51230 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal);
51231 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
51232 DAG.getIntPtrConstant(0, DL));
51233 }
51234 }
51235
51236 return SDValue();
51237}
51238
51239static SDValue combineConstantPoolLoads(SDNode *N, const SDLoc &dl,
51240 SelectionDAG &DAG,
51241 TargetLowering::DAGCombinerInfo &DCI,
51242 const X86Subtarget &Subtarget) {
51243 auto *Ld = cast<LoadSDNode>(N);
51244 EVT RegVT = Ld->getValueType(0);
51245 SDValue Ptr = Ld->getBasePtr();
51246 SDValue Chain = Ld->getChain();
51247 ISD::LoadExtType Ext = Ld->getExtensionType();
51248
51249 if (Ext != ISD::NON_EXTLOAD || !Subtarget.hasAVX() || !Ld->isSimple())
51250 return SDValue();
51251
51252 if (!(RegVT.is128BitVector() || RegVT.is256BitVector()))
51253 return SDValue();
51254
51255 const Constant *LdC = getTargetConstantFromBasePtr(Ptr);
51256 if (!LdC)
51257 return SDValue();
51258
51259 auto MatchingBits = [](const APInt &Undefs, const APInt &UserUndefs,
51260 ArrayRef<APInt> Bits, ArrayRef<APInt> UserBits) {
51261 for (unsigned I = 0, E = Undefs.getBitWidth(); I != E; ++I) {
51262 if (Undefs[I])
51263 continue;
51264 if (UserUndefs[I] || Bits[I] != UserBits[I])
51265 return false;
51266 }
51267 return true;
51268 };
51269
51270 // Look through all other loads/broadcasts in the chain for another constant
51271 // pool entry.
51272 for (SDNode *User : Chain->uses()) {
51273 auto *UserLd = dyn_cast<MemSDNode>(User);
51274 if (User != N && UserLd &&
51275 (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD ||
51276 User->getOpcode() == X86ISD::VBROADCAST_LOAD ||
51277 ISD::isNormalLoad(User)) &&
51278 UserLd->getChain() == Chain && !User->hasAnyUseOfValue(1) &&
51279 User->getValueSizeInBits(0).getFixedValue() >
51280 RegVT.getFixedSizeInBits()) {
51281 EVT UserVT = User->getValueType(0);
51282 SDValue UserPtr = UserLd->getBasePtr();
51283 const Constant *UserC = getTargetConstantFromBasePtr(UserPtr);
51284
51285 // See if we are loading a constant that matches in the lower
51286 // bits of a longer constant (but from a different constant pool ptr).
51287 if (UserC && UserPtr != Ptr) {
51288 unsigned LdSize = LdC->getType()->getPrimitiveSizeInBits();
51289 unsigned UserSize = UserC->getType()->getPrimitiveSizeInBits();
51290 if (LdSize < UserSize || !ISD::isNormalLoad(User)) {
51291 APInt Undefs, UserUndefs;
51292 SmallVector<APInt> Bits, UserBits;
51293 unsigned NumBits = std::min(RegVT.getScalarSizeInBits(),
51294 UserVT.getScalarSizeInBits());
51295 if (getTargetConstantBitsFromNode(SDValue(N, 0), NumBits, Undefs,
51296 Bits) &&
51297 getTargetConstantBitsFromNode(SDValue(User, 0), NumBits,
51298 UserUndefs, UserBits)) {
51299 if (MatchingBits(Undefs, UserUndefs, Bits, UserBits)) {
51300 SDValue Extract = extractSubVector(
51301 SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits());
51302 Extract = DAG.getBitcast(RegVT, Extract);
51303 return DCI.CombineTo(N, Extract, SDValue(User, 1));
51304 }
51305 }
51306 }
51307 }
51308 }
51309 }
51310
51311 return SDValue();
51312}
51313
51314static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
51315 TargetLowering::DAGCombinerInfo &DCI,
51316 const X86Subtarget &Subtarget) {
51317 auto *Ld = cast<LoadSDNode>(N);
51318 EVT RegVT = Ld->getValueType(0);
51319 EVT MemVT = Ld->getMemoryVT();
51320 SDLoc dl(Ld);
51321 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51322
51323 // For chips with slow 32-byte unaligned loads, break the 32-byte operation
51324 // into two 16-byte operations. Also split non-temporal aligned loads on
51325 // pre-AVX2 targets as 32-byte loads will lower to regular temporal loads.
51326 ISD::LoadExtType Ext = Ld->getExtensionType();
51327 unsigned Fast;
51328 if (RegVT.is256BitVector() && !DCI.isBeforeLegalizeOps() &&
51329 Ext == ISD::NON_EXTLOAD &&
51330 ((Ld->isNonTemporal() && !Subtarget.hasInt256() &&
51331 Ld->getAlign() >= Align(16)) ||
51332 (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), RegVT,
51333 *Ld->getMemOperand(), &Fast) &&
51334 !Fast))) {
51335 unsigned NumElems = RegVT.getVectorNumElements();
51336 if (NumElems < 2)
51337 return SDValue();
51338
51339 unsigned HalfOffset = 16;
51340 SDValue Ptr1 = Ld->getBasePtr();
51341 SDValue Ptr2 =
51342 DAG.getMemBasePlusOffset(Ptr1, TypeSize::getFixed(HalfOffset), dl);
51343 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
51344 NumElems / 2);
51345 SDValue Load1 =
51346 DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr1, Ld->getPointerInfo(),
51347 Ld->getOriginalAlign(),
51348 Ld->getMemOperand()->getFlags());
51349 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2,
51350 Ld->getPointerInfo().getWithOffset(HalfOffset),
51351 Ld->getOriginalAlign(),
51352 Ld->getMemOperand()->getFlags());
51353 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
51354 Load1.getValue(1), Load2.getValue(1));
51355
51356 SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2);
51357 return DCI.CombineTo(N, NewVec, TF, true);
51358 }
51359
51360 // Bool vector load - attempt to cast to an integer, as we have good
51361 // (vXiY *ext(vXi1 bitcast(iX))) handling.
51362 if (Ext == ISD::NON_EXTLOAD && !Subtarget.hasAVX512() && RegVT.isVector() &&
51363 RegVT.getScalarType() == MVT::i1 && DCI.isBeforeLegalize()) {
51364 unsigned NumElts = RegVT.getVectorNumElements();
51365 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
51366 if (TLI.isTypeLegal(IntVT)) {
51367 SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(),
51368 Ld->getPointerInfo(),
51369 Ld->getOriginalAlign(),
51370 Ld->getMemOperand()->getFlags());
51371 SDValue BoolVec = DAG.getBitcast(RegVT, IntLoad);
51372 return DCI.CombineTo(N, BoolVec, IntLoad.getValue(1), true);
51373 }
51374 }
51375
51376 // If we also broadcast this vector to a wider type, then just extract the
51377 // lowest subvector.
51378 if (Ext == ISD::NON_EXTLOAD && Subtarget.hasAVX() && Ld->isSimple() &&
51379 (RegVT.is128BitVector() || RegVT.is256BitVector())) {
51380 SDValue Ptr = Ld->getBasePtr();
51381 SDValue Chain = Ld->getChain();
51382 for (SDNode *User : Chain->uses()) {
51383 auto *UserLd = dyn_cast<MemSDNode>(User);
51384 if (User != N && UserLd &&
51385 User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
51386 UserLd->getChain() == Chain && UserLd->getBasePtr() == Ptr &&
51387 UserLd->getMemoryVT().getSizeInBits() == MemVT.getSizeInBits() &&
51388 !User->hasAnyUseOfValue(1) &&
51389 User->getValueSizeInBits(0).getFixedValue() >
51390 RegVT.getFixedSizeInBits()) {
51391 SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, dl,
51392 RegVT.getSizeInBits());
51393 Extract = DAG.getBitcast(RegVT, Extract);
51394 return DCI.CombineTo(N, Extract, SDValue(User, 1));
51395 }
51396 }
51397 }
51398
51399 if (SDValue V = combineConstantPoolLoads(Ld, dl, DAG, DCI, Subtarget))
51400 return V;
51401
51402 // Cast ptr32 and ptr64 pointers to the default address space before a load.
51403 unsigned AddrSpace = Ld->getAddressSpace();
51404 if (AddrSpace == X86AS::PTR64 || AddrSpace == X86AS::PTR32_SPTR ||
51405 AddrSpace == X86AS::PTR32_UPTR) {
51406 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
51407 if (PtrVT != Ld->getBasePtr().getSimpleValueType()) {
51408 SDValue Cast =
51409 DAG.getAddrSpaceCast(dl, PtrVT, Ld->getBasePtr(), AddrSpace, 0);
51410 return DAG.getExtLoad(Ext, dl, RegVT, Ld->getChain(), Cast,
51411 Ld->getPointerInfo(), MemVT, Ld->getOriginalAlign(),
51412 Ld->getMemOperand()->getFlags());
51413 }
51414 }
51415
51416 return SDValue();
51417}
51418
51419/// If V is a build vector of boolean constants and exactly one of those
51420/// constants is true, return the operand index of that true element.
51421/// Otherwise, return -1.
51422static int getOneTrueElt(SDValue V) {
51423 // This needs to be a build vector of booleans.
51424 // TODO: Checking for the i1 type matches the IR definition for the mask,
51425 // but the mask check could be loosened to i8 or other types. That might
51426 // also require checking more than 'allOnesValue'; eg, the x86 HW
51427 // instructions only require that the MSB is set for each mask element.
51428 // The ISD::MSTORE comments/definition do not specify how the mask operand
51429 // is formatted.
51430 auto *BV = dyn_cast<BuildVectorSDNode>(V);
51431 if (!BV || BV->getValueType(0).getVectorElementType() != MVT::i1)
51432 return -1;
51433
51434 int TrueIndex = -1;
51435 unsigned NumElts = BV->getValueType(0).getVectorNumElements();
51436 for (unsigned i = 0; i < NumElts; ++i) {
51437 const SDValue &Op = BV->getOperand(i);
51438 if (Op.isUndef())
51439 continue;
51440 auto *ConstNode = dyn_cast<ConstantSDNode>(Op);
51441 if (!ConstNode)
51442 return -1;
51443 if (ConstNode->getAPIntValue().countr_one() >= 1) {
51444 // If we already found a one, this is too many.
51445 if (TrueIndex >= 0)
51446 return -1;
51447 TrueIndex = i;
51448 }
51449 }
51450 return TrueIndex;
51451}
51452
51453/// Given a masked memory load/store operation, return true if it has one mask
51454/// bit set. If it has one mask bit set, then also return the memory address of
51455/// the scalar element to load/store, the vector index to insert/extract that
51456/// scalar element, and the alignment for the scalar memory access.
51457static bool getParamsForOneTrueMaskedElt(MaskedLoadStoreSDNode *MaskedOp,
51458 SelectionDAG &DAG, SDValue &Addr,
51459 SDValue &Index, Align &Alignment,
51460 unsigned &Offset) {
51461 int TrueMaskElt = getOneTrueElt(MaskedOp->getMask());
51462 if (TrueMaskElt < 0)
51463 return false;
51464
51465 // Get the address of the one scalar element that is specified by the mask
51466 // using the appropriate offset from the base pointer.
51467 EVT EltVT = MaskedOp->getMemoryVT().getVectorElementType();
51468 Offset = 0;
51469 Addr = MaskedOp->getBasePtr();
51470 if (TrueMaskElt != 0) {
51471 Offset = TrueMaskElt * EltVT.getStoreSize();
51472 Addr = DAG.getMemBasePlusOffset(Addr, TypeSize::getFixed(Offset),
51473 SDLoc(MaskedOp));
51474 }
51475
51476 Index = DAG.getIntPtrConstant(TrueMaskElt, SDLoc(MaskedOp));
51477 Alignment = commonAlignment(MaskedOp->getOriginalAlign(),
51478 EltVT.getStoreSize());
51479 return true;
51480}
51481
51482/// If exactly one element of the mask is set for a non-extending masked load,
51483/// it is a scalar load and vector insert.
51484/// Note: It is expected that the degenerate cases of an all-zeros or all-ones
51485/// mask have already been optimized in IR, so we don't bother with those here.
51486static SDValue
51487reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG,
51488 TargetLowering::DAGCombinerInfo &DCI,
51489 const X86Subtarget &Subtarget) {
51490 assert(ML->isUnindexed() && "Unexpected indexed masked load!");
51491 // TODO: This is not x86-specific, so it could be lifted to DAGCombiner.
51492 // However, some target hooks may need to be added to know when the transform
51493 // is profitable. Endianness would also have to be considered.
51494
51495 SDValue Addr, VecIndex;
51496 Align Alignment;
51497 unsigned Offset;
51498 if (!getParamsForOneTrueMaskedElt(ML, DAG, Addr, VecIndex, Alignment, Offset))
51499 return SDValue();
51500
51501 // Load the one scalar element that is specified by the mask using the
51502 // appropriate offset from the base pointer.
51503 SDLoc DL(ML);
51504 EVT VT = ML->getValueType(0);
51505 EVT EltVT = VT.getVectorElementType();
51506
51507 EVT CastVT = VT;
51508 if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
51509 EltVT = MVT::f64;
51510 CastVT = VT.changeVectorElementType(EltVT);
51511 }
51512
51513 SDValue Load =
51514 DAG.getLoad(EltVT, DL, ML->getChain(), Addr,
51515 ML->getPointerInfo().getWithOffset(Offset),
51516 Alignment, ML->getMemOperand()->getFlags());
51517
51518 SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru());
51519
51520 // Insert the loaded element into the appropriate place in the vector.
51521 SDValue Insert =
51522 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, CastVT, PassThru, Load, VecIndex);
51523 Insert = DAG.getBitcast(VT, Insert);
51524 return DCI.CombineTo(ML, Insert, Load.getValue(1), true);
51525}
51526
51527static SDValue
51528combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG,
51529 TargetLowering::DAGCombinerInfo &DCI) {
51530 assert(ML->isUnindexed() && "Unexpected indexed masked load!");
51531 if (!ISD::isBuildVectorOfConstantSDNodes(ML->getMask().getNode()))
51532 return SDValue();
51533
51534 SDLoc DL(ML);
51535 EVT VT = ML->getValueType(0);
51536
51537 // If we are loading the first and last elements of a vector, it is safe and
51538 // always faster to load the whole vector. Replace the masked load with a
51539 // vector load and select.
51540 unsigned NumElts = VT.getVectorNumElements();
51541 BuildVectorSDNode *MaskBV = cast<BuildVectorSDNode>(ML->getMask());
51542 bool LoadFirstElt = !isNullConstant(MaskBV->getOperand(0));
51543 bool LoadLastElt = !isNullConstant(MaskBV->getOperand(NumElts - 1));
51544 if (LoadFirstElt && LoadLastElt) {
51545 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
51546 ML->getMemOperand());
51547 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
51548 ML->getPassThru());
51549 return DCI.CombineTo(ML, Blend, VecLd.getValue(1), true);
51550 }
51551
51552 // Convert a masked load with a constant mask into a masked load and a select.
51553 // This allows the select operation to use a faster kind of select instruction
51554 // (for example, vblendvps -> vblendps).
51555
51556 // Don't try this if the pass-through operand is already undefined. That would
51557 // cause an infinite loop because that's what we're about to create.
51558 if (ML->getPassThru().isUndef())
51559 return SDValue();
51560
51561 if (ISD::isBuildVectorAllZeros(ML->getPassThru().getNode()))
51562 return SDValue();
51563
51564 // The new masked load has an undef pass-through operand. The select uses the
51565 // original pass-through operand.
51566 SDValue NewML = DAG.getMaskedLoad(
51567 VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(),
51568 DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(),
51569 ML->getAddressingMode(), ML->getExtensionType());
51570 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
51571 ML->getPassThru());
51572
51573 return DCI.CombineTo(ML, Blend, NewML.getValue(1), true);
51574}
51575
51576static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
51577 TargetLowering::DAGCombinerInfo &DCI,
51578 const X86Subtarget &Subtarget) {
51579 auto *Mld = cast<MaskedLoadSDNode>(N);
51580
51581 // TODO: Expanding load with constant mask may be optimized as well.
51582 if (Mld->isExpandingLoad())
51583 return SDValue();
51584
51585 if (Mld->getExtensionType() == ISD::NON_EXTLOAD) {
51586 if (SDValue ScalarLoad =
51587 reduceMaskedLoadToScalarLoad(Mld, DAG, DCI, Subtarget))
51588 return ScalarLoad;
51589
51590 // TODO: Do some AVX512 subsets benefit from this transform?
51591 if (!Subtarget.hasAVX512())
51592 if (SDValue Blend = combineMaskedLoadConstantMask(Mld, DAG, DCI))
51593 return Blend;
51594 }
51595
51596 // If the mask value has been legalized to a non-boolean vector, try to
51597 // simplify ops leading up to it. We only demand the MSB of each lane.
51598 SDValue Mask = Mld->getMask();
51599 if (Mask.getScalarValueSizeInBits() != 1) {
51600 EVT VT = Mld->getValueType(0);
51601 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51602 APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits()));
51603 if (TLI.SimplifyDemandedBits(Mask, DemandedBits, DCI)) {
51604 if (N->getOpcode() != ISD::DELETED_NODE)
51605 DCI.AddToWorklist(N);
51606 return SDValue(N, 0);
51607 }
51608 if (SDValue NewMask =
51609 TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG))
51610 return DAG.getMaskedLoad(
51611 VT, SDLoc(N), Mld->getChain(), Mld->getBasePtr(), Mld->getOffset(),
51612 NewMask, Mld->getPassThru(), Mld->getMemoryVT(), Mld->getMemOperand(),
51613 Mld->getAddressingMode(), Mld->getExtensionType());
51614 }
51615
51616 return SDValue();
51617}
51618
51619/// If exactly one element of the mask is set for a non-truncating masked store,
51620/// it is a vector extract and scalar store.
51621/// Note: It is expected that the degenerate cases of an all-zeros or all-ones
51622/// mask have already been optimized in IR, so we don't bother with those here.
51623static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS,
51624 SelectionDAG &DAG,
51625 const X86Subtarget &Subtarget) {
51626 // TODO: This is not x86-specific, so it could be lifted to DAGCombiner.
51627 // However, some target hooks may need to be added to know when the transform
51628 // is profitable. Endianness would also have to be considered.
51629
51630 SDValue Addr, VecIndex;
51631 Align Alignment;
51632 unsigned Offset;
51633 if (!getParamsForOneTrueMaskedElt(MS, DAG, Addr, VecIndex, Alignment, Offset))
51634 return SDValue();
51635
51636 // Extract the one scalar element that is actually being stored.
51637 SDLoc DL(MS);
51638 SDValue Value = MS->getValue();
51639 EVT VT = Value.getValueType();
51640 EVT EltVT = VT.getVectorElementType();
51641 if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
51642 EltVT = MVT::f64;
51643 EVT CastVT = VT.changeVectorElementType(EltVT);
51644 Value = DAG.getBitcast(CastVT, Value);
51645 }
51646 SDValue Extract =
51647 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Value, VecIndex);
51648
51649 // Store that element at the appropriate offset from the base pointer.
51650 return DAG.getStore(MS->getChain(), DL, Extract, Addr,
51651 MS->getPointerInfo().getWithOffset(Offset),
51652 Alignment, MS->getMemOperand()->getFlags());
51653}
51654
51655static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
51656 TargetLowering::DAGCombinerInfo &DCI,
51657 const X86Subtarget &Subtarget) {
51658 MaskedStoreSDNode *Mst = cast<MaskedStoreSDNode>(N);
51659 if (Mst->isCompressingStore())
51660 return SDValue();
51661
51662 EVT VT = Mst->getValue().getValueType();
51663 SDLoc dl(Mst);
51664 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51665
51666 if (Mst->isTruncatingStore())
51667 return SDValue();
51668
51669 if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG, Subtarget))
51670 return ScalarStore;
51671
51672 // If the mask value has been legalized to a non-boolean vector, try to
51673 // simplify ops leading up to it. We only demand the MSB of each lane.
51674 SDValue Mask = Mst->getMask();
51675 if (Mask.getScalarValueSizeInBits() != 1) {
51676 APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits()));
51677 if (TLI.SimplifyDemandedBits(Mask, DemandedBits, DCI)) {
51678 if (N->getOpcode() != ISD::DELETED_NODE)
51679 DCI.AddToWorklist(N);
51680 return SDValue(N, 0);
51681 }
51682 if (SDValue NewMask =
51683 TLI.SimplifyMultipleUseDemandedBits(Mask, DemandedBits, DAG))
51684 return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Mst->getValue(),
51685 Mst->getBasePtr(), Mst->getOffset(), NewMask,
51686 Mst->getMemoryVT(), Mst->getMemOperand(),
51687 Mst->getAddressingMode());
51688 }
51689
51690 SDValue Value = Mst->getValue();
51691 if (Value.getOpcode() == ISD::TRUNCATE && Value.getNode()->hasOneUse() &&
51692 TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
51693 Mst->getMemoryVT())) {
51694 return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0),
51695 Mst->getBasePtr(), Mst->getOffset(), Mask,
51696 Mst->getMemoryVT(), Mst->getMemOperand(),
51697 Mst->getAddressingMode(), true);
51698 }
51699
51700 return SDValue();
51701}
51702
51703static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
51704 TargetLowering::DAGCombinerInfo &DCI,
51705 const X86Subtarget &Subtarget) {
51706 StoreSDNode *St = cast<StoreSDNode>(N);
51707 EVT StVT = St->getMemoryVT();
51708 SDLoc dl(St);
51709 SDValue StoredVal = St->getValue();
51710 EVT VT = StoredVal.getValueType();
51711 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51712
51713 // Convert a store of vXi1 into a store of iX and a bitcast.
51714 if (!Subtarget.hasAVX512() && VT == StVT && VT.isVector() &&
51715 VT.getVectorElementType() == MVT::i1) {
51716
51717 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());
51718 StoredVal = DAG.getBitcast(NewVT, StoredVal);
51719
51720 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
51721 St->getPointerInfo(), St->getOriginalAlign(),
51722 St->getMemOperand()->getFlags());
51723 }
51724
51725 // If this is a store of a scalar_to_vector to v1i1, just use a scalar store.
51726 // This will avoid a copy to k-register.
51727 if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() &&
51728 StoredVal.getOpcode() == ISD::SCALAR_TO_VECTOR &&
51729 StoredVal.getOperand(0).getValueType() == MVT::i8) {
51730 SDValue Val = StoredVal.getOperand(0);
51731 // We must store zeros to the unused bits.
51732 Val = DAG.getZeroExtendInReg(Val, dl, MVT::i1);
51733 return DAG.getStore(St->getChain(), dl, Val,
51734 St->getBasePtr(), St->getPointerInfo(),
51735 St->getOriginalAlign(),
51736 St->getMemOperand()->getFlags());
51737 }
51738
51739 // Widen v2i1/v4i1 stores to v8i1.
51740 if ((VT == MVT::v1i1 || VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT &&
51741 Subtarget.hasAVX512()) {
51742 unsigned NumConcats = 8 / VT.getVectorNumElements();
51743 // We must store zeros to the unused bits.
51744 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT));
51745 Ops[0] = StoredVal;
51746 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
51747 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
51748 St->getPointerInfo(), St->getOriginalAlign(),
51749 St->getMemOperand()->getFlags());
51750 }
51751
51752 // Turn vXi1 stores of constants into a scalar store.
51753 if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 ||
51754 VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) &&
51755 ISD::isBuildVectorOfConstantSDNodes(StoredVal.getNode())) {
51756 // If its a v64i1 store without 64-bit support, we need two stores.
51757 if (!DCI.isBeforeLegalize() && VT == MVT::v64i1 && !Subtarget.is64Bit()) {
51758 SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
51759 StoredVal->ops().slice(0, 32));
51760 Lo = combinevXi1ConstantToInteger(Lo, DAG);
51761 SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
51762 StoredVal->ops().slice(32, 32));
51763 Hi = combinevXi1ConstantToInteger(Hi, DAG);
51764
51765 SDValue Ptr0 = St->getBasePtr();
51766 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, TypeSize::getFixed(4), dl);
51767
51768 SDValue Ch0 =
51769 DAG.getStore(St->getChain(), dl, Lo, Ptr0, St->getPointerInfo(),
51770 St->getOriginalAlign(),
51771 St->getMemOperand()->getFlags());
51772 SDValue Ch1 =
51773 DAG.getStore(St->getChain(), dl, Hi, Ptr1,
51774 St->getPointerInfo().getWithOffset(4),
51775 St->getOriginalAlign(),
51776 St->getMemOperand()->getFlags());
51777 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);
51778 }
51779
51780 StoredVal = combinevXi1ConstantToInteger(StoredVal, DAG);
51781 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
51782 St->getPointerInfo(), St->getOriginalAlign(),
51783 St->getMemOperand()->getFlags());
51784 }
51785
51786 // If we are saving a 32-byte vector and 32-byte stores are slow, such as on
51787 // Sandy Bridge, perform two 16-byte stores.
51788 unsigned Fast;
51789 if (VT.is256BitVector() && StVT == VT &&
51790 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
51791 *St->getMemOperand(), &Fast) &&
51792 !Fast) {
51793 unsigned NumElems = VT.getVectorNumElements();
51794 if (NumElems < 2)
51795 return SDValue();
51796
51797 return splitVectorStore(St, DAG);
51798 }
51799
51800 // Split under-aligned vector non-temporal stores.
51801 if (St->isNonTemporal() && StVT == VT &&
51802 St->getAlign().value() < VT.getStoreSize()) {
51803 // ZMM/YMM nt-stores - either it can be stored as a series of shorter
51804 // vectors or the legalizer can scalarize it to use MOVNTI.
51805 if (VT.is256BitVector() || VT.is512BitVector()) {
51806 unsigned NumElems = VT.getVectorNumElements();
51807 if (NumElems < 2)
51808 return SDValue();
51809 return splitVectorStore(St, DAG);
51810 }
51811
51812 // XMM nt-stores - scalarize this to f64 nt-stores on SSE4A, else i32/i64
51813 // to use MOVNTI.
51814 if (VT.is128BitVector() && Subtarget.hasSSE2()) {
51815 MVT NTVT = Subtarget.hasSSE4A()
51816 ? MVT::v2f64
51817 : (TLI.isTypeLegal(MVT::i64) ? MVT::v2i64 : MVT::v4i32);
51818 return scalarizeVectorStore(St, NTVT, DAG);
51819 }
51820 }
51821
51822 // Try to optimize v16i16->v16i8 truncating stores when BWI is not
51823 // supported, but avx512f is by extending to v16i32 and truncating.
51824 if (!St->isTruncatingStore() && VT == MVT::v16i8 && !Subtarget.hasBWI() &&
51825 St->getValue().getOpcode() == ISD::TRUNCATE &&
51826 St->getValue().getOperand(0).getValueType() == MVT::v16i16 &&
51827 TLI.isTruncStoreLegal(MVT::v16i32, MVT::v16i8) &&
51828 St->getValue().hasOneUse() && !DCI.isBeforeLegalizeOps()) {
51829 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32,
51830 St->getValue().getOperand(0));
51831 return DAG.getTruncStore(St->getChain(), dl, Ext, St->getBasePtr(),
51832 MVT::v16i8, St->getMemOperand());
51833 }
51834
51835 // Try to fold a VTRUNCUS or VTRUNCS into a truncating store.
51836 if (!St->isTruncatingStore() &&
51837 (StoredVal.getOpcode() == X86ISD::VTRUNCUS ||
51838 StoredVal.getOpcode() == X86ISD::VTRUNCS) &&
51839 StoredVal.hasOneUse() &&
51840 TLI.isTruncStoreLegal(StoredVal.getOperand(0).getValueType(), VT)) {
51841 bool IsSigned = StoredVal.getOpcode() == X86ISD::VTRUNCS;
51842 return EmitTruncSStore(IsSigned, St->getChain(),
51843 dl, StoredVal.getOperand(0), St->getBasePtr(),
51844 VT, St->getMemOperand(), DAG);
51845 }
51846
51847 // Try to fold a extract_element(VTRUNC) pattern into a truncating store.
51848 if (!St->isTruncatingStore()) {
51849 auto IsExtractedElement = [](SDValue V) {
51850 if (V.getOpcode() == ISD::TRUNCATE && V.hasOneUse())
51851 V = V.getOperand(0);
51852 unsigned Opc = V.getOpcode();
51853 if ((Opc == ISD::EXTRACT_VECTOR_ELT || Opc == X86ISD::PEXTRW) &&
51854 isNullConstant(V.getOperand(1)) && V.hasOneUse() &&
51855 V.getOperand(0).hasOneUse())
51856 return V.getOperand(0);
51857 return SDValue();
51858 };
51859 if (SDValue Extract = IsExtractedElement(StoredVal)) {
51860 SDValue Trunc = peekThroughOneUseBitcasts(Extract);
51861 if (Trunc.getOpcode() == X86ISD::VTRUNC) {
51862 SDValue Src = Trunc.getOperand(0);
51863 MVT DstVT = Trunc.getSimpleValueType();
51864 MVT SrcVT = Src.getSimpleValueType();
51865 unsigned NumSrcElts = SrcVT.getVectorNumElements();
51866 unsigned NumTruncBits = DstVT.getScalarSizeInBits() * NumSrcElts;
51867 MVT TruncVT = MVT::getVectorVT(DstVT.getScalarType(), NumSrcElts);
51868 if (NumTruncBits == VT.getSizeInBits() &&
51869 TLI.isTruncStoreLegal(SrcVT, TruncVT)) {
51870 return DAG.getTruncStore(St->getChain(), dl, Src, St->getBasePtr(),
51871 TruncVT, St->getMemOperand());
51872 }
51873 }
51874 }
51875 }
51876
51877 // Optimize trunc store (of multiple scalars) to shuffle and store.
51878 // First, pack all of the elements in one place. Next, store to memory
51879 // in fewer chunks.
51880 if (St->isTruncatingStore() && VT.isVector()) {
51881 if (TLI.isTruncStoreLegal(VT, StVT)) {
51882 if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT()))
51883 return EmitTruncSStore(true /* Signed saturation */, St->getChain(),
51884 dl, Val, St->getBasePtr(),
51885 St->getMemoryVT(), St->getMemOperand(), DAG);
51886 if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(),
51887 DAG, dl))
51888 return EmitTruncSStore(false /* Unsigned saturation */, St->getChain(),
51889 dl, Val, St->getBasePtr(),
51890 St->getMemoryVT(), St->getMemOperand(), DAG);
51891 }
51892
51893 return SDValue();
51894 }
51895
51896 // Cast ptr32 and ptr64 pointers to the default address space before a store.
51897 unsigned AddrSpace = St->getAddressSpace();
51898 if (AddrSpace == X86AS::PTR64 || AddrSpace == X86AS::PTR32_SPTR ||
51899 AddrSpace == X86AS::PTR32_UPTR) {
51900 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
51901 if (PtrVT != St->getBasePtr().getSimpleValueType()) {
51902 SDValue Cast =
51903 DAG.getAddrSpaceCast(dl, PtrVT, St->getBasePtr(), AddrSpace, 0);
51904 return DAG.getTruncStore(
51905 St->getChain(), dl, StoredVal, Cast, St->getPointerInfo(), StVT,
51906 St->getOriginalAlign(), St->getMemOperand()->getFlags(),
51907 St->getAAInfo());
51908 }
51909 }
51910
51911 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering
51912 // the FP state in cases where an emms may be missing.
51913 // A preferable solution to the general problem is to figure out the right
51914 // places to insert EMMS. This qualifies as a quick hack.
51915
51916 // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
51917 if (VT.getSizeInBits() != 64)
51918 return SDValue();
51919
51920 const Function &F = DAG.getMachineFunction().getFunction();
51921 bool NoImplicitFloatOps = F.hasFnAttribute(Attribute::NoImplicitFloat);
51922 bool F64IsLegal =
51923 !Subtarget.useSoftFloat() && !NoImplicitFloatOps && Subtarget.hasSSE2();
51924
51925 if (!F64IsLegal || Subtarget.is64Bit())
51926 return SDValue();
51927
51928 if (VT == MVT::i64 && isa<LoadSDNode>(St->getValue()) &&
51929 cast<LoadSDNode>(St->getValue())->isSimple() &&
51930 St->getChain().hasOneUse() && St->isSimple()) {
51931 auto *Ld = cast<LoadSDNode>(St->getValue());
51932
51933 if (!ISD::isNormalLoad(Ld))
51934 return SDValue();
51935
51936 // Avoid the transformation if there are multiple uses of the loaded value.
51937 if (!Ld->hasNUsesOfValue(1, 0))
51938 return SDValue();
51939
51940 SDLoc LdDL(Ld);
51941 SDLoc StDL(N);
51942 // Lower to a single movq load/store pair.
51943 SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(),
51944 Ld->getBasePtr(), Ld->getMemOperand());
51945
51946 // Make sure new load is placed in same chain order.
51947 DAG.makeEquivalentMemoryOrdering(Ld, NewLd);
51948 return DAG.getStore(St->getChain(), StDL, NewLd, St->getBasePtr(),
51949 St->getMemOperand());
51950 }
51951
51952 // This is similar to the above case, but here we handle a scalar 64-bit
51953 // integer store that is extracted from a vector on a 32-bit target.
51954 // If we have SSE2, then we can treat it like a floating-point double
51955 // to get past legalization. The execution dependencies fixup pass will
51956 // choose the optimal machine instruction for the store if this really is
51957 // an integer or v2f32 rather than an f64.
51958 if (VT == MVT::i64 &&
51959 St->getOperand(1).getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
51960 SDValue OldExtract = St->getOperand(1);
51961 SDValue ExtOp0 = OldExtract.getOperand(0);
51962 unsigned VecSize = ExtOp0.getValueSizeInBits();
51963 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64);
51964 SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0);
51965 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
51966 BitCast, OldExtract.getOperand(1));
51967 return DAG.getStore(St->getChain(), dl, NewExtract, St->getBasePtr(),
51968 St->getPointerInfo(), St->getOriginalAlign(),
51969 St->getMemOperand()->getFlags());
51970 }
51971
51972 return SDValue();
51973}
51974
51975static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG,
51976 TargetLowering::DAGCombinerInfo &DCI,
51977 const X86Subtarget &Subtarget) {
51978 auto *St = cast<MemIntrinsicSDNode>(N);
51979
51980 SDValue StoredVal = N->getOperand(1);
51981 MVT VT = StoredVal.getSimpleValueType();
51982 EVT MemVT = St->getMemoryVT();
51983
51984 // Figure out which elements we demand.
51985 unsigned StElts = MemVT.getSizeInBits() / VT.getScalarSizeInBits();
51986 APInt DemandedElts = APInt::getLowBitsSet(VT.getVectorNumElements(), StElts);
51987
51988 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
51989 if (TLI.SimplifyDemandedVectorElts(StoredVal, DemandedElts, DCI)) {
51990 if (N->getOpcode() != ISD::DELETED_NODE)
51991 DCI.AddToWorklist(N);
51992 return SDValue(N, 0);
51993 }
51994
51995 return SDValue();
51996}
51997
51998/// Return 'true' if this vector operation is "horizontal"
51999/// and return the operands for the horizontal operation in LHS and RHS. A
52000/// horizontal operation performs the binary operation on successive elements
52001/// of its first operand, then on successive elements of its second operand,
52002/// returning the resulting values in a vector. For example, if
52003/// A = < float a0, float a1, float a2, float a3 >
52004/// and
52005/// B = < float b0, float b1, float b2, float b3 >
52006/// then the result of doing a horizontal operation on A and B is
52007/// A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
52008/// In short, LHS and RHS are inspected to see if LHS op RHS is of the form
52009/// A horizontal-op B, for some already available A and B, and if so then LHS is
52010/// set to A, RHS to B, and the routine returns 'true'.
52011static bool isHorizontalBinOp(unsigned HOpcode, SDValue &LHS, SDValue &RHS,
52012 SelectionDAG &DAG, const X86Subtarget &Subtarget,
52013 bool IsCommutative,
52014 SmallVectorImpl<int> &PostShuffleMask,
52015 bool ForceHorizOp) {
52016 // If either operand is undef, bail out. The binop should be simplified.
52017 if (LHS.isUndef() || RHS.isUndef())
52018 return false;
52019
52020 // Look for the following pattern:
52021 // A = < float a0, float a1, float a2, float a3 >
52022 // B = < float b0, float b1, float b2, float b3 >
52023 // and
52024 // LHS = VECTOR_SHUFFLE A, B, <0, 2, 4, 6>
52025 // RHS = VECTOR_SHUFFLE A, B, <1, 3, 5, 7>
52026 // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
52027 // which is A horizontal-op B.
52028
52029 MVT VT = LHS.getSimpleValueType();
52030 assert((VT.is128BitVector() || VT.is256BitVector()) &&
52031 "Unsupported vector type for horizontal add/sub");
52032 unsigned NumElts = VT.getVectorNumElements();
52033
52034 auto GetShuffle = [&](SDValue Op, SDValue &N0, SDValue &N1,
52035 SmallVectorImpl<int> &ShuffleMask) {
52036 bool UseSubVector = false;
52037 if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
52038 Op.getOperand(0).getValueType().is256BitVector() &&
52039 llvm::isNullConstant(Op.getOperand(1))) {
52040 Op = Op.getOperand(0);
52041 UseSubVector = true;
52042 }
52043 SmallVector<SDValue, 2> SrcOps;
52044 SmallVector<int, 16> SrcMask, ScaledMask;
52045 SDValue BC = peekThroughBitcasts(Op);
52046 if (getTargetShuffleInputs(BC, SrcOps, SrcMask, DAG) &&
52047 !isAnyZero(SrcMask) && all_of(SrcOps, [BC](SDValue Op) {
52048 return Op.getValueSizeInBits() == BC.getValueSizeInBits();
52049 })) {
52050 resolveTargetShuffleInputsAndMask(SrcOps, SrcMask);
52051 if (!UseSubVector && SrcOps.size() <= 2 &&
52052 scaleShuffleElements(SrcMask, NumElts, ScaledMask)) {
52053 N0 = !SrcOps.empty() ? SrcOps[0] : SDValue();
52054 N1 = SrcOps.size() > 1 ? SrcOps[1] : SDValue();
52055 ShuffleMask.assign(ScaledMask.begin(), ScaledMask.end());
52056 }
52057 if (UseSubVector && SrcOps.size() == 1 &&
52058 scaleShuffleElements(SrcMask, 2 * NumElts, ScaledMask)) {
52059 std::tie(N0, N1) = DAG.SplitVector(SrcOps[0], SDLoc(Op));
52060 ArrayRef<int> Mask = ArrayRef<int>(ScaledMask).slice(0, NumElts);
52061 ShuffleMask.assign(Mask.begin(), Mask.end());
52062 }
52063 }
52064 };
52065
52066 // View LHS in the form
52067 // LHS = VECTOR_SHUFFLE A, B, LMask
52068 // If LHS is not a shuffle, then pretend it is the identity shuffle:
52069 // LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1>
52070 // NOTE: A default initialized SDValue represents an UNDEF of type VT.
52071 SDValue A, B;
52072 SmallVector<int, 16> LMask;
52073 GetShuffle(LHS, A, B, LMask);
52074
52075 // Likewise, view RHS in the form
52076 // RHS = VECTOR_SHUFFLE C, D, RMask
52077 SDValue C, D;
52078 SmallVector<int, 16> RMask;
52079 GetShuffle(RHS, C, D, RMask);
52080
52081 // At least one of the operands should be a vector shuffle.
52082 unsigned NumShuffles = (LMask.empty() ? 0 : 1) + (RMask.empty() ? 0 : 1);
52083 if (NumShuffles == 0)
52084 return false;
52085
52086 if (LMask.empty()) {
52087 A = LHS;
52088 for (unsigned i = 0; i != NumElts; ++i)
52089 LMask.push_back(i);
52090 }
52091
52092 if (RMask.empty()) {
52093 C = RHS;
52094 for (unsigned i = 0; i != NumElts; ++i)
52095 RMask.push_back(i);
52096 }
52097
52098 // If we have an unary mask, ensure the other op is set to null.
52099 if (isUndefOrInRange(LMask, 0, NumElts))
52100 B = SDValue();
52101 else if (isUndefOrInRange(LMask, NumElts, NumElts * 2))
52102 A = SDValue();
52103
52104 if (isUndefOrInRange(RMask, 0, NumElts))
52105 D = SDValue();
52106 else if (isUndefOrInRange(RMask, NumElts, NumElts * 2))
52107 C = SDValue();
52108
52109 // If A and B occur in reverse order in RHS, then canonicalize by commuting
52110 // RHS operands and shuffle mask.
52111 if (A != C) {
52112 std::swap(C, D);
52113 ShuffleVectorSDNode::commuteMask(RMask);
52114 }
52115 // Check that the shuffles are both shuffling the same vectors.
52116 if (!(A == C && B == D))
52117 return false;
52118
52119 PostShuffleMask.clear();
52120 PostShuffleMask.append(NumElts, SM_SentinelUndef);
52121
52122 // LHS and RHS are now:
52123 // LHS = shuffle A, B, LMask
52124 // RHS = shuffle A, B, RMask
52125 // Check that the masks correspond to performing a horizontal operation.
52126 // AVX defines horizontal add/sub to operate independently on 128-bit lanes,
52127 // so we just repeat the inner loop if this is a 256-bit op.
52128 unsigned Num128BitChunks = VT.getSizeInBits() / 128;
52129 unsigned NumEltsPer128BitChunk = NumElts / Num128BitChunks;
52130 unsigned NumEltsPer64BitChunk = NumEltsPer128BitChunk / 2;
52131 assert((NumEltsPer128BitChunk % 2 == 0) &&
52132 "Vector type should have an even number of elements in each lane");
52133 for (unsigned j = 0; j != NumElts; j += NumEltsPer128BitChunk) {
52134 for (unsigned i = 0; i != NumEltsPer128BitChunk; ++i) {
52135 // Ignore undefined components.
52136 int LIdx = LMask[i + j], RIdx = RMask[i + j];
52137 if (LIdx < 0 || RIdx < 0 ||
52138 (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) ||
52139 (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts)))
52140 continue;
52141
52142 // Check that successive odd/even elements are being operated on. If not,
52143 // this is not a horizontal operation.
52144 if (!((RIdx & 1) == 1 && (LIdx + 1) == RIdx) &&
52145 !((LIdx & 1) == 1 && (RIdx + 1) == LIdx && IsCommutative))
52146 return false;
52147
52148 // Compute the post-shuffle mask index based on where the element
52149 // is stored in the HOP result, and where it needs to be moved to.
52150 int Base = LIdx & ~1u;
52151 int Index = ((Base % NumEltsPer128BitChunk) / 2) +
52152 ((Base % NumElts) & ~(NumEltsPer128BitChunk - 1));
52153
52154 // The low half of the 128-bit result must choose from A.
52155 // The high half of the 128-bit result must choose from B,
52156 // unless B is undef. In that case, we are always choosing from A.
52157 if ((B && Base >= (int)NumElts) || (!B && i >= NumEltsPer64BitChunk))
52158 Index += NumEltsPer64BitChunk;
52159 PostShuffleMask[i + j] = Index;
52160 }
52161 }
52162
52163 SDValue NewLHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
52164 SDValue NewRHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
52165
52166 bool IsIdentityPostShuffle =
52167 isSequentialOrUndefInRange(PostShuffleMask, 0, NumElts, 0);
52168 if (IsIdentityPostShuffle)
52169 PostShuffleMask.clear();
52170
52171 // Avoid 128-bit multi lane shuffles if pre-AVX2 and FP (integer will split).
52172 if (!IsIdentityPostShuffle && !Subtarget.hasAVX2() && VT.isFloatingPoint() &&
52173 isMultiLaneShuffleMask(128, VT.getScalarSizeInBits(), PostShuffleMask))
52174 return false;
52175
52176 // If the source nodes are already used in HorizOps then always accept this.
52177 // Shuffle folding should merge these back together.
52178 auto FoundHorizUser = [&](SDNode *User) {
52179 return User->getOpcode() == HOpcode && User->getValueType(0) == VT;
52180 };
52181 ForceHorizOp =
52182 ForceHorizOp || (llvm::any_of(NewLHS->uses(), FoundHorizUser) &&
52183 llvm::any_of(NewRHS->uses(), FoundHorizUser));
52184
52185 // Assume a SingleSource HOP if we only shuffle one input and don't need to
52186 // shuffle the result.
52187 if (!ForceHorizOp &&
52188 !shouldUseHorizontalOp(NewLHS == NewRHS &&
52189 (NumShuffles < 2 || !IsIdentityPostShuffle),
52190 DAG, Subtarget))
52191 return false;
52192
52193 LHS = DAG.getBitcast(VT, NewLHS);
52194 RHS = DAG.getBitcast(VT, NewRHS);
52195 return true;
52196}
52197
52198// Try to synthesize horizontal (f)hadd/hsub from (f)adds/subs of shuffles.
52199static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG,
52200 const X86Subtarget &Subtarget) {
52201 EVT VT = N->getValueType(0);
52202 unsigned Opcode = N->getOpcode();
52203 bool IsAdd = (Opcode == ISD::FADD) || (Opcode == ISD::ADD);
52204 SmallVector<int, 8> PostShuffleMask;
52205
52206 auto MergableHorizOp = [N](unsigned HorizOpcode) {
52207 return N->hasOneUse() &&
52208 N->use_begin()->getOpcode() == ISD::VECTOR_SHUFFLE &&
52209 (N->use_begin()->getOperand(0).getOpcode() == HorizOpcode ||
52210 N->use_begin()->getOperand(1).getOpcode() == HorizOpcode);
52211 };
52212
52213 switch (Opcode) {
52214 case ISD::FADD:
52215 case ISD::FSUB:
52216 if ((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
52217 (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) {
52218 SDValue LHS = N->getOperand(0);
52219 SDValue RHS = N->getOperand(1);
52220 auto HorizOpcode = IsAdd ? X86ISD::FHADD : X86ISD::FHSUB;
52221 if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd,
52222 PostShuffleMask, MergableHorizOp(HorizOpcode))) {
52223 SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
52224 if (!PostShuffleMask.empty())
52225 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
52226 DAG.getUNDEF(VT), PostShuffleMask);
52227 return HorizBinOp;
52228 }
52229 }
52230 break;
52231 case ISD::ADD:
52232 case ISD::SUB:
52233 if (Subtarget.hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32 ||
52234 VT == MVT::v16i16 || VT == MVT::v8i32)) {
52235 SDValue LHS = N->getOperand(0);
52236 SDValue RHS = N->getOperand(1);
52237 auto HorizOpcode = IsAdd ? X86ISD::HADD : X86ISD::HSUB;
52238 if (isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsAdd,
52239 PostShuffleMask, MergableHorizOp(HorizOpcode))) {
52240 auto HOpBuilder = [HorizOpcode](SelectionDAG &DAG, const SDLoc &DL,
52241 ArrayRef<SDValue> Ops) {
52242 return DAG.getNode(HorizOpcode, DL, Ops[0].getValueType(), Ops);
52243 };
52244 SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT,
52245 {LHS, RHS}, HOpBuilder);
52246 if (!PostShuffleMask.empty())
52247 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp,
52248 DAG.getUNDEF(VT), PostShuffleMask);
52249 return HorizBinOp;
52250 }
52251 }
52252 break;
52253 }
52254
52255 return SDValue();
52256}
52257
52258// Try to combine the following nodes
52259// t29: i64 = X86ISD::Wrapper TargetConstantPool:i64
52260// <i32 -2147483648[float -0.000000e+00]> 0
52261// t27: v16i32[v16f32],ch = X86ISD::VBROADCAST_LOAD
52262// <(load 4 from constant-pool)> t0, t29
52263// [t30: v16i32 = bitcast t27]
52264// t6: v16i32 = xor t7, t27[t30]
52265// t11: v16f32 = bitcast t6
52266// t21: v16f32 = X86ISD::VFMULC[X86ISD::VCFMULC] t11, t8
52267// into X86ISD::VFCMULC[X86ISD::VFMULC] if possible:
52268// t22: v16f32 = bitcast t7
52269// t23: v16f32 = X86ISD::VFCMULC[X86ISD::VFMULC] t8, t22
52270// t24: v32f16 = bitcast t23
52271static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG,
52272 const X86Subtarget &Subtarget) {
52273 EVT VT = N->getValueType(0);
52274 SDValue LHS = N->getOperand(0);
52275 SDValue RHS = N->getOperand(1);
52276 int CombineOpcode =
52277 N->getOpcode() == X86ISD::VFCMULC ? X86ISD::VFMULC : X86ISD::VFCMULC;
52278 auto combineConjugation = [&](SDValue &r) {
52279 if (LHS->getOpcode() == ISD::BITCAST && RHS.hasOneUse()) {
52280 SDValue XOR = LHS.getOperand(0);
52281 if (XOR->getOpcode() == ISD::XOR && XOR.hasOneUse()) {
52282 KnownBits XORRHS = DAG.computeKnownBits(XOR.getOperand(1));
52283 if (XORRHS.isConstant()) {
52284 APInt ConjugationInt32 = APInt(32, 0x80000000, true);
52285 APInt ConjugationInt64 = APInt(64, 0x8000000080000000ULL, true);
52286 if ((XORRHS.getBitWidth() == 32 &&
52287 XORRHS.getConstant() == ConjugationInt32) ||
52288 (XORRHS.getBitWidth() == 64 &&
52289 XORRHS.getConstant() == ConjugationInt64)) {
52290 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
52291 SDValue I2F = DAG.getBitcast(VT, LHS.getOperand(0).getOperand(0));
52292 SDValue FCMulC = DAG.getNode(CombineOpcode, SDLoc(N), VT, RHS, I2F);
52293 r = DAG.getBitcast(VT, FCMulC);
52294 return true;
52295 }
52296 }
52297 }
52298 }
52299 return false;
52300 };
52301 SDValue Res;
52302 if (combineConjugation(Res))
52303 return Res;
52304 std::swap(LHS, RHS);
52305 if (combineConjugation(Res))
52306 return Res;
52307 return Res;
52308}
52309
52310// Try to combine the following nodes:
52311// FADD(A, FMA(B, C, 0)) and FADD(A, FMUL(B, C)) to FMA(B, C, A)
52312static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG,
52313 const X86Subtarget &Subtarget) {
52314 auto AllowContract = [&DAG](const SDNodeFlags &Flags) {
52315 return DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast ||
52316 Flags.hasAllowContract();
52317 };
52318
52319 auto HasNoSignedZero = [&DAG](const SDNodeFlags &Flags) {
52320 return DAG.getTarget().Options.NoSignedZerosFPMath ||
52321 Flags.hasNoSignedZeros();
52322 };
52323 auto IsVectorAllNegativeZero = [&DAG](SDValue Op) {
52324 APInt AI = APInt(32, 0x80008000, true);
52325 KnownBits Bits = DAG.computeKnownBits(Op);
52326 return Bits.getBitWidth() == 32 && Bits.isConstant() &&
52327 Bits.getConstant() == AI;
52328 };
52329
52330 if (N->getOpcode() != ISD::FADD || !Subtarget.hasFP16() ||
52331 !AllowContract(N->getFlags()))
52332 return SDValue();
52333
52334 EVT VT = N->getValueType(0);
52335 if (VT != MVT::v8f16 && VT != MVT::v16f16 && VT != MVT::v32f16)
52336 return SDValue();
52337
52338 SDValue LHS = N->getOperand(0);
52339 SDValue RHS = N->getOperand(1);
52340 bool IsConj;
52341 SDValue FAddOp1, MulOp0, MulOp1;
52342 auto GetCFmulFrom = [&MulOp0, &MulOp1, &IsConj, &AllowContract,
52343 &IsVectorAllNegativeZero,
52344 &HasNoSignedZero](SDValue N) -> bool {
52345 if (!N.hasOneUse() || N.getOpcode() != ISD::BITCAST)
52346 return false;
52347 SDValue Op0 = N.getOperand(0);
52348 unsigned Opcode = Op0.getOpcode();
52349 if (Op0.hasOneUse() && AllowContract(Op0->getFlags())) {
52350 if ((Opcode == X86ISD::VFMULC || Opcode == X86ISD::VFCMULC)) {
52351 MulOp0 = Op0.getOperand(0);
52352 MulOp1 = Op0.getOperand(1);
52353 IsConj = Opcode == X86ISD::VFCMULC;
52354 return true;
52355 }
52356 if ((Opcode == X86ISD::VFMADDC || Opcode == X86ISD::VFCMADDC) &&
52357 ((ISD::isBuildVectorAllZeros(Op0->getOperand(2).getNode()) &&
52358 HasNoSignedZero(Op0->getFlags())) ||
52359 IsVectorAllNegativeZero(Op0->getOperand(2)))) {
52360 MulOp0 = Op0.getOperand(0);
52361 MulOp1 = Op0.getOperand(1);
52362 IsConj = Opcode == X86ISD::VFCMADDC;
52363 return true;
52364 }
52365 }
52366 return false;
52367 };
52368
52369 if (GetCFmulFrom(LHS))
52370 FAddOp1 = RHS;
52371 else if (GetCFmulFrom(RHS))
52372 FAddOp1 = LHS;
52373 else
52374 return SDValue();
52375
52376 MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);
52377 FAddOp1 = DAG.getBitcast(CVT, FAddOp1);
52378 unsigned NewOp = IsConj ? X86ISD::VFCMADDC : X86ISD::VFMADDC;
52379 // FIXME: How do we handle when fast math flags of FADD are different from
52380 // CFMUL's?
52381 SDValue CFmul =
52382 DAG.getNode(NewOp, SDLoc(N), CVT, MulOp0, MulOp1, FAddOp1, N->getFlags());
52383 return DAG.getBitcast(VT, CFmul);
52384}
52385
52386/// Do target-specific dag combines on floating-point adds/subs.
52387static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
52388 const X86Subtarget &Subtarget) {
52389 if (SDValue HOp = combineToHorizontalAddSub(N, DAG, Subtarget))
52390 return HOp;
52391
52392 if (SDValue COp = combineFaddCFmul(N, DAG, Subtarget))
52393 return COp;
52394
52395 return SDValue();
52396}
52397
52398static SDValue combineLRINT_LLRINT(SDNode *N, SelectionDAG &DAG,
52399 const X86Subtarget &Subtarget) {
52400 EVT VT = N->getValueType(0);
52401 SDValue Src = N->getOperand(0);
52402 EVT SrcVT = Src.getValueType();
52403 SDLoc DL(N);
52404
52405 if (!Subtarget.hasDQI() || !Subtarget.hasVLX() || VT != MVT::v2i64 ||
52406 SrcVT != MVT::v2f32)
52407 return SDValue();
52408
52409 return DAG.getNode(X86ISD::CVTP2SI, DL, VT,
52410 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, Src,
52411 DAG.getUNDEF(SrcVT)));
52412}
52413
52414/// Attempt to pre-truncate inputs to arithmetic ops if it will simplify
52415/// the codegen.
52416/// e.g. TRUNC( BINOP( X, Y ) ) --> BINOP( TRUNC( X ), TRUNC( Y ) )
52417/// TODO: This overlaps with the generic combiner's visitTRUNCATE. Remove
52418/// anything that is guaranteed to be transformed by DAGCombiner.
52419static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG,
52420 const X86Subtarget &Subtarget,
52421 const SDLoc &DL) {
52422 assert(N->getOpcode() == ISD::TRUNCATE && "Wrong opcode");
52423 SDValue Src = N->getOperand(0);
52424 unsigned SrcOpcode = Src.getOpcode();
52425 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52426
52427 EVT VT = N->getValueType(0);
52428 EVT SrcVT = Src.getValueType();
52429
52430 auto IsFreeTruncation = [VT](SDValue Op) {
52431 unsigned TruncSizeInBits = VT.getScalarSizeInBits();
52432
52433 // See if this has been extended from a smaller/equal size to
52434 // the truncation size, allowing a truncation to combine with the extend.
52435 unsigned Opcode = Op.getOpcode();
52436 if ((Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND ||
52437 Opcode == ISD::ZERO_EXTEND) &&
52438 Op.getOperand(0).getScalarValueSizeInBits() <= TruncSizeInBits)
52439 return true;
52440
52441 // See if this is a single use constant which can be constant folded.
52442 // NOTE: We don't peek throught bitcasts here because there is currently
52443 // no support for constant folding truncate+bitcast+vector_of_constants. So
52444 // we'll just send up with a truncate on both operands which will
52445 // get turned back into (truncate (binop)) causing an infinite loop.
52446 return ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
52447 };
52448
52449 auto TruncateArithmetic = [&](SDValue N0, SDValue N1) {
52450 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
52451 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
52452 return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1);
52453 };
52454
52455 // Don't combine if the operation has other uses.
52456 if (!Src.hasOneUse())
52457 return SDValue();
52458
52459 // Only support vector truncation for now.
52460 // TODO: i64 scalar math would benefit as well.
52461 if (!VT.isVector())
52462 return SDValue();
52463
52464 // In most cases its only worth pre-truncating if we're only facing the cost
52465 // of one truncation.
52466 // i.e. if one of the inputs will constant fold or the input is repeated.
52467 switch (SrcOpcode) {
52468 case ISD::MUL:
52469 // X86 is rubbish at scalar and vector i64 multiplies (until AVX512DQ) - its
52470 // better to truncate if we have the chance.
52471 if (SrcVT.getScalarType() == MVT::i64 &&
52472 TLI.isOperationLegal(SrcOpcode, VT) &&
52473 !TLI.isOperationLegal(SrcOpcode, SrcVT))
52474 return TruncateArithmetic(Src.getOperand(0), Src.getOperand(1));
52475 [[fallthrough]];
52476 case ISD::AND:
52477 case ISD::XOR:
52478 case ISD::OR:
52479 case ISD::ADD:
52480 case ISD::SUB: {
52481 SDValue Op0 = Src.getOperand(0);
52482 SDValue Op1 = Src.getOperand(1);
52483 if (TLI.isOperationLegal(SrcOpcode, VT) &&
52484 (Op0 == Op1 || IsFreeTruncation(Op0) || IsFreeTruncation(Op1)))
52485 return TruncateArithmetic(Op0, Op1);
52486 break;
52487 }
52488 }
52489
52490 return SDValue();
52491}
52492
52493// Try to form a MULHU or MULHS node by looking for
52494// (trunc (srl (mul ext, ext), 16))
52495// TODO: This is X86 specific because we want to be able to handle wide types
52496// before type legalization. But we can only do it if the vector will be
52497// legalized via widening/splitting. Type legalization can't handle promotion
52498// of a MULHU/MULHS. There isn't a way to convey this to the generic DAG
52499// combiner.
52500static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
52501 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
52502 // First instruction should be a right shift of a multiply.
52503 if (Src.getOpcode() != ISD::SRL ||
52504 Src.getOperand(0).getOpcode() != ISD::MUL)
52505 return SDValue();
52506
52507 if (!Subtarget.hasSSE2())
52508 return SDValue();
52509
52510 // Only handle vXi16 types that are at least 128-bits unless they will be
52511 // widened.
52512 if (!VT.isVector() || VT.getVectorElementType() != MVT::i16)
52513 return SDValue();
52514
52515 // Input type should be at least vXi32.
52516 EVT InVT = Src.getValueType();
52517 if (InVT.getVectorElementType().getSizeInBits() < 32)
52518 return SDValue();
52519
52520 // Need a shift by 16.
52521 APInt ShiftAmt;
52522 if (!ISD::isConstantSplatVector(Src.getOperand(1).getNode(), ShiftAmt) ||
52523 ShiftAmt != 16)
52524 return SDValue();
52525
52526 SDValue LHS = Src.getOperand(0).getOperand(0);
52527 SDValue RHS = Src.getOperand(0).getOperand(1);
52528
52529 // Count leading sign/zero bits on both inputs - if there are enough then
52530 // truncation back to vXi16 will be cheap - either as a pack/shuffle
52531 // sequence or using AVX512 truncations. If the inputs are sext/zext then the
52532 // truncations may actually be free by peeking through to the ext source.
52533 auto IsSext = [&DAG](SDValue V) {
52534 return DAG.ComputeMaxSignificantBits(V) <= 16;
52535 };
52536 auto IsZext = [&DAG](SDValue V) {
52537 return DAG.computeKnownBits(V).countMaxActiveBits() <= 16;
52538 };
52539
52540 bool IsSigned = IsSext(LHS) && IsSext(RHS);
52541 bool IsUnsigned = IsZext(LHS) && IsZext(RHS);
52542 if (!IsSigned && !IsUnsigned)
52543 return SDValue();
52544
52545 // Check if both inputs are extensions, which will be removed by truncation.
52546 bool IsTruncateFree = (LHS.getOpcode() == ISD::SIGN_EXTEND ||
52547 LHS.getOpcode() == ISD::ZERO_EXTEND) &&
52548 (RHS.getOpcode() == ISD::SIGN_EXTEND ||
52549 RHS.getOpcode() == ISD::ZERO_EXTEND) &&
52550 LHS.getOperand(0).getScalarValueSizeInBits() <= 16 &&
52551 RHS.getOperand(0).getScalarValueSizeInBits() <= 16;
52552
52553 // For AVX2+ targets, with the upper bits known zero, we can perform MULHU on
52554 // the (bitcasted) inputs directly, and then cheaply pack/truncate the result
52555 // (upper elts will be zero). Don't attempt this with just AVX512F as MULHU
52556 // will have to split anyway.
52557 unsigned InSizeInBits = InVT.getSizeInBits();
52558 if (IsUnsigned && !IsTruncateFree && Subtarget.hasInt256() &&
52559 !(Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.is256BitVector()) &&
52560 (InSizeInBits % 16) == 0) {
52561 EVT BCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
52562 InVT.getSizeInBits() / 16);
52563 SDValue Res = DAG.getNode(ISD::MULHU, DL, BCVT, DAG.getBitcast(BCVT, LHS),
52564 DAG.getBitcast(BCVT, RHS));
52565 return DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getBitcast(InVT, Res));
52566 }
52567
52568 // Truncate back to source type.
52569 LHS = DAG.getNode(ISD::TRUNCATE, DL, VT, LHS);
52570 RHS = DAG.getNode(ISD::TRUNCATE, DL, VT, RHS);
52571
52572 unsigned Opc = IsSigned ? ISD::MULHS : ISD::MULHU;
52573 return DAG.getNode(Opc, DL, VT, LHS, RHS);
52574}
52575
52576// Attempt to match PMADDUBSW, which multiplies corresponding unsigned bytes
52577// from one vector with signed bytes from another vector, adds together
52578// adjacent pairs of 16-bit products, and saturates the result before
52579// truncating to 16-bits.
52580//
52581// Which looks something like this:
52582// (i16 (ssat (add (mul (zext (even elts (i8 A))), (sext (even elts (i8 B)))),
52583// (mul (zext (odd elts (i8 A)), (sext (odd elts (i8 B))))))))
52584static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
52585 const X86Subtarget &Subtarget,
52586 const SDLoc &DL) {
52587 if (!VT.isVector() || !Subtarget.hasSSSE3())
52588 return SDValue();
52589
52590 unsigned NumElems = VT.getVectorNumElements();
52591 EVT ScalarVT = VT.getVectorElementType();
52592 if (ScalarVT != MVT::i16 || NumElems < 8 || !isPowerOf2_32(NumElems))
52593 return SDValue();
52594
52595 SDValue SSatVal = detectSSatPattern(In, VT);
52596 if (!SSatVal || SSatVal.getOpcode() != ISD::ADD)
52597 return SDValue();
52598
52599 // Ok this is a signed saturation of an ADD. See if this ADD is adding pairs
52600 // of multiplies from even/odd elements.
52601 SDValue N0 = SSatVal.getOperand(0);
52602 SDValue N1 = SSatVal.getOperand(1);
52603
52604 if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL)
52605 return SDValue();
52606
52607 SDValue N00 = N0.getOperand(0);
52608 SDValue N01 = N0.getOperand(1);
52609 SDValue N10 = N1.getOperand(0);
52610 SDValue N11 = N1.getOperand(1);
52611
52612 // TODO: Handle constant vectors and use knownbits/computenumsignbits?
52613 // Canonicalize zero_extend to LHS.
52614 if (N01.getOpcode() == ISD::ZERO_EXTEND)
52615 std::swap(N00, N01);
52616 if (N11.getOpcode() == ISD::ZERO_EXTEND)
52617 std::swap(N10, N11);
52618
52619 // Ensure we have a zero_extend and a sign_extend.
52620 if (N00.getOpcode() != ISD::ZERO_EXTEND ||
52621 N01.getOpcode() != ISD::SIGN_EXTEND ||
52622 N10.getOpcode() != ISD::ZERO_EXTEND ||
52623 N11.getOpcode() != ISD::SIGN_EXTEND)
52624 return SDValue();
52625
52626 // Peek through the extends.
52627 N00 = N00.getOperand(0);
52628 N01 = N01.getOperand(0);
52629 N10 = N10.getOperand(0);
52630 N11 = N11.getOperand(0);
52631
52632 // Ensure the extend is from vXi8.
52633 if (N00.getValueType().getVectorElementType() != MVT::i8 ||
52634 N01.getValueType().getVectorElementType() != MVT::i8 ||
52635 N10.getValueType().getVectorElementType() != MVT::i8 ||
52636 N11.getValueType().getVectorElementType() != MVT::i8)
52637 return SDValue();
52638
52639 // All inputs should be build_vectors.
52640 if (N00.getOpcode() != ISD::BUILD_VECTOR ||
52641 N01.getOpcode() != ISD::BUILD_VECTOR ||
52642 N10.getOpcode() != ISD::BUILD_VECTOR ||
52643 N11.getOpcode() != ISD::BUILD_VECTOR)
52644 return SDValue();
52645
52646 // N00/N10 are zero extended. N01/N11 are sign extended.
52647
52648 // For each element, we need to ensure we have an odd element from one vector
52649 // multiplied by the odd element of another vector and the even element from
52650 // one of the same vectors being multiplied by the even element from the
52651 // other vector. So we need to make sure for each element i, this operator
52652 // is being performed:
52653 // A[2 * i] * B[2 * i] + A[2 * i + 1] * B[2 * i + 1]
52654 SDValue ZExtIn, SExtIn;
52655 for (unsigned i = 0; i != NumElems; ++i) {
52656 SDValue N00Elt = N00.getOperand(i);
52657 SDValue N01Elt = N01.getOperand(i);
52658 SDValue N10Elt = N10.getOperand(i);
52659 SDValue N11Elt = N11.getOperand(i);
52660 // TODO: Be more tolerant to undefs.
52661 if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
52662 N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
52663 N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
52664 N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
52665 return SDValue();
52666 auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1));
52667 auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1));
52668 auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1));
52669 auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1));
52670 if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt)
52671 return SDValue();
52672 unsigned IdxN00 = ConstN00Elt->getZExtValue();
52673 unsigned IdxN01 = ConstN01Elt->getZExtValue();
52674 unsigned IdxN10 = ConstN10Elt->getZExtValue();
52675 unsigned IdxN11 = ConstN11Elt->getZExtValue();
52676 // Add is commutative so indices can be reordered.
52677 if (IdxN00 > IdxN10) {
52678 std::swap(IdxN00, IdxN10);
52679 std::swap(IdxN01, IdxN11);
52680 }
52681 // N0 indices be the even element. N1 indices must be the next odd element.
52682 if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
52683 IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
52684 return SDValue();
52685 SDValue N00In = N00Elt.getOperand(0);
52686 SDValue N01In = N01Elt.getOperand(0);
52687 SDValue N10In = N10Elt.getOperand(0);
52688 SDValue N11In = N11Elt.getOperand(0);
52689 // First time we find an input capture it.
52690 if (!ZExtIn) {
52691 ZExtIn = N00In;
52692 SExtIn = N01In;
52693 }
52694 if (ZExtIn != N00In || SExtIn != N01In ||
52695 ZExtIn != N10In || SExtIn != N11In)
52696 return SDValue();
52697 }
52698
52699 auto ExtractVec = [&DAG, &DL, NumElems](SDValue &Ext) {
52700 EVT ExtVT = Ext.getValueType();
52701 if (ExtVT.getVectorNumElements() != NumElems * 2) {
52702 MVT NVT = MVT::getVectorVT(MVT::i8, NumElems * 2);
52703 Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, Ext,
52704 DAG.getIntPtrConstant(0, DL));
52705 }
52706 };
52707 ExtractVec(ZExtIn);
52708 ExtractVec(SExtIn);
52709
52710 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
52711 ArrayRef<SDValue> Ops) {
52712 // Shrink by adding truncate nodes and let DAGCombine fold with the
52713 // sources.
52714 EVT InVT = Ops[0].getValueType();
52715 assert(InVT.getScalarType() == MVT::i8 &&
52716 "Unexpected scalar element type");
52717 assert(InVT == Ops[1].getValueType() && "Operands' types mismatch");
52718 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
52719 InVT.getVectorNumElements() / 2);
52720 return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]);
52721 };
52722 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn },
52723 PMADDBuilder);
52724}
52725
52726static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
52727 const X86Subtarget &Subtarget) {
52728 EVT VT = N->getValueType(0);
52729 SDValue Src = N->getOperand(0);
52730 SDLoc DL(N);
52731
52732 // Attempt to pre-truncate inputs to arithmetic ops instead.
52733 if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL))
52734 return V;
52735
52736 // Try to detect PMADD
52737 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))
52738 return PMAdd;
52739
52740 // Try to combine truncation with signed/unsigned saturation.
52741 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))
52742 return Val;
52743
52744 // Try to combine PMULHUW/PMULHW for vXi16.
52745 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
52746 return V;
52747
52748 // The bitcast source is a direct mmx result.
52749 // Detect bitcasts between i32 to x86mmx
52750 if (Src.getOpcode() == ISD::BITCAST && VT == MVT::i32) {
52751 SDValue BCSrc = Src.getOperand(0);
52752 if (BCSrc.getValueType() == MVT::x86mmx)
52753 return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);
52754 }
52755
52756 // Try to combine (trunc (vNi64 (lrint x))) to (vNi32 (lrint x)).
52757 if (Src.getOpcode() == ISD::LRINT && VT.getScalarType() == MVT::i32 &&
52758 Src.hasOneUse())
52759 return DAG.getNode(ISD::LRINT, DL, VT, Src.getOperand(0));
52760
52761 return SDValue();
52762}
52763
52764static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG,
52765 TargetLowering::DAGCombinerInfo &DCI) {
52766 EVT VT = N->getValueType(0);
52767 SDValue In = N->getOperand(0);
52768 SDLoc DL(N);
52769
52770 if (SDValue SSatVal = detectSSatPattern(In, VT))
52771 return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal);
52772 if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL))
52773 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal);
52774
52775 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52776 APInt DemandedMask(APInt::getAllOnes(VT.getScalarSizeInBits()));
52777 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
52778 return SDValue(N, 0);
52779
52780 return SDValue();
52781}
52782
52783/// Returns the negated value if the node \p N flips sign of FP value.
52784///
52785/// FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000)
52786/// or FSUB(0, x)
52787/// AVX512F does not have FXOR, so FNEG is lowered as
52788/// (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))).
52789/// In this case we go though all bitcasts.
52790/// This also recognizes splat of a negated value and returns the splat of that
52791/// value.
52792static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth = 0) {
52793 if (N->getOpcode() == ISD::FNEG)
52794 return N->getOperand(0);
52795
52796 // Don't recurse exponentially.
52797 if (Depth > SelectionDAG::MaxRecursionDepth)
52798 return SDValue();
52799
52800 unsigned ScalarSize = N->getValueType(0).getScalarSizeInBits();
52801
52802 SDValue Op = peekThroughBitcasts(SDValue(N, 0));
52803 EVT VT = Op->getValueType(0);
52804
52805 // Make sure the element size doesn't change.
52806 if (VT.getScalarSizeInBits() != ScalarSize)
52807 return SDValue();
52808
52809 unsigned Opc = Op.getOpcode();
52810 switch (Opc) {
52811 case ISD::VECTOR_SHUFFLE: {
52812 // For a VECTOR_SHUFFLE(VEC1, VEC2), if the VEC2 is undef, then the negate
52813 // of this is VECTOR_SHUFFLE(-VEC1, UNDEF). The mask can be anything here.
52814 if (!Op.getOperand(1).isUndef())
52815 return SDValue();
52816 if (SDValue NegOp0 = isFNEG(DAG, Op.getOperand(0).getNode(), Depth + 1))
52817 if (NegOp0.getValueType() == VT) // FIXME: Can we do better?
52818 return DAG.getVectorShuffle(VT, SDLoc(Op), NegOp0, DAG.getUNDEF(VT),
52819 cast<ShuffleVectorSDNode>(Op)->getMask());
52820 break;
52821 }
52822 case ISD::INSERT_VECTOR_ELT: {
52823 // Negate of INSERT_VECTOR_ELT(UNDEF, V, INDEX) is INSERT_VECTOR_ELT(UNDEF,
52824 // -V, INDEX).
52825 SDValue InsVector = Op.getOperand(0);
52826 SDValue InsVal = Op.getOperand(1);
52827 if (!InsVector.isUndef())
52828 return SDValue();
52829 if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1))
52830 if (NegInsVal.getValueType() == VT.getVectorElementType()) // FIXME
52831 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector,
52832 NegInsVal, Op.getOperand(2));
52833 break;
52834 }
52835 case ISD::FSUB:
52836 case ISD::XOR:
52837 case X86ISD::FXOR: {
52838 SDValue Op1 = Op.getOperand(1);
52839 SDValue Op0 = Op.getOperand(0);
52840
52841 // For XOR and FXOR, we want to check if constant
52842 // bits of Op1 are sign bit masks. For FSUB, we
52843 // have to check if constant bits of Op0 are sign
52844 // bit masks and hence we swap the operands.
52845 if (Opc == ISD::FSUB)
52846 std::swap(Op0, Op1);
52847
52848 APInt UndefElts;
52849 SmallVector<APInt, 16> EltBits;
52850 // Extract constant bits and see if they are all
52851 // sign bit masks. Ignore the undef elements.
52852 if (getTargetConstantBitsFromNode(Op1, ScalarSize, UndefElts, EltBits,
52853 /* AllowWholeUndefs */ true,
52854 /* AllowPartialUndefs */ false)) {
52855 for (unsigned I = 0, E = EltBits.size(); I < E; I++)
52856 if (!UndefElts[I] && !EltBits[I].isSignMask())
52857 return SDValue();
52858
52859 // Only allow bitcast from correctly-sized constant.
52860 Op0 = peekThroughBitcasts(Op0);
52861 if (Op0.getScalarValueSizeInBits() == ScalarSize)
52862 return Op0;
52863 }
52864 break;
52865 } // case
52866 } // switch
52867
52868 return SDValue();
52869}
52870
52871static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
52872 bool NegRes) {
52873 if (NegMul) {
52874 switch (Opcode) {
52875 // clang-format off
52876 default: llvm_unreachable("Unexpected opcode");
52877 case ISD::FMA: Opcode = X86ISD::FNMADD; break;
52878 case ISD::STRICT_FMA: Opcode = X86ISD::STRICT_FNMADD; break;
52879 case X86ISD::FMADD_RND: Opcode = X86ISD::FNMADD_RND; break;
52880 case X86ISD::FMSUB: Opcode = X86ISD::FNMSUB; break;
52881 case X86ISD::STRICT_FMSUB: Opcode = X86ISD::STRICT_FNMSUB; break;
52882 case X86ISD::FMSUB_RND: Opcode = X86ISD::FNMSUB_RND; break;
52883 case X86ISD::FNMADD: Opcode = ISD::FMA; break;
52884 case X86ISD::STRICT_FNMADD: Opcode = ISD::STRICT_FMA; break;
52885 case X86ISD::FNMADD_RND: Opcode = X86ISD::FMADD_RND; break;
52886 case X86ISD::FNMSUB: Opcode = X86ISD::FMSUB; break;
52887 case X86ISD::STRICT_FNMSUB: Opcode = X86ISD::STRICT_FMSUB; break;
52888 case X86ISD::FNMSUB_RND: Opcode = X86ISD::FMSUB_RND; break;
52889 // clang-format on
52890 }
52891 }
52892
52893 if (NegAcc) {
52894 switch (Opcode) {
52895 // clang-format off
52896 default: llvm_unreachable("Unexpected opcode");
52897 case ISD::FMA: Opcode = X86ISD::FMSUB; break;
52898 case ISD::STRICT_FMA: Opcode = X86ISD::STRICT_FMSUB; break;
52899 case X86ISD::FMADD_RND: Opcode = X86ISD::FMSUB_RND; break;
52900 case X86ISD::FMSUB: Opcode = ISD::FMA; break;
52901 case X86ISD::STRICT_FMSUB: Opcode = ISD::STRICT_FMA; break;
52902 case X86ISD::FMSUB_RND: Opcode = X86ISD::FMADD_RND; break;
52903 case X86ISD::FNMADD: Opcode = X86ISD::FNMSUB; break;
52904 case X86ISD::STRICT_FNMADD: Opcode = X86ISD::STRICT_FNMSUB; break;
52905 case X86ISD::FNMADD_RND: Opcode = X86ISD::FNMSUB_RND; break;
52906 case X86ISD::FNMSUB: Opcode = X86ISD::FNMADD; break;
52907 case X86ISD::STRICT_FNMSUB: Opcode = X86ISD::STRICT_FNMADD; break;
52908 case X86ISD::FNMSUB_RND: Opcode = X86ISD::FNMADD_RND; break;
52909 case X86ISD::FMADDSUB: Opcode = X86ISD::FMSUBADD; break;
52910 case X86ISD::FMADDSUB_RND: Opcode = X86ISD::FMSUBADD_RND; break;
52911 case X86ISD::FMSUBADD: Opcode = X86ISD::FMADDSUB; break;
52912 case X86ISD::FMSUBADD_RND: Opcode = X86ISD::FMADDSUB_RND; break;
52913 // clang-format on
52914 }
52915 }
52916
52917 if (NegRes) {
52918 switch (Opcode) {
52919 // For accuracy reason, we never combine fneg and fma under strict FP.
52920 // clang-format off
52921 default: llvm_unreachable("Unexpected opcode");
52922 case ISD::FMA: Opcode = X86ISD::FNMSUB; break;
52923 case X86ISD::FMADD_RND: Opcode = X86ISD::FNMSUB_RND; break;
52924 case X86ISD::FMSUB: Opcode = X86ISD::FNMADD; break;
52925 case X86ISD::FMSUB_RND: Opcode = X86ISD::FNMADD_RND; break;
52926 case X86ISD::FNMADD: Opcode = X86ISD::FMSUB; break;
52927 case X86ISD::FNMADD_RND: Opcode = X86ISD::FMSUB_RND; break;
52928 case X86ISD::FNMSUB: Opcode = ISD::FMA; break;
52929 case X86ISD::FNMSUB_RND: Opcode = X86ISD::FMADD_RND; break;
52930 // clang-format on
52931 }
52932 }
52933
52934 return Opcode;
52935}
52936
52937/// Do target-specific dag combines on floating point negations.
52938static SDValue combineFneg(SDNode *N, SelectionDAG &DAG,
52939 TargetLowering::DAGCombinerInfo &DCI,
52940 const X86Subtarget &Subtarget) {
52941 EVT OrigVT = N->getValueType(0);
52942 SDValue Arg = isFNEG(DAG, N);
52943 if (!Arg)
52944 return SDValue();
52945
52946 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
52947 EVT VT = Arg.getValueType();
52948 EVT SVT = VT.getScalarType();
52949 SDLoc DL(N);
52950
52951 // Let legalize expand this if it isn't a legal type yet.
52952 if (!TLI.isTypeLegal(VT))
52953 return SDValue();
52954
52955 // If we're negating a FMUL node on a target with FMA, then we can avoid the
52956 // use of a constant by performing (-0 - A*B) instead.
52957 // FIXME: Check rounding control flags as well once it becomes available.
52958 if (Arg.getOpcode() == ISD::FMUL && (SVT == MVT::f32 || SVT == MVT::f64) &&
52959 Arg->getFlags().hasNoSignedZeros() && Subtarget.hasAnyFMA()) {
52960 SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
52961 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
52962 Arg.getOperand(1), Zero);
52963 return DAG.getBitcast(OrigVT, NewNode);
52964 }
52965
52966 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
52967 bool LegalOperations = !DCI.isBeforeLegalizeOps();
52968 if (SDValue NegArg =
52969 TLI.getNegatedExpression(Arg, DAG, LegalOperations, CodeSize))
52970 return DAG.getBitcast(OrigVT, NegArg);
52971
52972 return SDValue();
52973}
52974
52975SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
52976 bool LegalOperations,
52977 bool ForCodeSize,
52978 NegatibleCost &Cost,
52979 unsigned Depth) const {
52980 // fneg patterns are removable even if they have multiple uses.
52981 if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth)) {
52982 Cost = NegatibleCost::Cheaper;
52983 return DAG.getBitcast(Op.getValueType(), Arg);
52984 }
52985
52986 EVT VT = Op.getValueType();
52987 EVT SVT = VT.getScalarType();
52988 unsigned Opc = Op.getOpcode();
52989 SDNodeFlags Flags = Op.getNode()->getFlags();
52990 switch (Opc) {
52991 case ISD::FMA:
52992 case X86ISD::FMSUB:
52993 case X86ISD::FNMADD:
52994 case X86ISD::FNMSUB:
52995 case X86ISD::FMADD_RND:
52996 case X86ISD::FMSUB_RND:
52997 case X86ISD::FNMADD_RND:
52998 case X86ISD::FNMSUB_RND: {
52999 if (!Op.hasOneUse() || !Subtarget.hasAnyFMA() || !isTypeLegal(VT) ||
53000 !(SVT == MVT::f32 || SVT == MVT::f64) ||
53001 !isOperationLegal(ISD::FMA, VT))
53002 break;
53003
53004 // Don't fold (fneg (fma (fneg x), y, (fneg z))) to (fma x, y, z)
53005 // if it may have signed zeros.
53006 if (!Flags.hasNoSignedZeros())
53007 break;
53008
53009 // This is always negatible for free but we might be able to remove some
53010 // extra operand negations as well.
53011 SmallVector<SDValue, 4> NewOps(Op.getNumOperands(), SDValue());
53012 for (int i = 0; i != 3; ++i)
53013 NewOps[i] = getCheaperNegatedExpression(
53014 Op.getOperand(i), DAG, LegalOperations, ForCodeSize, Depth + 1);
53015
53016 bool NegA = !!NewOps[0];
53017 bool NegB = !!NewOps[1];
53018 bool NegC = !!NewOps[2];
53019 unsigned NewOpc = negateFMAOpcode(Opc, NegA != NegB, NegC, true);
53020
53021 Cost = (NegA || NegB || NegC) ? NegatibleCost::Cheaper
53022 : NegatibleCost::Neutral;
53023
53024 // Fill in the non-negated ops with the original values.
53025 for (int i = 0, e = Op.getNumOperands(); i != e; ++i)
53026 if (!NewOps[i])
53027 NewOps[i] = Op.getOperand(i);
53028 return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps);
53029 }
53030 case X86ISD::FRCP:
53031 if (SDValue NegOp0 =
53032 getNegatedExpression(Op.getOperand(0), DAG, LegalOperations,
53033 ForCodeSize, Cost, Depth + 1))
53034 return DAG.getNode(Opc, SDLoc(Op), VT, NegOp0);
53035 break;
53036 }
53037
53038 return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations,
53039 ForCodeSize, Cost, Depth);
53040}
53041
53042static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
53043 const X86Subtarget &Subtarget) {
53044 MVT VT = N->getSimpleValueType(0);
53045 // If we have integer vector types available, use the integer opcodes.
53046 if (!VT.isVector() || !Subtarget.hasSSE2())
53047 return SDValue();
53048
53049 SDLoc dl(N);
53050
53051 unsigned IntBits = VT.getScalarSizeInBits();
53052 MVT IntSVT = MVT::getIntegerVT(IntBits);
53053 MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits);
53054
53055 SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
53056 SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
53057 unsigned IntOpcode;
53058 switch (N->getOpcode()) {
53059 // clang-format off
53060 default: llvm_unreachable("Unexpected FP logic op");
53061 case X86ISD::FOR: IntOpcode = ISD::OR; break;
53062 case X86ISD::FXOR: IntOpcode = ISD::XOR; break;
53063 case X86ISD::FAND: IntOpcode = ISD::AND; break;
53064 case X86ISD::FANDN: IntOpcode = X86ISD::ANDNP; break;
53065 // clang-format on
53066 }
53067 SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
53068 return DAG.getBitcast(VT, IntOp);
53069}
53070
53071
53072/// Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
53073static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
53074 if (N->getOpcode() != ISD::XOR)
53075 return SDValue();
53076
53077 SDValue LHS = N->getOperand(0);
53078 if (!isOneConstant(N->getOperand(1)) || LHS->getOpcode() != X86ISD::SETCC)
53079 return SDValue();
53080
53081 X86::CondCode NewCC = X86::GetOppositeBranchCondition(
53082 X86::CondCode(LHS->getConstantOperandVal(0)));
53083 SDLoc DL(N);
53084 return getSETCC(NewCC, LHS->getOperand(1), DL, DAG);
53085}
53086
53087static SDValue combineXorSubCTLZ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
53088 const X86Subtarget &Subtarget) {
53089 assert((N->getOpcode() == ISD::XOR || N->getOpcode() == ISD::SUB) &&
53090 "Invalid opcode for combing with CTLZ");
53091 if (Subtarget.hasFastLZCNT())
53092 return SDValue();
53093
53094 EVT VT = N->getValueType(0);
53095 if (VT != MVT::i8 && VT != MVT::i16 && VT != MVT::i32 &&
53096 (VT != MVT::i64 || !Subtarget.is64Bit()))
53097 return SDValue();
53098
53099 SDValue N0 = N->getOperand(0);
53100 SDValue N1 = N->getOperand(1);
53101
53102 if (N0.getOpcode() != ISD::CTLZ_ZERO_UNDEF &&
53103 N1.getOpcode() != ISD::CTLZ_ZERO_UNDEF)
53104 return SDValue();
53105
53106 SDValue OpCTLZ;
53107 SDValue OpSizeTM1;
53108
53109 if (N1.getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
53110 OpCTLZ = N1;
53111 OpSizeTM1 = N0;
53112 } else if (N->getOpcode() == ISD::SUB) {
53113 return SDValue();
53114 } else {
53115 OpCTLZ = N0;
53116 OpSizeTM1 = N1;
53117 }
53118
53119 if (!OpCTLZ.hasOneUse())
53120 return SDValue();
53121 auto *C = dyn_cast<ConstantSDNode>(OpSizeTM1);
53122 if (!C)
53123 return SDValue();
53124
53125 if (C->getZExtValue() != uint64_t(OpCTLZ.getValueSizeInBits() - 1))
53126 return SDValue();
53127 EVT OpVT = VT;
53128 SDValue Op = OpCTLZ.getOperand(0);
53129 if (VT == MVT::i8) {
53130 // Zero extend to i32 since there is not an i8 bsr.
53131 OpVT = MVT::i32;
53132 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, OpVT, Op);
53133 }
53134
53135 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
53136 Op = DAG.getNode(X86ISD::BSR, DL, VTs, Op);
53137 if (VT == MVT::i8)
53138 Op = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Op);
53139
53140 return Op;
53141}
53142
53143static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
53144 TargetLowering::DAGCombinerInfo &DCI,
53145 const X86Subtarget &Subtarget) {
53146 SDValue N0 = N->getOperand(0);
53147 SDValue N1 = N->getOperand(1);
53148 EVT VT = N->getValueType(0);
53149 SDLoc DL(N);
53150
53151 // If this is SSE1 only convert to FXOR to avoid scalarization.
53152 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
53153 return DAG.getBitcast(MVT::v4i32,
53154 DAG.getNode(X86ISD::FXOR, DL, MVT::v4f32,
53155 DAG.getBitcast(MVT::v4f32, N0),
53156 DAG.getBitcast(MVT::v4f32, N1)));
53157 }
53158
53159 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
53160 return Cmp;
53161
53162 if (SDValue R = combineBitOpWithMOVMSK(N, DAG))
53163 return R;
53164
53165 if (SDValue R = combineBitOpWithShift(N, DAG))
53166 return R;
53167
53168 if (SDValue R = combineBitOpWithPACK(N, DAG))
53169 return R;
53170
53171 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, DCI, Subtarget))
53172 return FPLogic;
53173
53174 if (SDValue R = combineXorSubCTLZ(N, DL, DAG, Subtarget))
53175 return R;
53176
53177 if (DCI.isBeforeLegalizeOps())
53178 return SDValue();
53179
53180 if (SDValue SetCC = foldXor1SetCC(N, DAG))
53181 return SetCC;
53182
53183 if (SDValue R = combineOrXorWithSETCC(N, N0, N1, DAG))
53184 return R;
53185
53186 if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
53187 return RV;
53188
53189 // Fold not(iX bitcast(vXi1)) -> (iX bitcast(not(vec))) for legal boolvecs.
53190 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53191 if (llvm::isAllOnesConstant(N1) && N0.getOpcode() == ISD::BITCAST &&
53192 N0.getOperand(0).getValueType().isVector() &&
53193 N0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
53194 TLI.isTypeLegal(N0.getOperand(0).getValueType()) && N0.hasOneUse()) {
53195 return DAG.getBitcast(
53196 VT, DAG.getNOT(DL, N0.getOperand(0), N0.getOperand(0).getValueType()));
53197 }
53198
53199 // Handle AVX512 mask widening.
53200 // Fold not(insert_subvector(undef,sub)) -> insert_subvector(undef,not(sub))
53201 if (ISD::isBuildVectorAllOnes(N1.getNode()) && VT.isVector() &&
53202 VT.getVectorElementType() == MVT::i1 &&
53203 N0.getOpcode() == ISD::INSERT_SUBVECTOR && N0.getOperand(0).isUndef() &&
53204 TLI.isTypeLegal(N0.getOperand(1).getValueType())) {
53205 return DAG.getNode(
53206 ISD::INSERT_SUBVECTOR, DL, VT, N0.getOperand(0),
53207 DAG.getNOT(DL, N0.getOperand(1), N0.getOperand(1).getValueType()),
53208 N0.getOperand(2));
53209 }
53210
53211 // Fold xor(zext(xor(x,c1)),c2) -> xor(zext(x),xor(zext(c1),c2))
53212 // Fold xor(truncate(xor(x,c1)),c2) -> xor(truncate(x),xor(truncate(c1),c2))
53213 // TODO: Under what circumstances could this be performed in DAGCombine?
53214 if ((N0.getOpcode() == ISD::TRUNCATE || N0.getOpcode() == ISD::ZERO_EXTEND) &&
53215 N0.getOperand(0).getOpcode() == N->getOpcode()) {
53216 SDValue TruncExtSrc = N0.getOperand(0);
53217 auto *N1C = dyn_cast<ConstantSDNode>(N1);
53218 auto *N001C = dyn_cast<ConstantSDNode>(TruncExtSrc.getOperand(1));
53219 if (N1C && !N1C->isOpaque() && N001C && !N001C->isOpaque()) {
53220 SDValue LHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(0), DL, VT);
53221 SDValue RHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(1), DL, VT);
53222 return DAG.getNode(ISD::XOR, DL, VT, LHS,
53223 DAG.getNode(ISD::XOR, DL, VT, RHS, N1));
53224 }
53225 }
53226
53227 if (SDValue R = combineBMILogicOp(N, DAG, Subtarget))
53228 return R;
53229
53230 return combineFneg(N, DAG, DCI, Subtarget);
53231}
53232
53233static SDValue combineBITREVERSE(SDNode *N, SelectionDAG &DAG,
53234 TargetLowering::DAGCombinerInfo &DCI,
53235 const X86Subtarget &Subtarget) {
53236 SDValue N0 = N->getOperand(0);
53237 EVT VT = N->getValueType(0);
53238
53239 // Convert a (iX bitreverse(bitcast(vXi1 X))) -> (iX bitcast(shuffle(X)))
53240 if (VT.isInteger() && N0.getOpcode() == ISD::BITCAST && N0.hasOneUse()) {
53241 SDValue Src = N0.getOperand(0);
53242 EVT SrcVT = Src.getValueType();
53243 if (SrcVT.isVector() && SrcVT.getScalarType() == MVT::i1 &&
53244 (DCI.isBeforeLegalize() ||
53245 DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)) &&
53246 Subtarget.hasSSSE3()) {
53247 unsigned NumElts = SrcVT.getVectorNumElements();
53248 SmallVector<int, 32> ReverseMask(NumElts);
53249 for (unsigned I = 0; I != NumElts; ++I)
53250 ReverseMask[I] = (NumElts - 1) - I;
53251 SDValue Rev =
53252 DAG.getVectorShuffle(SrcVT, SDLoc(N), Src, Src, ReverseMask);
53253 return DAG.getBitcast(VT, Rev);
53254 }
53255 }
53256
53257 return SDValue();
53258}
53259
53260// Various combines to try to convert to avgceilu.
53261static SDValue combineAVG(SDNode *N, SelectionDAG &DAG,
53262 TargetLowering::DAGCombinerInfo &DCI,
53263 const X86Subtarget &Subtarget) {
53264 unsigned Opcode = N->getOpcode();
53265 SDValue N0 = N->getOperand(0);
53266 SDValue N1 = N->getOperand(1);
53267 EVT VT = N->getValueType(0);
53268 EVT SVT = VT.getScalarType();
53269 SDLoc DL(N);
53270
53271 // avgceils(x,y) -> flipsign(avgceilu(flipsign(x),flipsign(y)))
53272 // Only useful on vXi8 which doesn't have good SRA handling.
53273 if (Opcode == ISD::AVGCEILS && VT.isVector() && SVT == MVT::i8) {
53274 APInt SignBit = APInt::getSignMask(VT.getScalarSizeInBits());
53275 SDValue SignMask = DAG.getConstant(SignBit, DL, VT);
53276 N0 = DAG.getNode(ISD::XOR, DL, VT, N0, SignMask);
53277 N1 = DAG.getNode(ISD::XOR, DL, VT, N1, SignMask);
53278 return DAG.getNode(ISD::XOR, DL, VT,
53279 DAG.getNode(ISD::AVGCEILU, DL, VT, N0, N1), SignMask);
53280 }
53281
53282 return SDValue();
53283}
53284
53285static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG,
53286 TargetLowering::DAGCombinerInfo &DCI,
53287 const X86Subtarget &Subtarget) {
53288 EVT VT = N->getValueType(0);
53289 unsigned NumBits = VT.getSizeInBits();
53290
53291 // TODO - Constant Folding.
53292
53293 // Simplify the inputs.
53294 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53295 APInt DemandedMask(APInt::getAllOnes(NumBits));
53296 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
53297 return SDValue(N, 0);
53298
53299 return SDValue();
53300}
53301
53302static bool isNullFPScalarOrVectorConst(SDValue V) {
53303 return isNullFPConstant(V) || ISD::isBuildVectorAllZeros(V.getNode());
53304}
53305
53306/// If a value is a scalar FP zero or a vector FP zero (potentially including
53307/// undefined elements), return a zero constant that may be used to fold away
53308/// that value. In the case of a vector, the returned constant will not contain
53309/// undefined elements even if the input parameter does. This makes it suitable
53310/// to be used as a replacement operand with operations (eg, bitwise-and) where
53311/// an undef should not propagate.
53312static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG,
53313 const X86Subtarget &Subtarget) {
53314 if (!isNullFPScalarOrVectorConst(V))
53315 return SDValue();
53316
53317 if (V.getValueType().isVector())
53318 return getZeroVector(V.getSimpleValueType(), Subtarget, DAG, SDLoc(V));
53319
53320 return V;
53321}
53322
53323static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG,
53324 const X86Subtarget &Subtarget) {
53325 SDValue N0 = N->getOperand(0);
53326 SDValue N1 = N->getOperand(1);
53327 EVT VT = N->getValueType(0);
53328 SDLoc DL(N);
53329
53330 // Vector types are handled in combineANDXORWithAllOnesIntoANDNP().
53331 if (!((VT == MVT::f32 && Subtarget.hasSSE1()) ||
53332 (VT == MVT::f64 && Subtarget.hasSSE2()) ||
53333 (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2())))
53334 return SDValue();
53335
53336 auto isAllOnesConstantFP = [](SDValue V) {
53337 if (V.getSimpleValueType().isVector())
53338 return ISD::isBuildVectorAllOnes(V.getNode());
53339 auto *C = dyn_cast<ConstantFPSDNode>(V);
53340 return C && C->getConstantFPValue()->isAllOnesValue();
53341 };
53342
53343 // fand (fxor X, -1), Y --> fandn X, Y
53344 if (N0.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N0.getOperand(1)))
53345 return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1);
53346
53347 // fand X, (fxor Y, -1) --> fandn Y, X
53348 if (N1.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N1.getOperand(1)))
53349 return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0);
53350
53351 return SDValue();
53352}
53353
53354/// Do target-specific dag combines on X86ISD::FAND nodes.
53355static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG,
53356 const X86Subtarget &Subtarget) {
53357 // FAND(0.0, x) -> 0.0
53358 if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget))
53359 return V;
53360
53361 // FAND(x, 0.0) -> 0.0
53362 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
53363 return V;
53364
53365 if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget))
53366 return V;
53367
53368 return lowerX86FPLogicOp(N, DAG, Subtarget);
53369}
53370
53371/// Do target-specific dag combines on X86ISD::FANDN nodes.
53372static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG,
53373 const X86Subtarget &Subtarget) {
53374 // FANDN(0.0, x) -> x
53375 if (isNullFPScalarOrVectorConst(N->getOperand(0)))
53376 return N->getOperand(1);
53377
53378 // FANDN(x, 0.0) -> 0.0
53379 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
53380 return V;
53381
53382 return lowerX86FPLogicOp(N, DAG, Subtarget);
53383}
53384
53385/// Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
53386static SDValue combineFOr(SDNode *N, SelectionDAG &DAG,
53387 TargetLowering::DAGCombinerInfo &DCI,
53388 const X86Subtarget &Subtarget) {
53389 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
53390
53391 // F[X]OR(0.0, x) -> x
53392 if (isNullFPScalarOrVectorConst(N->getOperand(0)))
53393 return N->getOperand(1);
53394
53395 // F[X]OR(x, 0.0) -> x
53396 if (isNullFPScalarOrVectorConst(N->getOperand(1)))
53397 return N->getOperand(0);
53398
53399 if (SDValue NewVal = combineFneg(N, DAG, DCI, Subtarget))
53400 return NewVal;
53401
53402 return lowerX86FPLogicOp(N, DAG, Subtarget);
53403}
53404
53405/// Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
53406static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
53407 assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
53408
53409 // FMIN/FMAX are commutative if no NaNs and no negative zeros are allowed.
53410 if (!DAG.getTarget().Options.NoNaNsFPMath ||
53411 !DAG.getTarget().Options.NoSignedZerosFPMath)
53412 return SDValue();
53413
53414 // If we run in unsafe-math mode, then convert the FMAX and FMIN nodes
53415 // into FMINC and FMAXC, which are Commutative operations.
53416 unsigned NewOp = 0;
53417 switch (N->getOpcode()) {
53418 default: llvm_unreachable("unknown opcode");
53419 case X86ISD::FMIN: NewOp = X86ISD::FMINC; break;
53420 case X86ISD::FMAX: NewOp = X86ISD::FMAXC; break;
53421 }
53422
53423 return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
53424 N->getOperand(0), N->getOperand(1));
53425}
53426
53427static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
53428 const X86Subtarget &Subtarget) {
53429 EVT VT = N->getValueType(0);
53430 if (Subtarget.useSoftFloat() || isSoftF16(VT, Subtarget))
53431 return SDValue();
53432
53433 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53434
53435 if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||
53436 (Subtarget.hasSSE2() && VT == MVT::f64) ||
53437 (Subtarget.hasFP16() && VT == MVT::f16) ||
53438 (VT.isVector() && TLI.isTypeLegal(VT))))
53439 return SDValue();
53440
53441 SDValue Op0 = N->getOperand(0);
53442 SDValue Op1 = N->getOperand(1);
53443 SDLoc DL(N);
53444 auto MinMaxOp = N->getOpcode() == ISD::FMAXNUM ? X86ISD::FMAX : X86ISD::FMIN;
53445
53446 // If we don't have to respect NaN inputs, this is a direct translation to x86
53447 // min/max instructions.
53448 if (DAG.getTarget().Options.NoNaNsFPMath || N->getFlags().hasNoNaNs())
53449 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
53450
53451 // If one of the operands is known non-NaN use the native min/max instructions
53452 // with the non-NaN input as second operand.
53453 if (DAG.isKnownNeverNaN(Op1))
53454 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
53455 if (DAG.isKnownNeverNaN(Op0))
53456 return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags());
53457
53458 // If we have to respect NaN inputs, this takes at least 3 instructions.
53459 // Favor a library call when operating on a scalar and minimizing code size.
53460 if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize())
53461 return SDValue();
53462
53463 EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
53464 VT);
53465
53466 // There are 4 possibilities involving NaN inputs, and these are the required
53467 // outputs:
53468 // Op1
53469 // Num NaN
53470 // ----------------
53471 // Num | Max | Op0 |
53472 // Op0 ----------------
53473 // NaN | Op1 | NaN |
53474 // ----------------
53475 //
53476 // The SSE FP max/min instructions were not designed for this case, but rather
53477 // to implement:
53478 // Min = Op1 < Op0 ? Op1 : Op0
53479 // Max = Op1 > Op0 ? Op1 : Op0
53480 //
53481 // So they always return Op0 if either input is a NaN. However, we can still
53482 // use those instructions for fmaxnum by selecting away a NaN input.
53483
53484 // If either operand is NaN, the 2nd source operand (Op0) is passed through.
53485 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);
53486 SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO);
53487
53488 // If Op0 is a NaN, select Op1. Otherwise, select the max. If both operands
53489 // are NaN, the NaN value of Op1 is the result.
53490 return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax);
53491}
53492
53493static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
53494 TargetLowering::DAGCombinerInfo &DCI) {
53495 EVT VT = N->getValueType(0);
53496 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53497
53498 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
53499 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
53500 return SDValue(N, 0);
53501
53502 // Convert a full vector load into vzload when not all bits are needed.
53503 SDValue In = N->getOperand(0);
53504 MVT InVT = In.getSimpleValueType();
53505 if (VT.getVectorNumElements() < InVT.getVectorNumElements() &&
53506 ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) {
53507 assert(InVT.is128BitVector() && "Expected 128-bit input vector");
53508 LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(0));
53509 unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements();
53510 MVT MemVT = MVT::getIntegerVT(NumBits);
53511 MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
53512 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
53513 SDLoc dl(N);
53514 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
53515 DAG.getBitcast(InVT, VZLoad));
53516 DCI.CombineTo(N, Convert);
53517 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
53518 DCI.recursivelyDeleteUnusedNodes(LN);
53519 return SDValue(N, 0);
53520 }
53521 }
53522
53523 return SDValue();
53524}
53525
53526static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG,
53527 TargetLowering::DAGCombinerInfo &DCI) {
53528 bool IsStrict = N->isTargetStrictFPOpcode();
53529 EVT VT = N->getValueType(0);
53530
53531 // Convert a full vector load into vzload when not all bits are needed.
53532 SDValue In = N->getOperand(IsStrict ? 1 : 0);
53533 MVT InVT = In.getSimpleValueType();
53534 if (VT.getVectorNumElements() < InVT.getVectorNumElements() &&
53535 ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) {
53536 assert(InVT.is128BitVector() && "Expected 128-bit input vector");
53537 LoadSDNode *LN = cast<LoadSDNode>(In);
53538 unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements();
53539 MVT MemVT = MVT::getFloatingPointVT(NumBits);
53540 MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
53541 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MemVT, LoadVT, DAG)) {
53542 SDLoc dl(N);
53543 if (IsStrict) {
53544 SDValue Convert =
53545 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},
53546 {N->getOperand(0), DAG.getBitcast(InVT, VZLoad)});
53547 DCI.CombineTo(N, Convert, Convert.getValue(1));
53548 } else {
53549 SDValue Convert =
53550 DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad));
53551 DCI.CombineTo(N, Convert);
53552 }
53553 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
53554 DCI.recursivelyDeleteUnusedNodes(LN);
53555 return SDValue(N, 0);
53556 }
53557 }
53558
53559 return SDValue();
53560}
53561
53562/// Do target-specific dag combines on X86ISD::ANDNP nodes.
53563static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
53564 TargetLowering::DAGCombinerInfo &DCI,
53565 const X86Subtarget &Subtarget) {
53566 SDValue N0 = N->getOperand(0);
53567 SDValue N1 = N->getOperand(1);
53568 MVT VT = N->getSimpleValueType(0);
53569 int NumElts = VT.getVectorNumElements();
53570 unsigned EltSizeInBits = VT.getScalarSizeInBits();
53571 SDLoc DL(N);
53572
53573 // ANDNP(undef, x) -> 0
53574 // ANDNP(x, undef) -> 0
53575 if (N0.isUndef() || N1.isUndef())
53576 return DAG.getConstant(0, DL, VT);
53577
53578 // ANDNP(0, x) -> x
53579 if (ISD::isBuildVectorAllZeros(N0.getNode()))
53580 return N1;
53581
53582 // ANDNP(x, 0) -> 0
53583 if (ISD::isBuildVectorAllZeros(N1.getNode()))
53584 return DAG.getConstant(0, DL, VT);
53585
53586 // ANDNP(x, -1) -> NOT(x) -> XOR(x, -1)
53587 if (ISD::isBuildVectorAllOnes(N1.getNode()))
53588 return DAG.getNOT(DL, N0, VT);
53589
53590 // Turn ANDNP back to AND if input is inverted.
53591 if (SDValue Not = IsNOT(N0, DAG))
53592 return DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, Not), N1);
53593
53594 // Constant Folding
53595 APInt Undefs0, Undefs1;
53596 SmallVector<APInt> EltBits0, EltBits1;
53597 if (getTargetConstantBitsFromNode(N0, EltSizeInBits, Undefs0, EltBits0,
53598 /*AllowWholeUndefs*/ true,
53599 /*AllowPartialUndefs*/ true)) {
53600 if (getTargetConstantBitsFromNode(N1, EltSizeInBits, Undefs1, EltBits1,
53601 /*AllowWholeUndefs*/ true,
53602 /*AllowPartialUndefs*/ true)) {
53603 SmallVector<APInt> ResultBits;
53604 for (int I = 0; I != NumElts; ++I)
53605 ResultBits.push_back(~EltBits0[I] & EltBits1[I]);
53606 return getConstVector(ResultBits, VT, DAG, DL);
53607 }
53608
53609 // Constant fold NOT(N0) to allow us to use AND.
53610 // Ensure this is only performed if we can confirm that the bitcasted source
53611 // has oneuse to prevent an infinite loop with canonicalizeBitSelect.
53612 if (N0->hasOneUse()) {
53613 SDValue BC0 = peekThroughOneUseBitcasts(N0);
53614 if (BC0.getOpcode() != ISD::BITCAST) {
53615 for (APInt &Elt : EltBits0)
53616 Elt = ~Elt;
53617 SDValue Not = getConstVector(EltBits0, VT, DAG, DL);
53618 return DAG.getNode(ISD::AND, DL, VT, Not, N1);
53619 }
53620 }
53621 }
53622
53623 // Attempt to recursively combine a bitmask ANDNP with shuffles.
53624 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
53625 SDValue Op(N, 0);
53626 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
53627 return Res;
53628
53629 // If either operand is a constant mask, then only the elements that aren't
53630 // zero are actually demanded by the other operand.
53631 auto GetDemandedMasks = [&](SDValue Op, bool Invert = false) {
53632 APInt UndefElts;
53633 SmallVector<APInt> EltBits;
53634 APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
53635 APInt DemandedElts = APInt::getAllOnes(NumElts);
53636 if (getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts,
53637 EltBits)) {
53638 DemandedBits.clearAllBits();
53639 DemandedElts.clearAllBits();
53640 for (int I = 0; I != NumElts; ++I) {
53641 if (UndefElts[I]) {
53642 // We can't assume an undef src element gives an undef dst - the
53643 // other src might be zero.
53644 DemandedBits.setAllBits();
53645 DemandedElts.setBit(I);
53646 } else if ((Invert && !EltBits[I].isAllOnes()) ||
53647 (!Invert && !EltBits[I].isZero())) {
53648 DemandedBits |= Invert ? ~EltBits[I] : EltBits[I];
53649 DemandedElts.setBit(I);
53650 }
53651 }
53652 }
53653 return std::make_pair(DemandedBits, DemandedElts);
53654 };
53655 APInt Bits0, Elts0;
53656 APInt Bits1, Elts1;
53657 std::tie(Bits0, Elts0) = GetDemandedMasks(N1);
53658 std::tie(Bits1, Elts1) = GetDemandedMasks(N0, true);
53659
53660 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53661 if (TLI.SimplifyDemandedVectorElts(N0, Elts0, DCI) ||
53662 TLI.SimplifyDemandedVectorElts(N1, Elts1, DCI) ||
53663 TLI.SimplifyDemandedBits(N0, Bits0, Elts0, DCI) ||
53664 TLI.SimplifyDemandedBits(N1, Bits1, Elts1, DCI)) {
53665 if (N->getOpcode() != ISD::DELETED_NODE)
53666 DCI.AddToWorklist(N);
53667 return SDValue(N, 0);
53668 }
53669 }
53670
53671 // Folds for better commutativity:
53672 if (N1->hasOneUse()) {
53673 // ANDNP(x,NOT(y)) -> AND(NOT(x),NOT(y)) -> NOT(OR(X,Y)).
53674 if (SDValue Not = IsNOT(N1, DAG))
53675 return DAG.getNOT(
53676 DL, DAG.getNode(ISD::OR, DL, VT, N0, DAG.getBitcast(VT, Not)), VT);
53677
53678 // ANDNP(x,PSHUFB(y,z)) -> PSHUFB(y,OR(z,x))
53679 // Zero out elements by setting the PSHUFB mask value to 0xFF.
53680 if (DAG.ComputeNumSignBits(N0) == EltSizeInBits) {
53681 SDValue BC1 = peekThroughOneUseBitcasts(N1);
53682 if (BC1.getOpcode() == X86ISD::PSHUFB) {
53683 EVT ShufVT = BC1.getValueType();
53684 SDValue NewMask = DAG.getNode(ISD::OR, DL, ShufVT, BC1.getOperand(1),
53685 DAG.getBitcast(ShufVT, N0));
53686 SDValue NewShuf =
53687 DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, BC1.getOperand(0), NewMask);
53688 return DAG.getBitcast(VT, NewShuf);
53689 }
53690 }
53691 }
53692
53693 return SDValue();
53694}
53695
53696static SDValue combineBT(SDNode *N, SelectionDAG &DAG,
53697 TargetLowering::DAGCombinerInfo &DCI) {
53698 SDValue N1 = N->getOperand(1);
53699
53700 // BT ignores high bits in the bit index operand.
53701 unsigned BitWidth = N1.getValueSizeInBits();
53702 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
53703 if (DAG.getTargetLoweringInfo().SimplifyDemandedBits(N1, DemandedMask, DCI)) {
53704 if (N->getOpcode() != ISD::DELETED_NODE)
53705 DCI.AddToWorklist(N);
53706 return SDValue(N, 0);
53707 }
53708
53709 return SDValue();
53710}
53711
53712static SDValue combineCVTPH2PS(SDNode *N, SelectionDAG &DAG,
53713 TargetLowering::DAGCombinerInfo &DCI) {
53714 bool IsStrict = N->getOpcode() == X86ISD::STRICT_CVTPH2PS;
53715 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
53716
53717 if (N->getValueType(0) == MVT::v4f32 && Src.getValueType() == MVT::v8i16) {
53718 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
53719 APInt DemandedElts = APInt::getLowBitsSet(8, 4);
53720 if (TLI.SimplifyDemandedVectorElts(Src, DemandedElts, DCI)) {
53721 if (N->getOpcode() != ISD::DELETED_NODE)
53722 DCI.AddToWorklist(N);
53723 return SDValue(N, 0);
53724 }
53725
53726 // Convert a full vector load into vzload when not all bits are needed.
53727 if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse()) {
53728 LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(IsStrict ? 1 : 0));
53729 if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::i64, MVT::v2i64, DAG)) {
53730 SDLoc dl(N);
53731 if (IsStrict) {
53732 SDValue Convert = DAG.getNode(
53733 N->getOpcode(), dl, {MVT::v4f32, MVT::Other},
53734 {N->getOperand(0), DAG.getBitcast(MVT::v8i16, VZLoad)});
53735 DCI.CombineTo(N, Convert, Convert.getValue(1));
53736 } else {
53737 SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32,
53738 DAG.getBitcast(MVT::v8i16, VZLoad));
53739 DCI.CombineTo(N, Convert);
53740 }
53741
53742 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
53743 DCI.recursivelyDeleteUnusedNodes(LN);
53744 return SDValue(N, 0);
53745 }
53746 }
53747 }
53748
53749 return SDValue();
53750}
53751
53752// Try to combine sext_in_reg of a cmov of constants by extending the constants.
53753static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG) {
53754 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
53755
53756 EVT DstVT = N->getValueType(0);
53757
53758 SDValue N0 = N->getOperand(0);
53759 SDValue N1 = N->getOperand(1);
53760 EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
53761
53762 if (ExtraVT != MVT::i8 && ExtraVT != MVT::i16)
53763 return SDValue();
53764
53765 // Look through single use any_extends / truncs.
53766 SDValue IntermediateBitwidthOp;
53767 if ((N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::TRUNCATE) &&
53768 N0.hasOneUse()) {
53769 IntermediateBitwidthOp = N0;
53770 N0 = N0.getOperand(0);
53771 }
53772
53773 // See if we have a single use cmov.
53774 if (N0.getOpcode() != X86ISD::CMOV || !N0.hasOneUse())
53775 return SDValue();
53776
53777 SDValue CMovOp0 = N0.getOperand(0);
53778 SDValue CMovOp1 = N0.getOperand(1);
53779
53780 // Make sure both operands are constants.
53781 if (!isa<ConstantSDNode>(CMovOp0.getNode()) ||
53782 !isa<ConstantSDNode>(CMovOp1.getNode()))
53783 return SDValue();
53784
53785 SDLoc DL(N);
53786
53787 // If we looked through an any_extend/trunc above, add one to the constants.
53788 if (IntermediateBitwidthOp) {
53789 unsigned IntermediateOpc = IntermediateBitwidthOp.getOpcode();
53790 CMovOp0 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp0);
53791 CMovOp1 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp1);
53792 }
53793
53794 CMovOp0 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp0, N1);
53795 CMovOp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp1, N1);
53796
53797 EVT CMovVT = DstVT;
53798 // We do not want i16 CMOV's. Promote to i32 and truncate afterwards.
53799 if (DstVT == MVT::i16) {
53800 CMovVT = MVT::i32;
53801 CMovOp0 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp0);
53802 CMovOp1 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp1);
53803 }
53804
53805 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1,
53806 N0.getOperand(2), N0.getOperand(3));
53807
53808 if (CMovVT != DstVT)
53809 CMov = DAG.getNode(ISD::TRUNCATE, DL, DstVT, CMov);
53810
53811 return CMov;
53812}
53813
53814static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG,
53815 const X86Subtarget &Subtarget) {
53816 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
53817
53818 if (SDValue V = combineSextInRegCmov(N, DAG))
53819 return V;
53820
53821 EVT VT = N->getValueType(0);
53822 SDValue N0 = N->getOperand(0);
53823 SDValue N1 = N->getOperand(1);
53824 EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
53825 SDLoc dl(N);
53826
53827 // The SIGN_EXTEND_INREG to v4i64 is expensive operation on the
53828 // both SSE and AVX2 since there is no sign-extended shift right
53829 // operation on a vector with 64-bit elements.
53830 //(sext_in_reg (v4i64 anyext (v4i32 x )), ExtraVT) ->
53831 // (v4i64 sext (v4i32 sext_in_reg (v4i32 x , ExtraVT)))
53832 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||
53833 N0.getOpcode() == ISD::SIGN_EXTEND)) {
53834 SDValue N00 = N0.getOperand(0);
53835
53836 // EXTLOAD has a better solution on AVX2,
53837 // it may be replaced with X86ISD::VSEXT node.
53838 if (N00.getOpcode() == ISD::LOAD && Subtarget.hasInt256())
53839 if (!ISD::isNormalLoad(N00.getNode()))
53840 return SDValue();
53841
53842 // Attempt to promote any comparison mask ops before moving the
53843 // SIGN_EXTEND_INREG in the way.
53844 if (SDValue Promote = PromoteMaskArithmetic(N0, dl, DAG, Subtarget))
53845 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Promote, N1);
53846
53847 if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) {
53848 SDValue Tmp =
53849 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, N00, N1);
53850 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp);
53851 }
53852 }
53853 return SDValue();
53854}
53855
53856/// sext(add_nsw(x, C)) --> add(sext(x), C_sext)
53857/// zext(add_nuw(x, C)) --> add(zext(x), C_zext)
53858/// Promoting a sign/zero extension ahead of a no overflow 'add' exposes
53859/// opportunities to combine math ops, use an LEA, or use a complex addressing
53860/// mode. This can eliminate extend, add, and shift instructions.
53861static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
53862 const X86Subtarget &Subtarget) {
53863 if (Ext->getOpcode() != ISD::SIGN_EXTEND &&
53864 Ext->getOpcode() != ISD::ZERO_EXTEND)
53865 return SDValue();
53866
53867 // TODO: This should be valid for other integer types.
53868 EVT VT = Ext->getValueType(0);
53869 if (VT != MVT::i64)
53870 return SDValue();
53871
53872 SDValue Add = Ext->getOperand(0);
53873 if (Add.getOpcode() != ISD::ADD)
53874 return SDValue();
53875
53876 SDValue AddOp0 = Add.getOperand(0);
53877 SDValue AddOp1 = Add.getOperand(1);
53878 bool Sext = Ext->getOpcode() == ISD::SIGN_EXTEND;
53879 bool NSW = Add->getFlags().hasNoSignedWrap();
53880 bool NUW = Add->getFlags().hasNoUnsignedWrap();
53881 NSW = NSW || (Sext && DAG.willNotOverflowAdd(true, AddOp0, AddOp1));
53882 NUW = NUW || (!Sext && DAG.willNotOverflowAdd(false, AddOp0, AddOp1));
53883
53884 // We need an 'add nsw' feeding into the 'sext' or 'add nuw' feeding
53885 // into the 'zext'
53886 if ((Sext && !NSW) || (!Sext && !NUW))
53887 return SDValue();
53888
53889 // Having a constant operand to the 'add' ensures that we are not increasing
53890 // the instruction count because the constant is extended for free below.
53891 // A constant operand can also become the displacement field of an LEA.
53892 auto *AddOp1C = dyn_cast<ConstantSDNode>(AddOp1);
53893 if (!AddOp1C)
53894 return SDValue();
53895
53896 // Don't make the 'add' bigger if there's no hope of combining it with some
53897 // other 'add' or 'shl' instruction.
53898 // TODO: It may be profitable to generate simpler LEA instructions in place
53899 // of single 'add' instructions, but the cost model for selecting an LEA
53900 // currently has a high threshold.
53901 bool HasLEAPotential = false;
53902 for (auto *User : Ext->uses()) {
53903 if (User->getOpcode() == ISD::ADD || User->getOpcode() == ISD::SHL) {
53904 HasLEAPotential = true;
53905 break;
53906 }
53907 }
53908 if (!HasLEAPotential)
53909 return SDValue();
53910
53911 // Everything looks good, so pull the '{s|z}ext' ahead of the 'add'.
53912 int64_t AddC = Sext ? AddOp1C->getSExtValue() : AddOp1C->getZExtValue();
53913 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0);
53914 SDValue NewConstant = DAG.getConstant(AddC, SDLoc(Add), VT);
53915
53916 // The wider add is guaranteed to not wrap because both operands are
53917 // sign-extended.
53918 SDNodeFlags Flags;
53919 Flags.setNoSignedWrap(NSW);
53920 Flags.setNoUnsignedWrap(NUW);
53921 return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags);
53922}
53923
53924// If we face {ANY,SIGN,ZERO}_EXTEND that is applied to a CMOV with constant
53925// operands and the result of CMOV is not used anywhere else - promote CMOV
53926// itself instead of promoting its result. This could be beneficial, because:
53927// 1) X86TargetLowering::EmitLoweredSelect later can do merging of two
53928// (or more) pseudo-CMOVs only when they go one-after-another and
53929// getting rid of result extension code after CMOV will help that.
53930// 2) Promotion of constant CMOV arguments is free, hence the
53931// {ANY,SIGN,ZERO}_EXTEND will just be deleted.
53932// 3) 16-bit CMOV encoding is 4 bytes, 32-bit CMOV is 3-byte, so this
53933// promotion is also good in terms of code-size.
53934// (64-bit CMOV is 4-bytes, that's why we don't do 32-bit => 64-bit
53935// promotion).
53936static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) {
53937 SDValue CMovN = Extend->getOperand(0);
53938 if (CMovN.getOpcode() != X86ISD::CMOV || !CMovN.hasOneUse())
53939 return SDValue();
53940
53941 EVT TargetVT = Extend->getValueType(0);
53942 unsigned ExtendOpcode = Extend->getOpcode();
53943 SDLoc DL(Extend);
53944
53945 EVT VT = CMovN.getValueType();
53946 SDValue CMovOp0 = CMovN.getOperand(0);
53947 SDValue CMovOp1 = CMovN.getOperand(1);
53948
53949 if (!isa<ConstantSDNode>(CMovOp0.getNode()) ||
53950 !isa<ConstantSDNode>(CMovOp1.getNode()))
53951 return SDValue();
53952
53953 // Only extend to i32 or i64.
53954 if (TargetVT != MVT::i32 && TargetVT != MVT::i64)
53955 return SDValue();
53956
53957 // Only extend from i16 unless its a sign_extend from i32. Zext/aext from i32
53958 // are free.
53959 if (VT != MVT::i16 && !(ExtendOpcode == ISD::SIGN_EXTEND && VT == MVT::i32))
53960 return SDValue();
53961
53962 // If this a zero extend to i64, we should only extend to i32 and use a free
53963 // zero extend to finish.
53964 EVT ExtendVT = TargetVT;
53965 if (TargetVT == MVT::i64 && ExtendOpcode != ISD::SIGN_EXTEND)
53966 ExtendVT = MVT::i32;
53967
53968 CMovOp0 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp0);
53969 CMovOp1 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp1);
53970
53971 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
53972 CMovN.getOperand(2), CMovN.getOperand(3));
53973
53974 // Finish extending if needed.
53975 if (ExtendVT != TargetVT)
53976 Res = DAG.getNode(ExtendOpcode, DL, TargetVT, Res);
53977
53978 return Res;
53979}
53980
53981// Attempt to combine a (sext/zext (setcc)) to a setcc with a xmm/ymm/zmm
53982// result type.
53983static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
53984 const X86Subtarget &Subtarget) {
53985 SDValue N0 = N->getOperand(0);
53986 EVT VT = N->getValueType(0);
53987 SDLoc dl(N);
53988
53989 // Only do this combine with AVX512 for vector extends.
53990 if (!Subtarget.hasAVX512() || !VT.isVector() || N0.getOpcode() != ISD::SETCC)
53991 return SDValue();
53992
53993 // Only combine legal element types.
53994 EVT SVT = VT.getVectorElementType();
53995 if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 &&
53996 SVT != MVT::i64 && SVT != MVT::f32 && SVT != MVT::f64)
53997 return SDValue();
53998
53999 // We don't have CMPP Instruction for vxf16
54000 if (N0.getOperand(0).getValueType().getVectorElementType() == MVT::f16)
54001 return SDValue();
54002 // We can only do this if the vector size in 256 bits or less.
54003 unsigned Size = VT.getSizeInBits();
54004 if (Size > 256 && Subtarget.useAVX512Regs())
54005 return SDValue();
54006
54007 // Don't fold if the condition code can't be handled by PCMPEQ/PCMPGT since
54008 // that's the only integer compares with we have.
54009 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
54010 if (ISD::isUnsignedIntSetCC(CC))
54011 return SDValue();
54012
54013 // Only do this combine if the extension will be fully consumed by the setcc.
54014 EVT N00VT = N0.getOperand(0).getValueType();
54015 EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
54016 if (Size != MatchingVecType.getSizeInBits())
54017 return SDValue();
54018
54019 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
54020
54021 if (N->getOpcode() == ISD::ZERO_EXTEND)
54022 Res = DAG.getZeroExtendInReg(Res, dl, N0.getValueType());
54023
54024 return Res;
54025}
54026
54027static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
54028 TargetLowering::DAGCombinerInfo &DCI,
54029 const X86Subtarget &Subtarget) {
54030 SDValue N0 = N->getOperand(0);
54031 EVT VT = N->getValueType(0);
54032 SDLoc DL(N);
54033
54034 // (i32 (sext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry))
54035 if (!DCI.isBeforeLegalizeOps() &&
54036 N0.getOpcode() == X86ISD::SETCC_CARRY) {
54037 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0),
54038 N0->getOperand(1));
54039 bool ReplaceOtherUses = !N0.hasOneUse();
54040 DCI.CombineTo(N, Setcc);
54041 // Replace other uses with a truncate of the widened setcc_carry.
54042 if (ReplaceOtherUses) {
54043 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
54044 N0.getValueType(), Setcc);
54045 DCI.CombineTo(N0.getNode(), Trunc);
54046 }
54047
54048 return SDValue(N, 0);
54049 }
54050
54051 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
54052 return NewCMov;
54053
54054 if (!DCI.isBeforeLegalizeOps())
54055 return SDValue();
54056
54057 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
54058 return V;
54059
54060 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), DL, VT, N0,
54061 DAG, DCI, Subtarget))
54062 return V;
54063
54064 if (VT.isVector()) {
54065 if (SDValue R = PromoteMaskArithmetic(SDValue(N, 0), DL, DAG, Subtarget))
54066 return R;
54067
54068 if (N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG)
54069 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0));
54070 }
54071
54072 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
54073 return NewAdd;
54074
54075 return SDValue();
54076}
54077
54078// Inverting a constant vector is profitable if it can be eliminated and the
54079// inverted vector is already present in DAG. Otherwise, it will be loaded
54080// anyway.
54081//
54082// We determine which of the values can be completely eliminated and invert it.
54083// If both are eliminable, select a vector with the first negative element.
54084static SDValue getInvertedVectorForFMA(SDValue V, SelectionDAG &DAG) {
54085 assert(ISD::isBuildVectorOfConstantFPSDNodes(V.getNode()) &&
54086 "ConstantFP build vector expected");
54087 // Check if we can eliminate V. We assume if a value is only used in FMAs, we
54088 // can eliminate it. Since this function is invoked for each FMA with this
54089 // vector.
54090 auto IsNotFMA = [](SDNode *Use) {
54091 return Use->getOpcode() != ISD::FMA && Use->getOpcode() != ISD::STRICT_FMA;
54092 };
54093 if (llvm::any_of(V->uses(), IsNotFMA))
54094 return SDValue();
54095
54096 SmallVector<SDValue, 8> Ops;
54097 EVT VT = V.getValueType();
54098 EVT EltVT = VT.getVectorElementType();
54099 for (const SDValue &Op : V->op_values()) {
54100 if (auto *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
54101 Ops.push_back(DAG.getConstantFP(-Cst->getValueAPF(), SDLoc(Op), EltVT));
54102 } else {
54103 assert(Op.isUndef());
54104 Ops.push_back(DAG.getUNDEF(EltVT));
54105 }
54106 }
54107
54108 SDNode *NV = DAG.getNodeIfExists(ISD::BUILD_VECTOR, DAG.getVTList(VT), Ops);
54109 if (!NV)
54110 return SDValue();
54111
54112 // If an inverted version cannot be eliminated, choose it instead of the
54113 // original version.
54114 if (llvm::any_of(NV->uses(), IsNotFMA))
54115 return SDValue(NV, 0);
54116
54117 // If the inverted version also can be eliminated, we have to consistently
54118 // prefer one of the values. We prefer a constant with a negative value on
54119 // the first place.
54120 // N.B. We need to skip undefs that may precede a value.
54121 for (const SDValue &Op : V->op_values()) {
54122 if (auto *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
54123 if (Cst->isNegative())
54124 return SDValue();
54125 break;
54126 }
54127 }
54128 return SDValue(NV, 0);
54129}
54130
54131static SDValue combineFMA(SDNode *N, SelectionDAG &DAG,
54132 TargetLowering::DAGCombinerInfo &DCI,
54133 const X86Subtarget &Subtarget) {
54134 SDLoc dl(N);
54135 EVT VT = N->getValueType(0);
54136 bool IsStrict = N->isStrictFPOpcode() || N->isTargetStrictFPOpcode();
54137
54138 // Let legalize expand this if it isn't a legal type yet.
54139 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54140 if (!TLI.isTypeLegal(VT))
54141 return SDValue();
54142
54143 SDValue A = N->getOperand(IsStrict ? 1 : 0);
54144 SDValue B = N->getOperand(IsStrict ? 2 : 1);
54145 SDValue C = N->getOperand(IsStrict ? 3 : 2);
54146
54147 // If the operation allows fast-math and the target does not support FMA,
54148 // split this into mul+add to avoid libcall(s).
54149 SDNodeFlags Flags = N->getFlags();
54150 if (!IsStrict && Flags.hasAllowReassociation() &&
54151 TLI.isOperationExpand(ISD::FMA, VT)) {
54152 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags);
54153 return DAG.getNode(ISD::FADD, dl, VT, Fmul, C, Flags);
54154 }
54155
54156 EVT ScalarVT = VT.getScalarType();
54157 if (((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) ||
54158 !Subtarget.hasAnyFMA()) &&
54159 !(ScalarVT == MVT::f16 && Subtarget.hasFP16()))
54160 return SDValue();
54161
54162 auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) {
54163 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
54164 bool LegalOperations = !DCI.isBeforeLegalizeOps();
54165 if (SDValue NegV = TLI.getCheaperNegatedExpression(V, DAG, LegalOperations,
54166 CodeSize)) {
54167 V = NegV;
54168 return true;
54169 }
54170 // Look through extract_vector_elts. If it comes from an FNEG, create a
54171 // new extract from the FNEG input.
54172 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
54173 isNullConstant(V.getOperand(1))) {
54174 SDValue Vec = V.getOperand(0);
54175 if (SDValue NegV = TLI.getCheaperNegatedExpression(
54176 Vec, DAG, LegalOperations, CodeSize)) {
54177 V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), V.getValueType(),
54178 NegV, V.getOperand(1));
54179 return true;
54180 }
54181 }
54182 // Lookup if there is an inverted version of constant vector V in DAG.
54183 if (ISD::isBuildVectorOfConstantFPSDNodes(V.getNode())) {
54184 if (SDValue NegV = getInvertedVectorForFMA(V, DAG)) {
54185 V = NegV;
54186 return true;
54187 }
54188 }
54189 return false;
54190 };
54191
54192 // Do not convert the passthru input of scalar intrinsics.
54193 // FIXME: We could allow negations of the lower element only.
54194 bool NegA = invertIfNegative(A);
54195 bool NegB = invertIfNegative(B);
54196 bool NegC = invertIfNegative(C);
54197
54198 if (!NegA && !NegB && !NegC)
54199 return SDValue();
54200
54201 unsigned NewOpcode =
54202 negateFMAOpcode(N->getOpcode(), NegA != NegB, NegC, false);
54203
54204 // Propagate fast-math-flags to new FMA node.
54205 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
54206 if (IsStrict) {
54207 assert(N->getNumOperands() == 4 && "Shouldn't be greater than 4");
54208 return DAG.getNode(NewOpcode, dl, {VT, MVT::Other},
54209 {N->getOperand(0), A, B, C});
54210 } else {
54211 if (N->getNumOperands() == 4)
54212 return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3));
54213 return DAG.getNode(NewOpcode, dl, VT, A, B, C);
54214 }
54215}
54216
54217// Combine FMADDSUB(A, B, FNEG(C)) -> FMSUBADD(A, B, C)
54218// Combine FMSUBADD(A, B, FNEG(C)) -> FMADDSUB(A, B, C)
54219static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG,
54220 TargetLowering::DAGCombinerInfo &DCI) {
54221 SDLoc dl(N);
54222 EVT VT = N->getValueType(0);
54223 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54224 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
54225 bool LegalOperations = !DCI.isBeforeLegalizeOps();
54226
54227 SDValue N2 = N->getOperand(2);
54228
54229 SDValue NegN2 =
54230 TLI.getCheaperNegatedExpression(N2, DAG, LegalOperations, CodeSize);
54231 if (!NegN2)
54232 return SDValue();
54233 unsigned NewOpcode = negateFMAOpcode(N->getOpcode(), false, true, false);
54234
54235 if (N->getNumOperands() == 4)
54236 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
54237 NegN2, N->getOperand(3));
54238 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
54239 NegN2);
54240}
54241
54242static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
54243 TargetLowering::DAGCombinerInfo &DCI,
54244 const X86Subtarget &Subtarget) {
54245 SDLoc dl(N);
54246 SDValue N0 = N->getOperand(0);
54247 EVT VT = N->getValueType(0);
54248
54249 // (i32 (aext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry))
54250 // FIXME: Is this needed? We don't seem to have any tests for it.
54251 if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ANY_EXTEND &&
54252 N0.getOpcode() == X86ISD::SETCC_CARRY) {
54253 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0),
54254 N0->getOperand(1));
54255 bool ReplaceOtherUses = !N0.hasOneUse();
54256 DCI.CombineTo(N, Setcc);
54257 // Replace other uses with a truncate of the widened setcc_carry.
54258 if (ReplaceOtherUses) {
54259 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
54260 N0.getValueType(), Setcc);
54261 DCI.CombineTo(N0.getNode(), Trunc);
54262 }
54263
54264 return SDValue(N, 0);
54265 }
54266
54267 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
54268 return NewCMov;
54269
54270 if (DCI.isBeforeLegalizeOps())
54271 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
54272 return V;
54273
54274 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), dl, VT, N0,
54275 DAG, DCI, Subtarget))
54276 return V;
54277
54278 if (VT.isVector())
54279 if (SDValue R = PromoteMaskArithmetic(SDValue(N, 0), dl, DAG, Subtarget))
54280 return R;
54281
54282 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
54283 return NewAdd;
54284
54285 if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget))
54286 return R;
54287
54288 // TODO: Combine with any target/faux shuffle.
54289 if (N0.getOpcode() == X86ISD::PACKUS && N0.getValueSizeInBits() == 128 &&
54290 VT.getScalarSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits()) {
54291 SDValue N00 = N0.getOperand(0);
54292 SDValue N01 = N0.getOperand(1);
54293 unsigned NumSrcEltBits = N00.getScalarValueSizeInBits();
54294 APInt ZeroMask = APInt::getHighBitsSet(NumSrcEltBits, NumSrcEltBits / 2);
54295 if ((N00.isUndef() || DAG.MaskedValueIsZero(N00, ZeroMask)) &&
54296 (N01.isUndef() || DAG.MaskedValueIsZero(N01, ZeroMask))) {
54297 return concatSubVectors(N00, N01, DAG, dl);
54298 }
54299 }
54300
54301 return SDValue();
54302}
54303
54304/// If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just
54305/// pre-promote its result type since vXi1 vectors don't get promoted
54306/// during type legalization.
54307static SDValue truncateAVX512SetCCNoBWI(EVT VT, EVT OpVT, SDValue LHS,
54308 SDValue RHS, ISD::CondCode CC,
54309 const SDLoc &DL, SelectionDAG &DAG,
54310 const X86Subtarget &Subtarget) {
54311 if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.isVector() &&
54312 VT.getVectorElementType() == MVT::i1 &&
54313 (OpVT.getVectorElementType() == MVT::i8 ||
54314 OpVT.getVectorElementType() == MVT::i16)) {
54315 SDValue Setcc = DAG.getSetCC(DL, OpVT, LHS, RHS, CC);
54316 return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc);
54317 }
54318 return SDValue();
54319}
54320
54321static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
54322 TargetLowering::DAGCombinerInfo &DCI,
54323 const X86Subtarget &Subtarget) {
54324 const ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
54325 const SDValue LHS = N->getOperand(0);
54326 const SDValue RHS = N->getOperand(1);
54327 EVT VT = N->getValueType(0);
54328 EVT OpVT = LHS.getValueType();
54329 SDLoc DL(N);
54330
54331 if (CC == ISD::SETNE || CC == ISD::SETEQ) {
54332 if (SDValue V = combineVectorSizedSetCCEquality(VT, LHS, RHS, CC, DL, DAG,
54333 Subtarget))
54334 return V;
54335
54336 if (VT == MVT::i1) {
54337 X86::CondCode X86CC;
54338 if (SDValue V =
54339 MatchVectorAllEqualTest(LHS, RHS, CC, DL, Subtarget, DAG, X86CC))
54340 return DAG.getNode(ISD::TRUNCATE, DL, VT, getSETCC(X86CC, V, DL, DAG));
54341 }
54342
54343 if (OpVT.isScalarInteger()) {
54344 // cmpeq(or(X,Y),X) --> cmpeq(and(~X,Y),0)
54345 // cmpne(or(X,Y),X) --> cmpne(and(~X,Y),0)
54346 auto MatchOrCmpEq = [&](SDValue N0, SDValue N1) {
54347 if (N0.getOpcode() == ISD::OR && N0->hasOneUse()) {
54348 if (N0.getOperand(0) == N1)
54349 return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT),
54350 N0.getOperand(1));
54351 if (N0.getOperand(1) == N1)
54352 return DAG.getNode(ISD::AND, DL, OpVT, DAG.getNOT(DL, N1, OpVT),
54353 N0.getOperand(0));
54354 }
54355 return SDValue();
54356 };
54357 if (SDValue AndN = MatchOrCmpEq(LHS, RHS))
54358 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
54359 if (SDValue AndN = MatchOrCmpEq(RHS, LHS))
54360 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
54361
54362 // cmpeq(and(X,Y),Y) --> cmpeq(and(~X,Y),0)
54363 // cmpne(and(X,Y),Y) --> cmpne(and(~X,Y),0)
54364 auto MatchAndCmpEq = [&](SDValue N0, SDValue N1) {
54365 if (N0.getOpcode() == ISD::AND && N0->hasOneUse()) {
54366 if (N0.getOperand(0) == N1)
54367 return DAG.getNode(ISD::AND, DL, OpVT, N1,
54368 DAG.getNOT(DL, N0.getOperand(1), OpVT));
54369 if (N0.getOperand(1) == N1)
54370 return DAG.getNode(ISD::AND, DL, OpVT, N1,
54371 DAG.getNOT(DL, N0.getOperand(0), OpVT));
54372 }
54373 return SDValue();
54374 };
54375 if (SDValue AndN = MatchAndCmpEq(LHS, RHS))
54376 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
54377 if (SDValue AndN = MatchAndCmpEq(RHS, LHS))
54378 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC);
54379
54380 // cmpeq(trunc(x),C) --> cmpeq(x,C)
54381 // cmpne(trunc(x),C) --> cmpne(x,C)
54382 // iff x upper bits are zero.
54383 if (LHS.getOpcode() == ISD::TRUNCATE &&
54384 LHS.getOperand(0).getScalarValueSizeInBits() >= 32 &&
54385 isa<ConstantSDNode>(RHS) && !DCI.isBeforeLegalize()) {
54386 EVT SrcVT = LHS.getOperand(0).getValueType();
54387 APInt UpperBits = APInt::getBitsSetFrom(SrcVT.getScalarSizeInBits(),
54388 OpVT.getScalarSizeInBits());
54389 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54390 if (DAG.MaskedValueIsZero(LHS.getOperand(0), UpperBits) &&
54391 TLI.isTypeLegal(LHS.getOperand(0).getValueType()))
54392 return DAG.getSetCC(DL, VT, LHS.getOperand(0),
54393 DAG.getZExtOrTrunc(RHS, DL, SrcVT), CC);
54394 }
54395
54396 // With C as a power of 2 and C != 0 and C != INT_MIN:
54397 // icmp eq Abs(X) C ->
54398 // (icmp eq A, C) | (icmp eq A, -C)
54399 // icmp ne Abs(X) C ->
54400 // (icmp ne A, C) & (icmp ne A, -C)
54401 // Both of these patterns can be better optimized in
54402 // DAGCombiner::foldAndOrOfSETCC. Note this only applies for scalar
54403 // integers which is checked above.
54404 if (LHS.getOpcode() == ISD::ABS && LHS.hasOneUse()) {
54405 if (auto *C = dyn_cast<ConstantSDNode>(RHS)) {
54406 const APInt &CInt = C->getAPIntValue();
54407 // We can better optimize this case in DAGCombiner::foldAndOrOfSETCC.
54408 if (CInt.isPowerOf2() && !CInt.isMinSignedValue()) {
54409 SDValue BaseOp = LHS.getOperand(0);
54410 SDValue SETCC0 = DAG.getSetCC(DL, VT, BaseOp, RHS, CC);
54411 SDValue SETCC1 = DAG.getSetCC(
54412 DL, VT, BaseOp, DAG.getConstant(-CInt, DL, OpVT), CC);
54413 return DAG.getNode(CC == ISD::SETEQ ? ISD::OR : ISD::AND, DL, VT,
54414 SETCC0, SETCC1);
54415 }
54416 }
54417 }
54418 }
54419 }
54420
54421 if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
54422 (CC == ISD::SETNE || CC == ISD::SETEQ || ISD::isSignedIntSetCC(CC))) {
54423 // Using temporaries to avoid messing up operand ordering for later
54424 // transformations if this doesn't work.
54425 SDValue Op0 = LHS;
54426 SDValue Op1 = RHS;
54427 ISD::CondCode TmpCC = CC;
54428 // Put build_vector on the right.
54429 if (Op0.getOpcode() == ISD::BUILD_VECTOR) {
54430 std::swap(Op0, Op1);
54431 TmpCC = ISD::getSetCCSwappedOperands(TmpCC);
54432 }
54433
54434 bool IsSEXT0 =
54435 (Op0.getOpcode() == ISD::SIGN_EXTEND) &&
54436 (Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1);
54437 bool IsVZero1 = ISD::isBuildVectorAllZeros(Op1.getNode());
54438
54439 if (IsSEXT0 && IsVZero1) {
54440 assert(VT == Op0.getOperand(0).getValueType() &&
54441 "Unexpected operand type");
54442 if (TmpCC == ISD::SETGT)
54443 return DAG.getConstant(0, DL, VT);
54444 if (TmpCC == ISD::SETLE)
54445 return DAG.getConstant(1, DL, VT);
54446 if (TmpCC == ISD::SETEQ || TmpCC == ISD::SETGE)
54447 return DAG.getNOT(DL, Op0.getOperand(0), VT);
54448
54449 assert((TmpCC == ISD::SETNE || TmpCC == ISD::SETLT) &&
54450 "Unexpected condition code!");
54451 return Op0.getOperand(0);
54452 }
54453 }
54454
54455 // Try and make unsigned vector comparison signed. On pre AVX512 targets there
54456 // only are unsigned comparisons (`PCMPGT`) and on AVX512 its often better to
54457 // use `PCMPGT` if the result is mean to stay in a vector (and if its going to
54458 // a mask, there are signed AVX512 comparisons).
54459 if (VT.isVector() && OpVT.isVector() && OpVT.isInteger()) {
54460 bool CanMakeSigned = false;
54461 if (ISD::isUnsignedIntSetCC(CC)) {
54462 KnownBits CmpKnown =
54463 DAG.computeKnownBits(LHS).intersectWith(DAG.computeKnownBits(RHS));
54464 // If we know LHS/RHS share the same sign bit at each element we can
54465 // make this signed.
54466 // NOTE: `computeKnownBits` on a vector type aggregates common bits
54467 // across all lanes. So a pattern where the sign varies from lane to
54468 // lane, but at each lane Sign(LHS) is known to equal Sign(RHS), will be
54469 // missed. We could get around this by demanding each lane
54470 // independently, but this isn't the most important optimization and
54471 // that may eat into compile time.
54472 CanMakeSigned =
54473 CmpKnown.Zero.isSignBitSet() || CmpKnown.One.isSignBitSet();
54474 }
54475 if (CanMakeSigned || ISD::isSignedIntSetCC(CC)) {
54476 SDValue LHSOut = LHS;
54477 SDValue RHSOut = RHS;
54478 ISD::CondCode NewCC = CC;
54479 switch (CC) {
54480 case ISD::SETGE:
54481 case ISD::SETUGE:
54482 if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ true,
54483 /*NSW*/ true))
54484 LHSOut = NewLHS;
54485 else if (SDValue NewRHS = incDecVectorConstant(
54486 RHS, DAG, /*IsInc*/ false, /*NSW*/ true))
54487 RHSOut = NewRHS;
54488 else
54489 break;
54490
54491 [[fallthrough]];
54492 case ISD::SETUGT:
54493 NewCC = ISD::SETGT;
54494 break;
54495
54496 case ISD::SETLE:
54497 case ISD::SETULE:
54498 if (SDValue NewLHS = incDecVectorConstant(LHS, DAG, /*IsInc*/ false,
54499 /*NSW*/ true))
54500 LHSOut = NewLHS;
54501 else if (SDValue NewRHS = incDecVectorConstant(RHS, DAG, /*IsInc*/ true,
54502 /*NSW*/ true))
54503 RHSOut = NewRHS;
54504 else
54505 break;
54506
54507 [[fallthrough]];
54508 case ISD::SETULT:
54509 // Will be swapped to SETGT in LowerVSETCC*.
54510 NewCC = ISD::SETLT;
54511 break;
54512 default:
54513 break;
54514 }
54515 if (NewCC != CC) {
54516 if (SDValue R = truncateAVX512SetCCNoBWI(VT, OpVT, LHSOut, RHSOut,
54517 NewCC, DL, DAG, Subtarget))
54518 return R;
54519 return DAG.getSetCC(DL, VT, LHSOut, RHSOut, NewCC);
54520 }
54521 }
54522 }
54523
54524 if (SDValue R =
54525 truncateAVX512SetCCNoBWI(VT, OpVT, LHS, RHS, CC, DL, DAG, Subtarget))
54526 return R;
54527
54528 // In the middle end transforms:
54529 // `(or (icmp eq X, C), (icmp eq X, C+1))`
54530 // -> `(icmp ult (add x, -C), 2)`
54531 // Likewise inverted cases with `ugt`.
54532 //
54533 // Since x86, pre avx512, doesn't have unsigned vector compares, this results
54534 // in worse codegen. So, undo the middle-end transform and go back to `(or
54535 // (icmp eq), (icmp eq))` form.
54536 // Also skip AVX1 with ymm vectors, as the umin approach combines better than
54537 // the xmm approach.
54538 //
54539 // NB: We don't handle the similiar simplication of `(and (icmp ne), (icmp
54540 // ne))` as it doesn't end up instruction positive.
54541 // TODO: We might want to do this for avx512 as well if we `sext` the result.
54542 if (VT.isVector() && OpVT.isVector() && OpVT.isInteger() &&
54543 ISD::isUnsignedIntSetCC(CC) && LHS.getOpcode() == ISD::ADD &&
54544 !Subtarget.hasAVX512() &&
54545 (OpVT.getSizeInBits() <= 128 || !Subtarget.hasAVX() ||
54546 Subtarget.hasAVX2()) &&
54547 LHS.hasOneUse()) {
54548
54549 APInt CmpC;
54550 SDValue AddC = LHS.getOperand(1);
54551 if (ISD::isConstantSplatVector(RHS.getNode(), CmpC) &&
54552 DAG.isConstantIntBuildVectorOrConstantInt(AddC)) {
54553 // See which form we have depending on the constant/condition.
54554 SDValue C0 = SDValue();
54555 SDValue C1 = SDValue();
54556
54557 // If we had `(add x, -1)` and can lower with `umin`, don't transform as
54558 // we will end up generating an additional constant. Keeping in the
54559 // current form has a slight latency cost, but it probably worth saving a
54560 // constant.
54561 if (ISD::isConstantSplatVectorAllOnes(AddC.getNode()) &&
54562 DAG.getTargetLoweringInfo().isOperationLegal(ISD::UMIN, OpVT)) {
54563 // Pass
54564 }
54565 // Normal Cases
54566 else if ((CC == ISD::SETULT && CmpC == 2) ||
54567 (CC == ISD::SETULE && CmpC == 1)) {
54568 // These will constant fold.
54569 C0 = DAG.getNegative(AddC, DL, OpVT);
54570 C1 = DAG.getNode(ISD::SUB, DL, OpVT, C0,
54571 DAG.getAllOnesConstant(DL, OpVT));
54572 }
54573 // Inverted Cases
54574 else if ((CC == ISD::SETUGT && (-CmpC) == 3) ||
54575 (CC == ISD::SETUGE && (-CmpC) == 2)) {
54576 // These will constant fold.
54577 C0 = DAG.getNOT(DL, AddC, OpVT);
54578 C1 = DAG.getNode(ISD::ADD, DL, OpVT, C0,
54579 DAG.getAllOnesConstant(DL, OpVT));
54580 }
54581 if (C0 && C1) {
54582 SDValue NewLHS =
54583 DAG.getSetCC(DL, VT, LHS.getOperand(0), C0, ISD::SETEQ);
54584 SDValue NewRHS =
54585 DAG.getSetCC(DL, VT, LHS.getOperand(0), C1, ISD::SETEQ);
54586 return DAG.getNode(ISD::OR, DL, VT, NewLHS, NewRHS);
54587 }
54588 }
54589 }
54590
54591 // For an SSE1-only target, lower a comparison of v4f32 to X86ISD::CMPP early
54592 // to avoid scalarization via legalization because v4i32 is not a legal type.
54593 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 &&
54594 LHS.getValueType() == MVT::v4f32)
54595 return LowerVSETCC(SDValue(N, 0), Subtarget, DAG);
54596
54597 // X pred 0.0 --> X pred -X
54598 // If the negation of X already exists, use it in the comparison. This removes
54599 // the need to materialize 0.0 and allows matching to SSE's MIN/MAX
54600 // instructions in patterns with a 'select' node.
54601 if (isNullFPScalarOrVectorConst(RHS)) {
54602 SDVTList FNegVT = DAG.getVTList(OpVT);
54603 if (SDNode *FNeg = DAG.getNodeIfExists(ISD::FNEG, FNegVT, {LHS}))
54604 return DAG.getSetCC(DL, VT, LHS, SDValue(FNeg, 0), CC);
54605 }
54606
54607 return SDValue();
54608}
54609
54610static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
54611 TargetLowering::DAGCombinerInfo &DCI,
54612 const X86Subtarget &Subtarget) {
54613 SDValue Src = N->getOperand(0);
54614 MVT SrcVT = Src.getSimpleValueType();
54615 MVT VT = N->getSimpleValueType(0);
54616 unsigned NumBits = VT.getScalarSizeInBits();
54617 unsigned NumElts = SrcVT.getVectorNumElements();
54618 unsigned NumBitsPerElt = SrcVT.getScalarSizeInBits();
54619 assert(VT == MVT::i32 && NumElts <= NumBits && "Unexpected MOVMSK types");
54620
54621 // Perform constant folding.
54622 APInt UndefElts;
54623 SmallVector<APInt, 32> EltBits;
54624 if (getTargetConstantBitsFromNode(Src, NumBitsPerElt, UndefElts, EltBits,
54625 /*AllowWholeUndefs*/ true,
54626 /*AllowPartialUndefs*/ true)) {
54627 APInt Imm(32, 0);
54628 for (unsigned Idx = 0; Idx != NumElts; ++Idx)
54629 if (!UndefElts[Idx] && EltBits[Idx].isNegative())
54630 Imm.setBit(Idx);
54631
54632 return DAG.getConstant(Imm, SDLoc(N), VT);
54633 }
54634
54635 // Look through int->fp bitcasts that don't change the element width.
54636 unsigned EltWidth = SrcVT.getScalarSizeInBits();
54637 if (Subtarget.hasSSE2() && Src.getOpcode() == ISD::BITCAST &&
54638 Src.getOperand(0).getScalarValueSizeInBits() == EltWidth)
54639 return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0));
54640
54641 // Fold movmsk(not(x)) -> not(movmsk(x)) to improve folding of movmsk results
54642 // with scalar comparisons.
54643 if (SDValue NotSrc = IsNOT(Src, DAG)) {
54644 SDLoc DL(N);
54645 APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts);
54646 NotSrc = DAG.getBitcast(SrcVT, NotSrc);
54647 return DAG.getNode(ISD::XOR, DL, VT,
54648 DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc),
54649 DAG.getConstant(NotMask, DL, VT));
54650 }
54651
54652 // Fold movmsk(icmp_sgt(x,-1)) -> not(movmsk(x)) to improve folding of movmsk
54653 // results with scalar comparisons.
54654 if (Src.getOpcode() == X86ISD::PCMPGT &&
54655 ISD::isBuildVectorAllOnes(Src.getOperand(1).getNode())) {
54656 SDLoc DL(N);
54657 APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts);
54658 return DAG.getNode(ISD::XOR, DL, VT,
54659 DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)),
54660 DAG.getConstant(NotMask, DL, VT));
54661 }
54662
54663 // Fold movmsk(icmp_eq(and(x,c1),c1)) -> movmsk(shl(x,c2))
54664 // Fold movmsk(icmp_eq(and(x,c1),0)) -> movmsk(not(shl(x,c2)))
54665 // iff pow2splat(c1).
54666 // Use KnownBits to determine if only a single bit is non-zero
54667 // in each element (pow2 or zero), and shift that bit to the msb.
54668 if (Src.getOpcode() == X86ISD::PCMPEQ) {
54669 KnownBits KnownLHS = DAG.computeKnownBits(Src.getOperand(0));
54670 KnownBits KnownRHS = DAG.computeKnownBits(Src.getOperand(1));
54671 unsigned ShiftAmt = KnownLHS.countMinLeadingZeros();
54672 if (KnownLHS.countMaxPopulation() == 1 &&
54673 (KnownRHS.isZero() || (KnownRHS.countMaxPopulation() == 1 &&
54674 ShiftAmt == KnownRHS.countMinLeadingZeros()))) {
54675 SDLoc DL(N);
54676 MVT ShiftVT = SrcVT;
54677 SDValue ShiftLHS = Src.getOperand(0);
54678 SDValue ShiftRHS = Src.getOperand(1);
54679 if (ShiftVT.getScalarType() == MVT::i8) {
54680 // vXi8 shifts - we only care about the signbit so can use PSLLW.
54681 ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
54682 ShiftLHS = DAG.getBitcast(ShiftVT, ShiftLHS);
54683 ShiftRHS = DAG.getBitcast(ShiftVT, ShiftRHS);
54684 }
54685 ShiftLHS = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, ShiftVT,
54686 ShiftLHS, ShiftAmt, DAG);
54687 ShiftRHS = getTargetVShiftByConstNode(X86ISD::VSHLI, DL, ShiftVT,
54688 ShiftRHS, ShiftAmt, DAG);
54689 ShiftLHS = DAG.getBitcast(SrcVT, ShiftLHS);
54690 ShiftRHS = DAG.getBitcast(SrcVT, ShiftRHS);
54691 SDValue Res = DAG.getNode(ISD::XOR, DL, SrcVT, ShiftLHS, ShiftRHS);
54692 return DAG.getNode(X86ISD::MOVMSK, DL, VT, DAG.getNOT(DL, Res, SrcVT));
54693 }
54694 }
54695
54696 // Fold movmsk(logic(X,C)) -> logic(movmsk(X),C)
54697 if (N->isOnlyUserOf(Src.getNode())) {
54698 SDValue SrcBC = peekThroughOneUseBitcasts(Src);
54699 if (ISD::isBitwiseLogicOp(SrcBC.getOpcode())) {
54700 APInt UndefElts;
54701 SmallVector<APInt, 32> EltBits;
54702 if (getTargetConstantBitsFromNode(SrcBC.getOperand(1), NumBitsPerElt,
54703 UndefElts, EltBits)) {
54704 APInt Mask = APInt::getZero(NumBits);
54705 for (unsigned Idx = 0; Idx != NumElts; ++Idx) {
54706 if (!UndefElts[Idx] && EltBits[Idx].isNegative())
54707 Mask.setBit(Idx);
54708 }
54709 SDLoc DL(N);
54710 SDValue NewSrc = DAG.getBitcast(SrcVT, SrcBC.getOperand(0));
54711 SDValue NewMovMsk = DAG.getNode(X86ISD::MOVMSK, DL, VT, NewSrc);
54712 return DAG.getNode(SrcBC.getOpcode(), DL, VT, NewMovMsk,
54713 DAG.getConstant(Mask, DL, VT));
54714 }
54715 }
54716 }
54717
54718 // Simplify the inputs.
54719 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54720 APInt DemandedMask(APInt::getAllOnes(NumBits));
54721 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
54722 return SDValue(N, 0);
54723
54724 return SDValue();
54725}
54726
54727static SDValue combineTESTP(SDNode *N, SelectionDAG &DAG,
54728 TargetLowering::DAGCombinerInfo &DCI,
54729 const X86Subtarget &Subtarget) {
54730 MVT VT = N->getSimpleValueType(0);
54731 unsigned NumBits = VT.getScalarSizeInBits();
54732
54733 // Simplify the inputs.
54734 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54735 APInt DemandedMask(APInt::getAllOnes(NumBits));
54736 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
54737 return SDValue(N, 0);
54738
54739 return SDValue();
54740}
54741
54742static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG,
54743 TargetLowering::DAGCombinerInfo &DCI) {
54744 auto *MemOp = cast<X86MaskedGatherScatterSDNode>(N);
54745 SDValue Mask = MemOp->getMask();
54746
54747 // With vector masks we only demand the upper bit of the mask.
54748 if (Mask.getScalarValueSizeInBits() != 1) {
54749 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54750 APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
54751 if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI)) {
54752 if (N->getOpcode() != ISD::DELETED_NODE)
54753 DCI.AddToWorklist(N);
54754 return SDValue(N, 0);
54755 }
54756 }
54757
54758 return SDValue();
54759}
54760
54761static SDValue rebuildGatherScatter(MaskedGatherScatterSDNode *GorS,
54762 SDValue Index, SDValue Base, SDValue Scale,
54763 SelectionDAG &DAG) {
54764 SDLoc DL(GorS);
54765
54766 if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
54767 SDValue Ops[] = { Gather->getChain(), Gather->getPassThru(),
54768 Gather->getMask(), Base, Index, Scale } ;
54769 return DAG.getMaskedGather(Gather->getVTList(),
54770 Gather->getMemoryVT(), DL, Ops,
54771 Gather->getMemOperand(),
54772 Gather->getIndexType(),
54773 Gather->getExtensionType());
54774 }
54775 auto *Scatter = cast<MaskedScatterSDNode>(GorS);
54776 SDValue Ops[] = { Scatter->getChain(), Scatter->getValue(),
54777 Scatter->getMask(), Base, Index, Scale };
54778 return DAG.getMaskedScatter(Scatter->getVTList(),
54779 Scatter->getMemoryVT(), DL,
54780 Ops, Scatter->getMemOperand(),
54781 Scatter->getIndexType(),
54782 Scatter->isTruncatingStore());
54783}
54784
54785static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
54786 TargetLowering::DAGCombinerInfo &DCI) {
54787 SDLoc DL(N);
54788 auto *GorS = cast<MaskedGatherScatterSDNode>(N);
54789 SDValue Index = GorS->getIndex();
54790 SDValue Base = GorS->getBasePtr();
54791 SDValue Scale = GorS->getScale();
54792 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
54793
54794 if (DCI.isBeforeLegalize()) {
54795 unsigned IndexWidth = Index.getScalarValueSizeInBits();
54796
54797 // Shrink constant indices if they are larger than 32-bits.
54798 // Only do this before legalize types since v2i64 could become v2i32.
54799 // FIXME: We could check that the type is legal if we're after legalize
54800 // types, but then we would need to construct test cases where that happens.
54801 // FIXME: We could support more than just constant vectors, but we need to
54802 // careful with costing. A truncate that can be optimized out would be fine.
54803 // Otherwise we might only want to create a truncate if it avoids a split.
54804 if (auto *BV = dyn_cast<BuildVectorSDNode>(Index)) {
54805 if (BV->isConstant() && IndexWidth > 32 &&
54806 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
54807 EVT NewVT = Index.getValueType().changeVectorElementType(MVT::i32);
54808 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
54809 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
54810 }
54811 }
54812
54813 // Shrink any sign/zero extends from 32 or smaller to larger than 32 if
54814 // there are sufficient sign bits. Only do this before legalize types to
54815 // avoid creating illegal types in truncate.
54816 if ((Index.getOpcode() == ISD::SIGN_EXTEND ||
54817 Index.getOpcode() == ISD::ZERO_EXTEND) &&
54818 IndexWidth > 32 &&
54819 Index.getOperand(0).getScalarValueSizeInBits() <= 32 &&
54820 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
54821 EVT NewVT = Index.getValueType().changeVectorElementType(MVT::i32);
54822 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
54823 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
54824 }
54825 }
54826
54827 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
54828 // Try to move splat constant adders from the index operand to the base
54829 // pointer operand. Taking care to multiply by the scale. We can only do
54830 // this when index element type is the same as the pointer type.
54831 // Otherwise we need to be sure the math doesn't wrap before the scale.
54832 if (Index.getOpcode() == ISD::ADD &&
54833 Index.getValueType().getVectorElementType() == PtrVT &&
54834 isa<ConstantSDNode>(Scale)) {
54835 uint64_t ScaleAmt = Scale->getAsZExtVal();
54836 if (auto *BV = dyn_cast<BuildVectorSDNode>(Index.getOperand(1))) {
54837 BitVector UndefElts;
54838 if (ConstantSDNode *C = BV->getConstantSplatNode(&UndefElts)) {
54839 // FIXME: Allow non-constant?
54840 if (UndefElts.none()) {
54841 // Apply the scale.
54842 APInt Adder = C->getAPIntValue() * ScaleAmt;
54843 // Add it to the existing base.
54844 Base = DAG.getNode(ISD::ADD, DL, PtrVT, Base,
54845 DAG.getConstant(Adder, DL, PtrVT));
54846 Index = Index.getOperand(0);
54847 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
54848 }
54849 }
54850
54851 // It's also possible base is just a constant. In that case, just
54852 // replace it with 0 and move the displacement into the index.
54853 if (BV->isConstant() && isa<ConstantSDNode>(Base) &&
54854 isOneConstant(Scale)) {
54855 SDValue Splat = DAG.getSplatBuildVector(Index.getValueType(), DL, Base);
54856 // Combine the constant build_vector and the constant base.
54857 Splat = DAG.getNode(ISD::ADD, DL, Index.getValueType(),
54858 Index.getOperand(1), Splat);
54859 // Add to the LHS of the original Index add.
54860 Index = DAG.getNode(ISD::ADD, DL, Index.getValueType(),
54861 Index.getOperand(0), Splat);
54862 Base = DAG.getConstant(0, DL, Base.getValueType());
54863 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
54864 }
54865 }
54866 }
54867
54868 if (DCI.isBeforeLegalizeOps()) {
54869 unsigned IndexWidth = Index.getScalarValueSizeInBits();
54870
54871 // Make sure the index is either i32 or i64
54872 if (IndexWidth != 32 && IndexWidth != 64) {
54873 MVT EltVT = IndexWidth > 32 ? MVT::i64 : MVT::i32;
54874 EVT IndexVT = Index.getValueType().changeVectorElementType(EltVT);
54875 Index = DAG.getSExtOrTrunc(Index, DL, IndexVT);
54876 return rebuildGatherScatter(GorS, Index, Base, Scale, DAG);
54877 }
54878 }
54879
54880 // With vector masks we only demand the upper bit of the mask.
54881 SDValue Mask = GorS->getMask();
54882 if (Mask.getScalarValueSizeInBits() != 1) {
54883 APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
54884 if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI)) {
54885 if (N->getOpcode() != ISD::DELETED_NODE)
54886 DCI.AddToWorklist(N);
54887 return SDValue(N, 0);
54888 }
54889 }
54890
54891 return SDValue();
54892}
54893
54894// Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
54895static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG,
54896 const X86Subtarget &Subtarget) {
54897 SDLoc DL(N);
54898 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0));
54899 SDValue EFLAGS = N->getOperand(1);
54900
54901 // Try to simplify the EFLAGS and condition code operands.
54902 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget))
54903 return getSETCC(CC, Flags, DL, DAG);
54904
54905 return SDValue();
54906}
54907
54908/// Optimize branch condition evaluation.
54909static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG,
54910 const X86Subtarget &Subtarget) {
54911 SDLoc DL(N);
54912 SDValue EFLAGS = N->getOperand(3);
54913 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2));
54914
54915 // Try to simplify the EFLAGS and condition code operands.
54916 // Make sure to not keep references to operands, as combineSetCCEFLAGS can
54917 // RAUW them under us.
54918 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) {
54919 SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
54920 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0),
54921 N->getOperand(1), Cond, Flags);
54922 }
54923
54924 return SDValue();
54925}
54926
54927// TODO: Could we move this to DAGCombine?
54928static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
54929 SelectionDAG &DAG) {
54930 // Take advantage of vector comparisons (etc.) producing 0 or -1 in each lane
54931 // to optimize away operation when it's from a constant.
54932 //
54933 // The general transformation is:
54934 // UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
54935 // AND(VECTOR_CMP(x,y), constant2)
54936 // constant2 = UNARYOP(constant)
54937
54938 // Early exit if this isn't a vector operation, the operand of the
54939 // unary operation isn't a bitwise AND, or if the sizes of the operations
54940 // aren't the same.
54941 EVT VT = N->getValueType(0);
54942 bool IsStrict = N->isStrictFPOpcode();
54943 unsigned NumEltBits = VT.getScalarSizeInBits();
54944 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
54945 if (!VT.isVector() || Op0.getOpcode() != ISD::AND ||
54946 DAG.ComputeNumSignBits(Op0.getOperand(0)) != NumEltBits ||
54947 VT.getSizeInBits() != Op0.getValueSizeInBits())
54948 return SDValue();
54949
54950 // Now check that the other operand of the AND is a constant. We could
54951 // make the transformation for non-constant splats as well, but it's unclear
54952 // that would be a benefit as it would not eliminate any operations, just
54953 // perform one more step in scalar code before moving to the vector unit.
54954 if (auto *BV = dyn_cast<BuildVectorSDNode>(Op0.getOperand(1))) {
54955 // Bail out if the vector isn't a constant.
54956 if (!BV->isConstant())
54957 return SDValue();
54958
54959 // Everything checks out. Build up the new and improved node.
54960 SDLoc DL(N);
54961 EVT IntVT = BV->getValueType(0);
54962 // Create a new constant of the appropriate type for the transformed
54963 // DAG.
54964 SDValue SourceConst;
54965 if (IsStrict)
54966 SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other},
54967 {N->getOperand(0), SDValue(BV, 0)});
54968 else
54969 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
54970 // The AND node needs bitcasts to/from an integer vector type around it.
54971 SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
54972 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
54973 MaskConst);
54974 SDValue Res = DAG.getBitcast(VT, NewAnd);
54975 if (IsStrict)
54976 return DAG.getMergeValues({Res, SourceConst.getValue(1)}, DL);
54977 return Res;
54978 }
54979
54980 return SDValue();
54981}
54982
54983/// If we are converting a value to floating-point, try to replace scalar
54984/// truncate of an extracted vector element with a bitcast. This tries to keep
54985/// the sequence on XMM registers rather than moving between vector and GPRs.
54986static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG) {
54987 // TODO: This is currently only used by combineSIntToFP, but it is generalized
54988 // to allow being called by any similar cast opcode.
54989 // TODO: Consider merging this into lowering: vectorizeExtractedCast().
54990 SDValue Trunc = N->getOperand(0);
54991 if (!Trunc.hasOneUse() || Trunc.getOpcode() != ISD::TRUNCATE)
54992 return SDValue();
54993
54994 SDValue ExtElt = Trunc.getOperand(0);
54995 if (!ExtElt.hasOneUse() || ExtElt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
54996 !isNullConstant(ExtElt.getOperand(1)))
54997 return SDValue();
54998
54999 EVT TruncVT = Trunc.getValueType();
55000 EVT SrcVT = ExtElt.getValueType();
55001 unsigned DestWidth = TruncVT.getSizeInBits();
55002 unsigned SrcWidth = SrcVT.getSizeInBits();
55003 if (SrcWidth % DestWidth != 0)
55004 return SDValue();
55005
55006 // inttofp (trunc (extelt X, 0)) --> inttofp (extelt (bitcast X), 0)
55007 EVT SrcVecVT = ExtElt.getOperand(0).getValueType();
55008 unsigned VecWidth = SrcVecVT.getSizeInBits();
55009 unsigned NumElts = VecWidth / DestWidth;
55010 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), TruncVT, NumElts);
55011 SDValue BitcastVec = DAG.getBitcast(BitcastVT, ExtElt.getOperand(0));
55012 SDLoc DL(N);
55013 SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT,
55014 BitcastVec, ExtElt.getOperand(1));
55015 return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), NewExtElt);
55016}
55017
55018static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG,
55019 const X86Subtarget &Subtarget) {
55020 bool IsStrict = N->isStrictFPOpcode();
55021 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
55022 EVT VT = N->getValueType(0);
55023 EVT InVT = Op0.getValueType();
55024
55025 // Using i16 as an intermediate type is a bad idea, unless we have HW support
55026 // for it. Therefore for type sizes equal or smaller than 32 just go with i32.
55027 // if hasFP16 support:
55028 // UINT_TO_FP(vXi1~15) -> SINT_TO_FP(ZEXT(vXi1~15 to vXi16))
55029 // UINT_TO_FP(vXi17~31) -> SINT_TO_FP(ZEXT(vXi17~31 to vXi32))
55030 // else
55031 // UINT_TO_FP(vXi1~31) -> SINT_TO_FP(ZEXT(vXi1~31 to vXi32))
55032 // UINT_TO_FP(vXi33~63) -> SINT_TO_FP(ZEXT(vXi33~63 to vXi64))
55033 if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) {
55034 unsigned ScalarSize = InVT.getScalarSizeInBits();
55035 if ((ScalarSize == 16 && Subtarget.hasFP16()) || ScalarSize == 32 ||
55036 ScalarSize >= 64)
55037 return SDValue();
55038 SDLoc dl(N);
55039 EVT DstVT =
55040 EVT::getVectorVT(*DAG.getContext(),
55041 (Subtarget.hasFP16() && ScalarSize < 16) ? MVT::i16
55042 : ScalarSize < 32 ? MVT::i32
55043 : MVT::i64,
55044 InVT.getVectorNumElements());
55045 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
55046 if (IsStrict)
55047 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
55048 {N->getOperand(0), P});
55049 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
55050 }
55051
55052 // UINT_TO_FP(vXi1) -> SINT_TO_FP(ZEXT(vXi1 to vXi32))
55053 // UINT_TO_FP(vXi8) -> SINT_TO_FP(ZEXT(vXi8 to vXi32))
55054 // UINT_TO_FP(vXi16) -> SINT_TO_FP(ZEXT(vXi16 to vXi32))
55055 if (InVT.isVector() && InVT.getScalarSizeInBits() < 32 &&
55056 VT.getScalarType() != MVT::f16) {
55057 SDLoc dl(N);
55058 EVT DstVT = InVT.changeVectorElementType(MVT::i32);
55059 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
55060
55061 // UINT_TO_FP isn't legal without AVX512 so use SINT_TO_FP.
55062 if (IsStrict)
55063 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
55064 {N->getOperand(0), P});
55065 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
55066 }
55067
55068 // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't
55069 // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform
55070 // the optimization here.
55071 SDNodeFlags Flags = N->getFlags();
55072 if (Flags.hasNonNeg() || DAG.SignBitIsZero(Op0)) {
55073 if (IsStrict)
55074 return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other},
55075 {N->getOperand(0), Op0});
55076 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0);
55077 }
55078
55079 return SDValue();
55080}
55081
55082static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
55083 TargetLowering::DAGCombinerInfo &DCI,
55084 const X86Subtarget &Subtarget) {
55085 // First try to optimize away the conversion entirely when it's
55086 // conditionally from a constant. Vectors only.
55087 bool IsStrict = N->isStrictFPOpcode();
55088 if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG))
55089 return Res;
55090
55091 // Now move on to more general possibilities.
55092 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
55093 EVT VT = N->getValueType(0);
55094 EVT InVT = Op0.getValueType();
55095
55096 // Using i16 as an intermediate type is a bad idea, unless we have HW support
55097 // for it. Therefore for type sizes equal or smaller than 32 just go with i32.
55098 // if hasFP16 support:
55099 // SINT_TO_FP(vXi1~15) -> SINT_TO_FP(SEXT(vXi1~15 to vXi16))
55100 // SINT_TO_FP(vXi17~31) -> SINT_TO_FP(SEXT(vXi17~31 to vXi32))
55101 // else
55102 // SINT_TO_FP(vXi1~31) -> SINT_TO_FP(ZEXT(vXi1~31 to vXi32))
55103 // SINT_TO_FP(vXi33~63) -> SINT_TO_FP(SEXT(vXi33~63 to vXi64))
55104 if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) {
55105 unsigned ScalarSize = InVT.getScalarSizeInBits();
55106 if ((ScalarSize == 16 && Subtarget.hasFP16()) || ScalarSize == 32 ||
55107 ScalarSize >= 64)
55108 return SDValue();
55109 SDLoc dl(N);
55110 EVT DstVT =
55111 EVT::getVectorVT(*DAG.getContext(),
55112 (Subtarget.hasFP16() && ScalarSize < 16) ? MVT::i16
55113 : ScalarSize < 32 ? MVT::i32
55114 : MVT::i64,
55115 InVT.getVectorNumElements());
55116 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
55117 if (IsStrict)
55118 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
55119 {N->getOperand(0), P});
55120 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
55121 }
55122
55123 // SINT_TO_FP(vXi1) -> SINT_TO_FP(SEXT(vXi1 to vXi32))
55124 // SINT_TO_FP(vXi8) -> SINT_TO_FP(SEXT(vXi8 to vXi32))
55125 // SINT_TO_FP(vXi16) -> SINT_TO_FP(SEXT(vXi16 to vXi32))
55126 if (InVT.isVector() && InVT.getScalarSizeInBits() < 32 &&
55127 VT.getScalarType() != MVT::f16) {
55128 SDLoc dl(N);
55129 EVT DstVT = InVT.changeVectorElementType(MVT::i32);
55130 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
55131 if (IsStrict)
55132 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
55133 {N->getOperand(0), P});
55134 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
55135 }
55136
55137 // Without AVX512DQ we only support i64 to float scalar conversion. For both
55138 // vectors and scalars, see if we know that the upper bits are all the sign
55139 // bit, in which case we can truncate the input to i32 and convert from that.
55140 if (InVT.getScalarSizeInBits() > 32 && !Subtarget.hasDQI()) {
55141 unsigned BitWidth = InVT.getScalarSizeInBits();
55142 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0);
55143 if (NumSignBits >= (BitWidth - 31)) {
55144 EVT TruncVT = MVT::i32;
55145 if (InVT.isVector())
55146 TruncVT = InVT.changeVectorElementType(TruncVT);
55147 SDLoc dl(N);
55148 if (DCI.isBeforeLegalize() || TruncVT != MVT::v2i32) {
55149 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0);
55150 if (IsStrict)
55151 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
55152 {N->getOperand(0), Trunc});
55153 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc);
55154 }
55155 // If we're after legalize and the type is v2i32 we need to shuffle and
55156 // use CVTSI2P.
55157 assert(InVT == MVT::v2i64 && "Unexpected VT!");
55158 SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
55159 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
55160 { 0, 2, -1, -1 });
55161 if (IsStrict)
55162 return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
55163 {N->getOperand(0), Shuf});
55164 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, Shuf);
55165 }
55166 }
55167
55168 // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have
55169 // a 32-bit target where SSE doesn't support i64->FP operations.
55170 if (!Subtarget.useSoftFloat() && Subtarget.hasX87() &&
55171 Op0.getOpcode() == ISD::LOAD) {
55172 LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode());
55173
55174 // This transformation is not supported if the result type is f16 or f128.
55175 if (VT == MVT::f16 || VT == MVT::f128)
55176 return SDValue();
55177
55178 // If we have AVX512DQ we can use packed conversion instructions unless
55179 // the VT is f80.
55180 if (Subtarget.hasDQI() && VT != MVT::f80)
55181 return SDValue();
55182
55183 if (Ld->isSimple() && !VT.isVector() && ISD::isNormalLoad(Op0.getNode()) &&
55184 Op0.hasOneUse() && !Subtarget.is64Bit() && InVT == MVT::i64) {
55185 std::pair<SDValue, SDValue> Tmp =
55186 Subtarget.getTargetLowering()->BuildFILD(
55187 VT, InVT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(),
55188 Ld->getPointerInfo(), Ld->getOriginalAlign(), DAG);
55189 DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Tmp.second);
55190 return Tmp.first;
55191 }
55192 }
55193
55194 if (IsStrict)
55195 return SDValue();
55196
55197 if (SDValue V = combineToFPTruncExtElt(N, DAG))
55198 return V;
55199
55200 return SDValue();
55201}
55202
55203static bool needCarryOrOverflowFlag(SDValue Flags) {
55204 assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!");
55205
55206 for (const SDNode *User : Flags->uses()) {
55207 X86::CondCode CC;
55208 switch (User->getOpcode()) {
55209 default:
55210 // Be conservative.
55211 return true;
55212 case X86ISD::SETCC:
55213 case X86ISD::SETCC_CARRY:
55214 CC = (X86::CondCode)User->getConstantOperandVal(0);
55215 break;
55216 case X86ISD::BRCOND:
55217 case X86ISD::CMOV:
55218 CC = (X86::CondCode)User->getConstantOperandVal(2);
55219 break;
55220 }
55221
55222 switch (CC) {
55223 // clang-format off
55224 default: break;
55225 case X86::COND_A: case X86::COND_AE:
55226 case X86::COND_B: case X86::COND_BE:
55227 case X86::COND_O: case X86::COND_NO:
55228 case X86::COND_G: case X86::COND_GE:
55229 case X86::COND_L: case X86::COND_LE:
55230 return true;
55231 // clang-format on
55232 }
55233 }
55234
55235 return false;
55236}
55237
55238static bool onlyZeroFlagUsed(SDValue Flags) {
55239 assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!");
55240
55241 for (const SDNode *User : Flags->uses()) {
55242 unsigned CCOpNo;
55243 switch (User->getOpcode()) {
55244 default:
55245 // Be conservative.
55246 return false;
55247 case X86ISD::SETCC:
55248 case X86ISD::SETCC_CARRY:
55249 CCOpNo = 0;
55250 break;
55251 case X86ISD::BRCOND:
55252 case X86ISD::CMOV:
55253 CCOpNo = 2;
55254 break;
55255 }
55256
55257 X86::CondCode CC = (X86::CondCode)User->getConstantOperandVal(CCOpNo);
55258 if (CC != X86::COND_E && CC != X86::COND_NE)
55259 return false;
55260 }
55261
55262 return true;
55263}
55264
55265static SDValue combineCMP(SDNode *N, SelectionDAG &DAG,
55266 TargetLowering::DAGCombinerInfo &DCI,
55267 const X86Subtarget &Subtarget) {
55268 // Only handle test patterns.
55269 if (!isNullConstant(N->getOperand(1)))
55270 return SDValue();
55271
55272 // If we have a CMP of a truncated binop, see if we can make a smaller binop
55273 // and use its flags directly.
55274 // TODO: Maybe we should try promoting compares that only use the zero flag
55275 // first if we can prove the upper bits with computeKnownBits?
55276 SDLoc dl(N);
55277 SDValue Op = N->getOperand(0);
55278 EVT VT = Op.getValueType();
55279 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55280
55281 if (SDValue CMP =
55282 combineX86SubCmpForFlags(N, SDValue(N, 0), DAG, DCI, Subtarget))
55283 return CMP;
55284
55285 // If we have a constant logical shift that's only used in a comparison
55286 // against zero turn it into an equivalent AND. This allows turning it into
55287 // a TEST instruction later.
55288 if ((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) &&
55289 Op.hasOneUse() && isa<ConstantSDNode>(Op.getOperand(1)) &&
55290 onlyZeroFlagUsed(SDValue(N, 0))) {
55291 unsigned BitWidth = VT.getSizeInBits();
55292 const APInt &ShAmt = Op.getConstantOperandAPInt(1);
55293 if (ShAmt.ult(BitWidth)) { // Avoid undefined shifts.
55294 unsigned MaskBits = BitWidth - ShAmt.getZExtValue();
55295 APInt Mask = Op.getOpcode() == ISD::SRL
55296 ? APInt::getHighBitsSet(BitWidth, MaskBits)
55297 : APInt::getLowBitsSet(BitWidth, MaskBits);
55298 if (Mask.isSignedIntN(32)) {
55299 Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0),
55300 DAG.getConstant(Mask, dl, VT));
55301 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
55302 DAG.getConstant(0, dl, VT));
55303 }
55304 }
55305 }
55306
55307 // If we're extracting from a avx512 bool vector and comparing against zero,
55308 // then try to just bitcast the vector to an integer to use TEST/BT directly.
55309 // (and (extract_elt (kshiftr vXi1, C), 0), 1) -> (and (bc vXi1), 1<<C)
55310 if (Op.getOpcode() == ISD::AND && isOneConstant(Op.getOperand(1)) &&
55311 Op.hasOneUse() && onlyZeroFlagUsed(SDValue(N, 0))) {
55312 SDValue Src = Op.getOperand(0);
55313 if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
55314 isNullConstant(Src.getOperand(1)) &&
55315 Src.getOperand(0).getValueType().getScalarType() == MVT::i1) {
55316 SDValue BoolVec = Src.getOperand(0);
55317 unsigned ShAmt = 0;
55318 if (BoolVec.getOpcode() == X86ISD::KSHIFTR) {
55319 ShAmt = BoolVec.getConstantOperandVal(1);
55320 BoolVec = BoolVec.getOperand(0);
55321 }
55322 BoolVec = widenMaskVector(BoolVec, false, Subtarget, DAG, dl);
55323 EVT VecVT = BoolVec.getValueType();
55324 unsigned BitWidth = VecVT.getVectorNumElements();
55325 EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), BitWidth);
55326 if (TLI.isTypeLegal(VecVT) && TLI.isTypeLegal(BCVT)) {
55327 APInt Mask = APInt::getOneBitSet(BitWidth, ShAmt);
55328 Op = DAG.getBitcast(BCVT, BoolVec);
55329 Op = DAG.getNode(ISD::AND, dl, BCVT, Op,
55330 DAG.getConstant(Mask, dl, BCVT));
55331 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
55332 DAG.getConstant(0, dl, BCVT));
55333 }
55334 }
55335 }
55336
55337 // Peek through any zero-extend if we're only testing for a zero result.
55338 if (Op.getOpcode() == ISD::ZERO_EXTEND && onlyZeroFlagUsed(SDValue(N, 0))) {
55339 SDValue Src = Op.getOperand(0);
55340 EVT SrcVT = Src.getValueType();
55341 if (SrcVT.getScalarSizeInBits() >= 8 && TLI.isTypeLegal(SrcVT))
55342 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Src,
55343 DAG.getConstant(0, dl, SrcVT));
55344 }
55345
55346 // Look for a truncate.
55347 if (Op.getOpcode() != ISD::TRUNCATE)
55348 return SDValue();
55349
55350 SDValue Trunc = Op;
55351 Op = Op.getOperand(0);
55352
55353 // See if we can compare with zero against the truncation source,
55354 // which should help using the Z flag from many ops. Only do this for
55355 // i32 truncated op to prevent partial-reg compares of promoted ops.
55356 EVT OpVT = Op.getValueType();
55357 APInt UpperBits =
55358 APInt::getBitsSetFrom(OpVT.getSizeInBits(), VT.getSizeInBits());
55359 if (OpVT == MVT::i32 && DAG.MaskedValueIsZero(Op, UpperBits) &&
55360 onlyZeroFlagUsed(SDValue(N, 0))) {
55361 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
55362 DAG.getConstant(0, dl, OpVT));
55363 }
55364
55365 // After this the truncate and arithmetic op must have a single use.
55366 if (!Trunc.hasOneUse() || !Op.hasOneUse())
55367 return SDValue();
55368
55369 unsigned NewOpc;
55370 switch (Op.getOpcode()) {
55371 default: return SDValue();
55372 case ISD::AND:
55373 // Skip and with constant. We have special handling for and with immediate
55374 // during isel to generate test instructions.
55375 if (isa<ConstantSDNode>(Op.getOperand(1)))
55376 return SDValue();
55377 NewOpc = X86ISD::AND;
55378 break;
55379 case ISD::OR: NewOpc = X86ISD::OR; break;
55380 case ISD::XOR: NewOpc = X86ISD::XOR; break;
55381 case ISD::ADD:
55382 // If the carry or overflow flag is used, we can't truncate.
55383 if (needCarryOrOverflowFlag(SDValue(N, 0)))
55384 return SDValue();
55385 NewOpc = X86ISD::ADD;
55386 break;
55387 case ISD::SUB:
55388 // If the carry or overflow flag is used, we can't truncate.
55389 if (needCarryOrOverflowFlag(SDValue(N, 0)))
55390 return SDValue();
55391 NewOpc = X86ISD::SUB;
55392 break;
55393 }
55394
55395 // We found an op we can narrow. Truncate its inputs.
55396 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0));
55397 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1));
55398
55399 // Use a X86 specific opcode to avoid DAG combine messing with it.
55400 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
55401 Op = DAG.getNode(NewOpc, dl, VTs, Op0, Op1);
55402
55403 // For AND, keep a CMP so that we can match the test pattern.
55404 if (NewOpc == X86ISD::AND)
55405 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
55406 DAG.getConstant(0, dl, VT));
55407
55408 // Return the flags.
55409 return Op.getValue(1);
55410}
55411
55412static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
55413 TargetLowering::DAGCombinerInfo &DCI,
55414 const X86Subtarget &ST) {
55415 assert((X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode()) &&
55416 "Expected X86ISD::ADD or X86ISD::SUB");
55417
55418 SDLoc DL(N);
55419 SDValue LHS = N->getOperand(0);
55420 SDValue RHS = N->getOperand(1);
55421 MVT VT = LHS.getSimpleValueType();
55422 bool IsSub = X86ISD::SUB == N->getOpcode();
55423 unsigned GenericOpc = IsSub ? ISD::SUB : ISD::ADD;
55424
55425 if (IsSub && isOneConstant(N->getOperand(1)) && !N->hasAnyUseOfValue(0))
55426 if (SDValue CMP = combineX86SubCmpForFlags(N, SDValue(N, 1), DAG, DCI, ST))
55427 return CMP;
55428
55429 // If we don't use the flag result, simplify back to a generic ADD/SUB.
55430 if (!N->hasAnyUseOfValue(1)) {
55431 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS);
55432 return DAG.getMergeValues({Res, DAG.getConstant(0, DL, MVT::i32)}, DL);
55433 }
55434
55435 // Fold any similar generic ADD/SUB opcodes to reuse this node.
55436 auto MatchGeneric = [&](SDValue N0, SDValue N1, bool Negate) {
55437 SDValue Ops[] = {N0, N1};
55438 SDVTList VTs = DAG.getVTList(N->getValueType(0));
55439 if (SDNode *GenericAddSub = DAG.getNodeIfExists(GenericOpc, VTs, Ops)) {
55440 SDValue Op(N, 0);
55441 if (Negate)
55442 Op = DAG.getNegative(Op, DL, VT);
55443 DCI.CombineTo(GenericAddSub, Op);
55444 }
55445 };
55446 MatchGeneric(LHS, RHS, false);
55447 MatchGeneric(RHS, LHS, X86ISD::SUB == N->getOpcode());
55448
55449 // TODO: Can we drop the ZeroSecondOpOnly limit? This is to guarantee that the
55450 // EFLAGS result doesn't change.
55451 return combineAddOrSubToADCOrSBB(IsSub, DL, VT, LHS, RHS, DAG,
55452 /*ZeroSecondOpOnly*/ true);
55453}
55454
55455static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
55456 SDValue LHS = N->getOperand(0);
55457 SDValue RHS = N->getOperand(1);
55458 SDValue BorrowIn = N->getOperand(2);
55459
55460 if (SDValue Flags = combineCarryThroughADD(BorrowIn, DAG)) {
55461 MVT VT = N->getSimpleValueType(0);
55462 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
55463 return DAG.getNode(X86ISD::SBB, SDLoc(N), VTs, LHS, RHS, Flags);
55464 }
55465
55466 // Fold SBB(SUB(X,Y),0,Carry) -> SBB(X,Y,Carry)
55467 // iff the flag result is dead.
55468 if (LHS.getOpcode() == ISD::SUB && isNullConstant(RHS) &&
55469 !N->hasAnyUseOfValue(1))
55470 return DAG.getNode(X86ISD::SBB, SDLoc(N), N->getVTList(), LHS.getOperand(0),
55471 LHS.getOperand(1), BorrowIn);
55472
55473 return SDValue();
55474}
55475
55476// Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
55477static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
55478 TargetLowering::DAGCombinerInfo &DCI) {
55479 SDValue LHS = N->getOperand(0);
55480 SDValue RHS = N->getOperand(1);
55481 SDValue CarryIn = N->getOperand(2);
55482 auto *LHSC = dyn_cast<ConstantSDNode>(LHS);
55483 auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
55484
55485 // Canonicalize constant to RHS.
55486 if (LHSC && !RHSC)
55487 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), RHS, LHS,
55488 CarryIn);
55489
55490 // If the LHS and RHS of the ADC node are zero, then it can't overflow and
55491 // the result is either zero or one (depending on the input carry bit).
55492 // Strength reduce this down to a "set on carry" aka SETCC_CARRY&1.
55493 if (LHSC && RHSC && LHSC->isZero() && RHSC->isZero() &&
55494 // We don't have a good way to replace an EFLAGS use, so only do this when
55495 // dead right now.
55496 SDValue(N, 1).use_empty()) {
55497 SDLoc DL(N);
55498 EVT VT = N->getValueType(0);
55499 SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
55500 SDValue Res1 = DAG.getNode(
55501 ISD::AND, DL, VT,
55502 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
55503 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), CarryIn),
55504 DAG.getConstant(1, DL, VT));
55505 return DCI.CombineTo(N, Res1, CarryOut);
55506 }
55507
55508 // Fold ADC(C1,C2,Carry) -> ADC(0,C1+C2,Carry)
55509 // iff the flag result is dead.
55510 // TODO: Allow flag result if C1+C2 doesn't signed/unsigned overflow.
55511 if (LHSC && RHSC && !LHSC->isZero() && !N->hasAnyUseOfValue(1)) {
55512 SDLoc DL(N);
55513 APInt Sum = LHSC->getAPIntValue() + RHSC->getAPIntValue();
55514 return DAG.getNode(X86ISD::ADC, DL, N->getVTList(),
55515 DAG.getConstant(0, DL, LHS.getValueType()),
55516 DAG.getConstant(Sum, DL, LHS.getValueType()), CarryIn);
55517 }
55518
55519 if (SDValue Flags = combineCarryThroughADD(CarryIn, DAG)) {
55520 MVT VT = N->getSimpleValueType(0);
55521 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
55522 return DAG.getNode(X86ISD::ADC, SDLoc(N), VTs, LHS, RHS, Flags);
55523 }
55524
55525 // Fold ADC(ADD(X,Y),0,Carry) -> ADC(X,Y,Carry)
55526 // iff the flag result is dead.
55527 if (LHS.getOpcode() == ISD::ADD && RHSC && RHSC->isZero() &&
55528 !N->hasAnyUseOfValue(1))
55529 return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), LHS.getOperand(0),
55530 LHS.getOperand(1), CarryIn);
55531
55532 return SDValue();
55533}
55534
55535static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
55536 const SDLoc &DL, EVT VT,
55537 const X86Subtarget &Subtarget) {
55538 // Example of pattern we try to detect:
55539 // t := (v8i32 mul (sext (v8i16 x0), (sext (v8i16 x1))))
55540 //(add (build_vector (extract_elt t, 0),
55541 // (extract_elt t, 2),
55542 // (extract_elt t, 4),
55543 // (extract_elt t, 6)),
55544 // (build_vector (extract_elt t, 1),
55545 // (extract_elt t, 3),
55546 // (extract_elt t, 5),
55547 // (extract_elt t, 7)))
55548
55549 if (!Subtarget.hasSSE2())
55550 return SDValue();
55551
55552 if (Op0.getOpcode() != ISD::BUILD_VECTOR ||
55553 Op1.getOpcode() != ISD::BUILD_VECTOR)
55554 return SDValue();
55555
55556 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
55557 VT.getVectorNumElements() < 4 ||
55558 !isPowerOf2_32(VT.getVectorNumElements()))
55559 return SDValue();
55560
55561 // Check if one of Op0,Op1 is of the form:
55562 // (build_vector (extract_elt Mul, 0),
55563 // (extract_elt Mul, 2),
55564 // (extract_elt Mul, 4),
55565 // ...
55566 // the other is of the form:
55567 // (build_vector (extract_elt Mul, 1),
55568 // (extract_elt Mul, 3),
55569 // (extract_elt Mul, 5),
55570 // ...
55571 // and identify Mul.
55572 SDValue Mul;
55573 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; i += 2) {
55574 SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i),
55575 Op0H = Op0->getOperand(i + 1), Op1H = Op1->getOperand(i + 1);
55576 // TODO: Be more tolerant to undefs.
55577 if (Op0L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
55578 Op1L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
55579 Op0H.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
55580 Op1H.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
55581 return SDValue();
55582 auto *Const0L = dyn_cast<ConstantSDNode>(Op0L->getOperand(1));
55583 auto *Const1L = dyn_cast<ConstantSDNode>(Op1L->getOperand(1));
55584 auto *Const0H = dyn_cast<ConstantSDNode>(Op0H->getOperand(1));
55585 auto *Const1H = dyn_cast<ConstantSDNode>(Op1H->getOperand(1));
55586 if (!Const0L || !Const1L || !Const0H || !Const1H)
55587 return SDValue();
55588 unsigned Idx0L = Const0L->getZExtValue(), Idx1L = Const1L->getZExtValue(),
55589 Idx0H = Const0H->getZExtValue(), Idx1H = Const1H->getZExtValue();
55590 // Commutativity of mul allows factors of a product to reorder.
55591 if (Idx0L > Idx1L)
55592 std::swap(Idx0L, Idx1L);
55593 if (Idx0H > Idx1H)
55594 std::swap(Idx0H, Idx1H);
55595 // Commutativity of add allows pairs of factors to reorder.
55596 if (Idx0L > Idx0H) {
55597 std::swap(Idx0L, Idx0H);
55598 std::swap(Idx1L, Idx1H);
55599 }
55600 if (Idx0L != 2 * i || Idx1L != 2 * i + 1 || Idx0H != 2 * i + 2 ||
55601 Idx1H != 2 * i + 3)
55602 return SDValue();
55603 if (!Mul) {
55604 // First time an extract_elt's source vector is visited. Must be a MUL
55605 // with 2X number of vector elements than the BUILD_VECTOR.
55606 // Both extracts must be from same MUL.
55607 Mul = Op0L->getOperand(0);
55608 if (Mul->getOpcode() != ISD::MUL ||
55609 Mul.getValueType().getVectorNumElements() != 2 * e)
55610 return SDValue();
55611 }
55612 // Check that the extract is from the same MUL previously seen.
55613 if (Mul != Op0L->getOperand(0) || Mul != Op1L->getOperand(0) ||
55614 Mul != Op0H->getOperand(0) || Mul != Op1H->getOperand(0))
55615 return SDValue();
55616 }
55617
55618 // Check if the Mul source can be safely shrunk.
55619 ShrinkMode Mode;
55620 if (!canReduceVMulWidth(Mul.getNode(), DAG, Mode) ||
55621 Mode == ShrinkMode::MULU16)
55622 return SDValue();
55623
55624 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
55625 VT.getVectorNumElements() * 2);
55626 SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(0));
55627 SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Mul.getOperand(1));
55628
55629 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
55630 ArrayRef<SDValue> Ops) {
55631 EVT InVT = Ops[0].getValueType();
55632 assert(InVT == Ops[1].getValueType() && "Operands' types mismatch");
55633 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
55634 InVT.getVectorNumElements() / 2);
55635 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
55636 };
55637 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder);
55638}
55639
55640// Attempt to turn this pattern into PMADDWD.
55641// (add (mul (sext (build_vector)), (sext (build_vector))),
55642// (mul (sext (build_vector)), (sext (build_vector)))
55643static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
55644 const SDLoc &DL, EVT VT,
55645 const X86Subtarget &Subtarget) {
55646 if (!Subtarget.hasSSE2())
55647 return SDValue();
55648
55649 if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL)
55650 return SDValue();
55651
55652 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
55653 VT.getVectorNumElements() < 4 ||
55654 !isPowerOf2_32(VT.getVectorNumElements()))
55655 return SDValue();
55656
55657 SDValue N00 = N0.getOperand(0);
55658 SDValue N01 = N0.getOperand(1);
55659 SDValue N10 = N1.getOperand(0);
55660 SDValue N11 = N1.getOperand(1);
55661
55662 // All inputs need to be sign extends.
55663 // TODO: Support ZERO_EXTEND from known positive?
55664 if (N00.getOpcode() != ISD::SIGN_EXTEND ||
55665 N01.getOpcode() != ISD::SIGN_EXTEND ||
55666 N10.getOpcode() != ISD::SIGN_EXTEND ||
55667 N11.getOpcode() != ISD::SIGN_EXTEND)
55668 return SDValue();
55669
55670 // Peek through the extends.
55671 N00 = N00.getOperand(0);
55672 N01 = N01.getOperand(0);
55673 N10 = N10.getOperand(0);
55674 N11 = N11.getOperand(0);
55675
55676 // Must be extending from vXi16.
55677 EVT InVT = N00.getValueType();
55678 if (InVT.getVectorElementType() != MVT::i16 || N01.getValueType() != InVT ||
55679 N10.getValueType() != InVT || N11.getValueType() != InVT)
55680 return SDValue();
55681
55682 // All inputs should be build_vectors.
55683 if (N00.getOpcode() != ISD::BUILD_VECTOR ||
55684 N01.getOpcode() != ISD::BUILD_VECTOR ||
55685 N10.getOpcode() != ISD::BUILD_VECTOR ||
55686 N11.getOpcode() != ISD::BUILD_VECTOR)
55687 return SDValue();
55688
55689 // For each element, we need to ensure we have an odd element from one vector
55690 // multiplied by the odd element of another vector and the even element from
55691 // one of the same vectors being multiplied by the even element from the
55692 // other vector. So we need to make sure for each element i, this operator
55693 // is being performed:
55694 // A[2 * i] * B[2 * i] + A[2 * i + 1] * B[2 * i + 1]
55695 SDValue In0, In1;
55696 for (unsigned i = 0; i != N00.getNumOperands(); ++i) {
55697 SDValue N00Elt = N00.getOperand(i);
55698 SDValue N01Elt = N01.getOperand(i);
55699 SDValue N10Elt = N10.getOperand(i);
55700 SDValue N11Elt = N11.getOperand(i);
55701 // TODO: Be more tolerant to undefs.
55702 if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
55703 N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
55704 N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
55705 N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
55706 return SDValue();
55707 auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1));
55708 auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1));
55709 auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1));
55710 auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1));
55711 if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt)
55712 return SDValue();
55713 unsigned IdxN00 = ConstN00Elt->getZExtValue();
55714 unsigned IdxN01 = ConstN01Elt->getZExtValue();
55715 unsigned IdxN10 = ConstN10Elt->getZExtValue();
55716 unsigned IdxN11 = ConstN11Elt->getZExtValue();
55717 // Add is commutative so indices can be reordered.
55718 if (IdxN00 > IdxN10) {
55719 std::swap(IdxN00, IdxN10);
55720 std::swap(IdxN01, IdxN11);
55721 }
55722 // N0 indices be the even element. N1 indices must be the next odd element.
55723 if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
55724 IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
55725 return SDValue();
55726 SDValue N00In = N00Elt.getOperand(0);
55727 SDValue N01In = N01Elt.getOperand(0);
55728 SDValue N10In = N10Elt.getOperand(0);
55729 SDValue N11In = N11Elt.getOperand(0);
55730
55731 // First time we find an input capture it.
55732 if (!In0) {
55733 In0 = N00In;
55734 In1 = N01In;
55735
55736 // The input vectors must be at least as wide as the output.
55737 // If they are larger than the output, we extract subvector below.
55738 if (In0.getValueSizeInBits() < VT.getSizeInBits() ||
55739 In1.getValueSizeInBits() < VT.getSizeInBits())
55740 return SDValue();
55741 }
55742 // Mul is commutative so the input vectors can be in any order.
55743 // Canonicalize to make the compares easier.
55744 if (In0 != N00In)
55745 std::swap(N00In, N01In);
55746 if (In0 != N10In)
55747 std::swap(N10In, N11In);
55748 if (In0 != N00In || In1 != N01In || In0 != N10In || In1 != N11In)
55749 return SDValue();
55750 }
55751
55752 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
55753 ArrayRef<SDValue> Ops) {
55754 EVT OpVT = Ops[0].getValueType();
55755 assert(OpVT.getScalarType() == MVT::i16 &&
55756 "Unexpected scalar element type");
55757 assert(OpVT == Ops[1].getValueType() && "Operands' types mismatch");
55758 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
55759 OpVT.getVectorNumElements() / 2);
55760 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
55761 };
55762
55763 // If the output is narrower than an input, extract the low part of the input
55764 // vector.
55765 EVT OutVT16 = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
55766 VT.getVectorNumElements() * 2);
55767 if (OutVT16.bitsLT(In0.getValueType())) {
55768 In0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In0,
55769 DAG.getIntPtrConstant(0, DL));
55770 }
55771 if (OutVT16.bitsLT(In1.getValueType())) {
55772 In1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT16, In1,
55773 DAG.getIntPtrConstant(0, DL));
55774 }
55775 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 },
55776 PMADDBuilder);
55777}
55778
55779// ADD(VPMADDWD(X,Y),VPMADDWD(Z,W)) -> VPMADDWD(SHUFFLE(X,Z), SHUFFLE(Y,W))
55780// If upper element in each pair of both VPMADDWD are zero then we can merge
55781// the operand elements and use the implicit add of VPMADDWD.
55782// TODO: Add support for VPMADDUBSW (which isn't commutable).
55783static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
55784 const SDLoc &DL, EVT VT) {
55785 if (N0.getOpcode() != N1.getOpcode() || N0.getOpcode() != X86ISD::VPMADDWD)
55786 return SDValue();
55787
55788 // TODO: Add 256/512-bit support once VPMADDWD combines with shuffles.
55789 if (VT.getSizeInBits() > 128)
55790 return SDValue();
55791
55792 unsigned NumElts = VT.getVectorNumElements();
55793 MVT OpVT = N0.getOperand(0).getSimpleValueType();
55794 APInt DemandedBits = APInt::getAllOnes(OpVT.getScalarSizeInBits());
55795 APInt DemandedHiElts = APInt::getSplat(2 * NumElts, APInt(2, 2));
55796
55797 bool Op0HiZero =
55798 DAG.MaskedValueIsZero(N0.getOperand(0), DemandedBits, DemandedHiElts) ||
55799 DAG.MaskedValueIsZero(N0.getOperand(1), DemandedBits, DemandedHiElts);
55800 bool Op1HiZero =
55801 DAG.MaskedValueIsZero(N1.getOperand(0), DemandedBits, DemandedHiElts) ||
55802 DAG.MaskedValueIsZero(N1.getOperand(1), DemandedBits, DemandedHiElts);
55803
55804 // TODO: Check for zero lower elements once we have actual codegen that
55805 // creates them.
55806 if (!Op0HiZero || !Op1HiZero)
55807 return SDValue();
55808
55809 // Create a shuffle mask packing the lower elements from each VPMADDWD.
55810 SmallVector<int> Mask;
55811 for (int i = 0; i != (int)NumElts; ++i) {
55812 Mask.push_back(2 * i);
55813 Mask.push_back(2 * (i + NumElts));
55814 }
55815
55816 SDValue LHS =
55817 DAG.getVectorShuffle(OpVT, DL, N0.getOperand(0), N1.getOperand(0), Mask);
55818 SDValue RHS =
55819 DAG.getVectorShuffle(OpVT, DL, N0.getOperand(1), N1.getOperand(1), Mask);
55820 return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS);
55821}
55822
55823/// CMOV of constants requires materializing constant operands in registers.
55824/// Try to fold those constants into an 'add' instruction to reduce instruction
55825/// count. We do this with CMOV rather the generic 'select' because there are
55826/// earlier folds that may be used to turn select-of-constants into logic hacks.
55827static SDValue pushAddIntoCmovOfConsts(SDNode *N, const SDLoc &DL,
55828 SelectionDAG &DAG,
55829 const X86Subtarget &Subtarget) {
55830 // If an operand is zero, add-of-0 gets simplified away, so that's clearly
55831 // better because we eliminate 1-2 instructions. This transform is still
55832 // an improvement without zero operands because we trade 2 move constants and
55833 // 1 add for 2 adds (LEA) as long as the constants can be represented as
55834 // immediate asm operands (fit in 32-bits).
55835 auto isSuitableCmov = [](SDValue V) {
55836 if (V.getOpcode() != X86ISD::CMOV || !V.hasOneUse())
55837 return false;
55838 if (!isa<ConstantSDNode>(V.getOperand(0)) ||
55839 !isa<ConstantSDNode>(V.getOperand(1)))
55840 return false;
55841 return isNullConstant(V.getOperand(0)) || isNullConstant(V.getOperand(1)) ||
55842 (V.getConstantOperandAPInt(0).isSignedIntN(32) &&
55843 V.getConstantOperandAPInt(1).isSignedIntN(32));
55844 };
55845
55846 // Match an appropriate CMOV as the first operand of the add.
55847 SDValue Cmov = N->getOperand(0);
55848 SDValue OtherOp = N->getOperand(1);
55849 if (!isSuitableCmov(Cmov))
55850 std::swap(Cmov, OtherOp);
55851 if (!isSuitableCmov(Cmov))
55852 return SDValue();
55853
55854 // Don't remove a load folding opportunity for the add. That would neutralize
55855 // any improvements from removing constant materializations.
55856 if (X86::mayFoldLoad(OtherOp, Subtarget))
55857 return SDValue();
55858
55859 EVT VT = N->getValueType(0);
55860 SDValue FalseOp = Cmov.getOperand(0);
55861 SDValue TrueOp = Cmov.getOperand(1);
55862
55863 // We will push the add through the select, but we can potentially do better
55864 // if we know there is another add in the sequence and this is pointer math.
55865 // In that case, we can absorb an add into the trailing memory op and avoid
55866 // a 3-operand LEA which is likely slower than a 2-operand LEA.
55867 // TODO: If target has "slow3OpsLEA", do this even without the trailing memop?
55868 if (OtherOp.getOpcode() == ISD::ADD && OtherOp.hasOneUse() &&
55869 !isa<ConstantSDNode>(OtherOp.getOperand(0)) &&
55870 all_of(N->uses(), [&](SDNode *Use) {
55871 auto *MemNode = dyn_cast<MemSDNode>(Use);
55872 return MemNode && MemNode->getBasePtr().getNode() == N;
55873 })) {
55874 // add (cmov C1, C2), add (X, Y) --> add (cmov (add X, C1), (add X, C2)), Y
55875 // TODO: We are arbitrarily choosing op0 as the 1st piece of the sum, but
55876 // it is possible that choosing op1 might be better.
55877 SDValue X = OtherOp.getOperand(0), Y = OtherOp.getOperand(1);
55878 FalseOp = DAG.getNode(ISD::ADD, DL, VT, X, FalseOp);
55879 TrueOp = DAG.getNode(ISD::ADD, DL, VT, X, TrueOp);
55880 Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp,
55881 Cmov.getOperand(2), Cmov.getOperand(3));
55882 return DAG.getNode(ISD::ADD, DL, VT, Cmov, Y);
55883 }
55884
55885 // add (cmov C1, C2), OtherOp --> cmov (add OtherOp, C1), (add OtherOp, C2)
55886 FalseOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, FalseOp);
55887 TrueOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, TrueOp);
55888 return DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, Cmov.getOperand(2),
55889 Cmov.getOperand(3));
55890}
55891
55892static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
55893 TargetLowering::DAGCombinerInfo &DCI,
55894 const X86Subtarget &Subtarget) {
55895 EVT VT = N->getValueType(0);
55896 SDValue Op0 = N->getOperand(0);
55897 SDValue Op1 = N->getOperand(1);
55898 SDLoc DL(N);
55899
55900 if (SDValue Select = pushAddIntoCmovOfConsts(N, DL, DAG, Subtarget))
55901 return Select;
55902
55903 if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, DL, VT, Subtarget))
55904 return MAdd;
55905 if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, DL, VT, Subtarget))
55906 return MAdd;
55907 if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT))
55908 return MAdd;
55909
55910 // Try to synthesize horizontal adds from adds of shuffles.
55911 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
55912 return V;
55913
55914 // add(psadbw(X,0),psadbw(Y,0)) -> psadbw(add(X,Y),0)
55915 // iff X and Y won't overflow.
55916 if (Op0.getOpcode() == X86ISD::PSADBW && Op1.getOpcode() == X86ISD::PSADBW &&
55917 ISD::isBuildVectorAllZeros(Op0.getOperand(1).getNode()) &&
55918 ISD::isBuildVectorAllZeros(Op1.getOperand(1).getNode())) {
55919 if (DAG.willNotOverflowAdd(false, Op0.getOperand(0), Op1.getOperand(0))) {
55920 MVT OpVT = Op0.getOperand(1).getSimpleValueType();
55921 SDValue Sum =
55922 DAG.getNode(ISD::ADD, DL, OpVT, Op0.getOperand(0), Op1.getOperand(0));
55923 return DAG.getNode(X86ISD::PSADBW, DL, VT, Sum,
55924 getZeroVector(OpVT, Subtarget, DAG, DL));
55925 }
55926 }
55927
55928 // If vectors of i1 are legal, turn (add (zext (vXi1 X)), Y) into
55929 // (sub Y, (sext (vXi1 X))).
55930 // FIXME: We have the (sub Y, (zext (vXi1 X))) -> (add (sext (vXi1 X)), Y) in
55931 // generic DAG combine without a legal type check, but adding this there
55932 // caused regressions.
55933 if (VT.isVector()) {
55934 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
55935 if (Op0.getOpcode() == ISD::ZERO_EXTEND &&
55936 Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
55937 TLI.isTypeLegal(Op0.getOperand(0).getValueType())) {
55938 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
55939 return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt);
55940 }
55941
55942 if (Op1.getOpcode() == ISD::ZERO_EXTEND &&
55943 Op1.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
55944 TLI.isTypeLegal(Op1.getOperand(0).getValueType())) {
55945 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
55946 return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt);
55947 }
55948 }
55949
55950 // Fold ADD(ADC(Y,0,W),X) -> ADC(X,Y,W)
55951 if (Op0.getOpcode() == X86ISD::ADC && Op0->hasOneUse() &&
55952 X86::isZeroNode(Op0.getOperand(1))) {
55953 assert(!Op0->hasAnyUseOfValue(1) && "Overflow bit in use");
55954 return DAG.getNode(X86ISD::ADC, SDLoc(Op0), Op0->getVTList(), Op1,
55955 Op0.getOperand(0), Op0.getOperand(2));
55956 }
55957
55958 return combineAddOrSubToADCOrSBB(N, DL, DAG);
55959}
55960
55961// Try to fold (sub Y, cmovns X, -X) -> (add Y, cmovns -X, X) if the cmov
55962// condition comes from the subtract node that produced -X. This matches the
55963// cmov expansion for absolute value. By swapping the operands we convert abs
55964// to nabs.
55965static SDValue combineSubABS(SDNode *N, SelectionDAG &DAG) {
55966 SDValue N0 = N->getOperand(0);
55967 SDValue N1 = N->getOperand(1);
55968
55969 if (N1.getOpcode() != X86ISD::CMOV || !N1.hasOneUse())
55970 return SDValue();
55971
55972 X86::CondCode CC = (X86::CondCode)N1.getConstantOperandVal(2);
55973 if (CC != X86::COND_S && CC != X86::COND_NS)
55974 return SDValue();
55975
55976 // Condition should come from a negate operation.
55977 SDValue Cond = N1.getOperand(3);
55978 if (Cond.getOpcode() != X86ISD::SUB || !isNullConstant(Cond.getOperand(0)))
55979 return SDValue();
55980 assert(Cond.getResNo() == 1 && "Unexpected result number");
55981
55982 // Get the X and -X from the negate.
55983 SDValue NegX = Cond.getValue(0);
55984 SDValue X = Cond.getOperand(1);
55985
55986 SDValue FalseOp = N1.getOperand(0);
55987 SDValue TrueOp = N1.getOperand(1);
55988
55989 // Cmov operands should be X and NegX. Order doesn't matter.
55990 if (!(TrueOp == X && FalseOp == NegX) && !(TrueOp == NegX && FalseOp == X))
55991 return SDValue();
55992
55993 // Build a new CMOV with the operands swapped.
55994 SDLoc DL(N);
55995 MVT VT = N->getSimpleValueType(0);
55996 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp,
55997 N1.getOperand(2), Cond);
55998 // Convert sub to add.
55999 return DAG.getNode(ISD::ADD, DL, VT, N0, Cmov);
56000}
56001
56002static SDValue combineSubSetcc(SDNode *N, SelectionDAG &DAG) {
56003 SDValue Op0 = N->getOperand(0);
56004 SDValue Op1 = N->getOperand(1);
56005
56006 // (sub C (zero_extend (setcc)))
56007 // =>
56008 // (add (zero_extend (setcc inverted) C-1)) if C is a nonzero immediate
56009 // Don't disturb (sub 0 setcc), which is easily done with neg.
56010 EVT VT = N->getValueType(0);
56011 auto *Op0C = dyn_cast<ConstantSDNode>(Op0);
56012 if (Op1.getOpcode() == ISD::ZERO_EXTEND && Op1.hasOneUse() && Op0C &&
56013 !Op0C->isZero() && Op1.getOperand(0).getOpcode() == X86ISD::SETCC &&
56014 Op1.getOperand(0).hasOneUse()) {
56015 SDValue SetCC = Op1.getOperand(0);
56016 X86::CondCode CC = (X86::CondCode)SetCC.getConstantOperandVal(0);
56017 X86::CondCode NewCC = X86::GetOppositeBranchCondition(CC);
56018 APInt NewImm = Op0C->getAPIntValue() - 1;
56019 SDLoc DL(Op1);
56020 SDValue NewSetCC = getSETCC(NewCC, SetCC.getOperand(1), DL, DAG);
56021 NewSetCC = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NewSetCC);
56022 return DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(VT, VT), NewSetCC,
56023 DAG.getConstant(NewImm, DL, VT));
56024 }
56025
56026 return SDValue();
56027}
56028
56029static SDValue combineX86CloadCstore(SDNode *N, SelectionDAG &DAG) {
56030 // res, flags2 = sub 0, (setcc cc, flag)
56031 // cload/cstore ..., cond_ne, flag2
56032 // ->
56033 // cload/cstore cc, flag
56034 if (N->getConstantOperandVal(3) != X86::COND_NE)
56035 return SDValue();
56036
56037 SDValue Sub = N->getOperand(4);
56038 if (Sub.getOpcode() != X86ISD::SUB)
56039 return SDValue();
56040
56041 SDValue SetCC = Sub.getOperand(1);
56042
56043 if (!X86::isZeroNode(Sub.getOperand(0)) || SetCC.getOpcode() != X86ISD::SETCC)
56044 return SDValue();
56045
56046 SmallVector<SDValue, 5> Ops(N->op_values());
56047 Ops[3] = SetCC.getOperand(0);
56048 Ops[4] = SetCC.getOperand(1);
56049
56050 return DAG.getMemIntrinsicNode(N->getOpcode(), SDLoc(N), N->getVTList(), Ops,
56051 cast<MemSDNode>(N)->getMemoryVT(),
56052 cast<MemSDNode>(N)->getMemOperand());
56053}
56054
56055static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
56056 TargetLowering::DAGCombinerInfo &DCI,
56057 const X86Subtarget &Subtarget) {
56058 SDValue Op0 = N->getOperand(0);
56059 SDValue Op1 = N->getOperand(1);
56060 SDLoc DL(N);
56061
56062 // TODO: Add NoOpaque handling to isConstantIntBuildVectorOrConstantInt.
56063 auto IsNonOpaqueConstant = [&](SDValue Op) {
56064 if (SDNode *C = DAG.isConstantIntBuildVectorOrConstantInt(Op)) {
56065 if (auto *Cst = dyn_cast<ConstantSDNode>(C))
56066 return !Cst->isOpaque();
56067 return true;
56068 }
56069 return false;
56070 };
56071
56072 // X86 can't encode an immediate LHS of a sub. See if we can push the
56073 // negation into a preceding instruction. If the RHS of the sub is a XOR with
56074 // one use and a constant, invert the immediate, saving one register.
56075 // However, ignore cases where C1 is 0, as those will become a NEG.
56076 // sub(C1, xor(X, C2)) -> add(xor(X, ~C2), C1+1)
56077 if (Op1.getOpcode() == ISD::XOR && IsNonOpaqueConstant(Op0) &&
56078 !isNullConstant(Op0) && IsNonOpaqueConstant(Op1.getOperand(1)) &&
56079 Op1->hasOneUse()) {
56080 EVT VT = Op0.getValueType();
56081 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, Op1.getOperand(0),
56082 DAG.getNOT(SDLoc(Op1), Op1.getOperand(1), VT));
56083 SDValue NewAdd =
56084 DAG.getNode(ISD::ADD, DL, VT, Op0, DAG.getConstant(1, DL, VT));
56085 return DAG.getNode(ISD::ADD, DL, VT, NewXor, NewAdd);
56086 }
56087
56088 if (SDValue V = combineSubABS(N, DAG))
56089 return V;
56090
56091 // Try to synthesize horizontal subs from subs of shuffles.
56092 if (SDValue V = combineToHorizontalAddSub(N, DAG, Subtarget))
56093 return V;
56094
56095 // Fold SUB(X,ADC(Y,0,W)) -> SBB(X,Y,W)
56096 if (Op1.getOpcode() == X86ISD::ADC && Op1->hasOneUse() &&
56097 X86::isZeroNode(Op1.getOperand(1))) {
56098 assert(!Op1->hasAnyUseOfValue(1) && "Overflow bit in use");
56099 return DAG.getNode(X86ISD::SBB, SDLoc(Op1), Op1->getVTList(), Op0,
56100 Op1.getOperand(0), Op1.getOperand(2));
56101 }
56102
56103 // Fold SUB(X,SBB(Y,Z,W)) -> SUB(ADC(X,Z,W),Y)
56104 // Don't fold to ADC(0,0,W)/SETCC_CARRY pattern which will prevent more folds.
56105 if (Op1.getOpcode() == X86ISD::SBB && Op1->hasOneUse() &&
56106 !(X86::isZeroNode(Op0) && X86::isZeroNode(Op1.getOperand(1)))) {
56107 assert(!Op1->hasAnyUseOfValue(1) && "Overflow bit in use");
56108 SDValue ADC = DAG.getNode(X86ISD::ADC, SDLoc(Op1), Op1->getVTList(), Op0,
56109 Op1.getOperand(1), Op1.getOperand(2));
56110 return DAG.getNode(ISD::SUB, DL, Op0.getValueType(), ADC.getValue(0),
56111 Op1.getOperand(0));
56112 }
56113
56114 if (SDValue V = combineXorSubCTLZ(N, DL, DAG, Subtarget))
56115 return V;
56116
56117 if (SDValue V = combineAddOrSubToADCOrSBB(N, DL, DAG))
56118 return V;
56119
56120 return combineSubSetcc(N, DAG);
56121}
56122
56123static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG,
56124 const X86Subtarget &Subtarget) {
56125 unsigned Opcode = N->getOpcode();
56126 assert((Opcode == X86ISD::PCMPEQ || Opcode == X86ISD::PCMPGT) &&
56127 "Unknown PCMP opcode");
56128
56129 SDValue LHS = N->getOperand(0);
56130 SDValue RHS = N->getOperand(1);
56131 MVT VT = N->getSimpleValueType(0);
56132 unsigned EltBits = VT.getScalarSizeInBits();
56133 unsigned NumElts = VT.getVectorNumElements();
56134 SDLoc DL(N);
56135
56136 if (LHS == RHS)
56137 return (Opcode == X86ISD::PCMPEQ) ? DAG.getAllOnesConstant(DL, VT)
56138 : DAG.getConstant(0, DL, VT);
56139
56140 // Constant Folding.
56141 // PCMPEQ(X,UNDEF) -> UNDEF
56142 // PCMPGT(X,UNDEF) -> 0
56143 // PCMPGT(UNDEF,X) -> 0
56144 APInt LHSUndefs, RHSUndefs;
56145 SmallVector<APInt> LHSBits, RHSBits;
56146 if (getTargetConstantBitsFromNode(LHS, EltBits, LHSUndefs, LHSBits) &&
56147 getTargetConstantBitsFromNode(RHS, EltBits, RHSUndefs, RHSBits)) {
56148 APInt Ones = APInt::getAllOnes(EltBits);
56149 APInt Zero = APInt::getZero(EltBits);
56150 SmallVector<APInt> Results(NumElts);
56151 for (unsigned I = 0; I != NumElts; ++I) {
56152 if (Opcode == X86ISD::PCMPEQ) {
56153 Results[I] = (LHSBits[I] == RHSBits[I]) ? Ones : Zero;
56154 } else {
56155 bool AnyUndef = LHSUndefs[I] || RHSUndefs[I];
56156 Results[I] = (!AnyUndef && LHSBits[I].sgt(RHSBits[I])) ? Ones : Zero;
56157 }
56158 }
56159 if (Opcode == X86ISD::PCMPEQ)
56160 return getConstVector(Results, LHSUndefs | RHSUndefs, VT, DAG, DL);
56161 return getConstVector(Results, VT, DAG, DL);
56162 }
56163
56164 return SDValue();
56165}
56166
56167// Helper to determine if we can convert an integer comparison to a float
56168// comparison byt casting the operands.
56169static std::optional<unsigned>
56170CastIntSETCCtoFP(MVT VT, ISD::CondCode CC, unsigned NumSignificantBitsLHS,
56171 unsigned NumSignificantBitsRHS) {
56172 MVT SVT = VT.getScalarType();
56173 assert(SVT == MVT::f32 && "Only tested for float so far");
56174 const fltSemantics &Sem = SVT.getFltSemantics();
56175 assert((CC == ISD::SETEQ || CC == ISD::SETGT) &&
56176 "Only PCMPEQ/PCMPGT currently supported");
56177
56178 // TODO: Handle bitcastable integers.
56179
56180 // For cvt + signed compare we need lhs and rhs to be exactly representable as
56181 // a fp value.
56182 unsigned FPPrec = APFloat::semanticsPrecision(Sem);
56183 if (FPPrec >= NumSignificantBitsLHS && FPPrec >= NumSignificantBitsRHS)
56184 return ISD::SINT_TO_FP;
56185
56186 return std::nullopt;
56187}
56188
56189/// Helper that combines an array of subvector ops as if they were the operands
56190/// of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.
56191/// ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.
56192static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
56193 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
56194 TargetLowering::DAGCombinerInfo &DCI,
56195 const X86Subtarget &Subtarget) {
56196 assert(Subtarget.hasAVX() && "AVX assumed for concat_vectors");
56197 unsigned EltSizeInBits = VT.getScalarSizeInBits();
56198
56199 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
56200 return DAG.getUNDEF(VT);
56201
56202 if (llvm::all_of(Ops, [](SDValue Op) {
56203 return ISD::isBuildVectorAllZeros(Op.getNode());
56204 }))
56205 return getZeroVector(VT, Subtarget, DAG, DL);
56206
56207 SDValue Op0 = Ops[0];
56208 bool IsSplat = llvm::all_equal(Ops);
56209 unsigned NumOps = Ops.size();
56210 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
56211 LLVMContext &Ctx = *DAG.getContext();
56212
56213 // Repeated subvectors.
56214 if (IsSplat &&
56215 (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) {
56216 // If this broadcast is inserted into both halves, use a larger broadcast.
56217 if (Op0.getOpcode() == X86ISD::VBROADCAST)
56218 return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0));
56219
56220 // concat_vectors(movddup(x),movddup(x)) -> broadcast(x)
56221 if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 &&
56222 (Subtarget.hasAVX2() ||
56223 X86::mayFoldLoadIntoBroadcastFromMem(Op0.getOperand(0),
56224 VT.getScalarType(), Subtarget)))
56225 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
56226 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64,
56227 Op0.getOperand(0),
56228 DAG.getIntPtrConstant(0, DL)));
56229
56230 // concat_vectors(scalar_to_vector(x),scalar_to_vector(x)) -> broadcast(x)
56231 if (Op0.getOpcode() == ISD::SCALAR_TO_VECTOR &&
56232 (Subtarget.hasAVX2() ||
56233 (EltSizeInBits >= 32 &&
56234 X86::mayFoldLoad(Op0.getOperand(0), Subtarget))) &&
56235 Op0.getOperand(0).getValueType() == VT.getScalarType())
56236 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0));
56237
56238 // concat_vectors(extract_subvector(broadcast(x)),
56239 // extract_subvector(broadcast(x))) -> broadcast(x)
56240 // concat_vectors(extract_subvector(subv_broadcast(x)),
56241 // extract_subvector(subv_broadcast(x))) -> subv_broadcast(x)
56242 if (Op0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
56243 Op0.getOperand(0).getValueType() == VT) {
56244 SDValue SrcVec = Op0.getOperand(0);
56245 if (SrcVec.getOpcode() == X86ISD::VBROADCAST ||
56246 SrcVec.getOpcode() == X86ISD::VBROADCAST_LOAD)
56247 return Op0.getOperand(0);
56248 if (SrcVec.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
56249 Op0.getValueType() == cast<MemSDNode>(SrcVec)->getMemoryVT())
56250 return Op0.getOperand(0);
56251 }
56252
56253 // concat_vectors(permq(x),permq(x)) -> permq(concat_vectors(x,x))
56254 if (Op0.getOpcode() == X86ISD::VPERMI && Subtarget.useAVX512Regs() &&
56255 !X86::mayFoldLoad(Op0.getOperand(0), Subtarget))
56256 return DAG.getNode(Op0.getOpcode(), DL, VT,
56257 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT,
56258 Op0.getOperand(0), Op0.getOperand(0)),
56259 Op0.getOperand(1));
56260 }
56261
56262 // concat(extract_subvector(v0,c0), extract_subvector(v1,c1)) -> vperm2x128.
56263 // Only concat of subvector high halves which vperm2x128 is best at.
56264 // TODO: This should go in combineX86ShufflesRecursively eventually.
56265 if (VT.is256BitVector() && NumOps == 2) {
56266 SDValue Src0 = peekThroughBitcasts(Ops[0]);
56267 SDValue Src1 = peekThroughBitcasts(Ops[1]);
56268 if (Src0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
56269 Src1.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
56270 EVT SrcVT0 = Src0.getOperand(0).getValueType();
56271 EVT SrcVT1 = Src1.getOperand(0).getValueType();
56272 unsigned NumSrcElts0 = SrcVT0.getVectorNumElements();
56273 unsigned NumSrcElts1 = SrcVT1.getVectorNumElements();
56274 if (SrcVT0.is256BitVector() && SrcVT1.is256BitVector() &&
56275 Src0.getConstantOperandAPInt(1) == (NumSrcElts0 / 2) &&
56276 Src1.getConstantOperandAPInt(1) == (NumSrcElts1 / 2)) {
56277 return DAG.getNode(X86ISD::VPERM2X128, DL, VT,
56278 DAG.getBitcast(VT, Src0.getOperand(0)),
56279 DAG.getBitcast(VT, Src1.getOperand(0)),
56280 DAG.getTargetConstant(0x31, DL, MVT::i8));
56281 }
56282 }
56283 }
56284
56285 // Repeated opcode.
56286 // TODO - combineX86ShufflesRecursively should handle shuffle concatenation
56287 // but it currently struggles with different vector widths.
56288 if (llvm::all_of(Ops, [Op0](SDValue Op) {
56289 return Op.getOpcode() == Op0.getOpcode() && Op.hasOneUse();
56290 })) {
56291 auto ConcatSubOperand = [&](EVT VT, ArrayRef<SDValue> SubOps, unsigned I) {
56292 SmallVector<SDValue> Subs;
56293 for (SDValue SubOp : SubOps)
56294 Subs.push_back(SubOp.getOperand(I));
56295 // Attempt to peek through bitcasts and concat the original subvectors.
56296 EVT SubVT = peekThroughBitcasts(Subs[0]).getValueType();
56297 if (SubVT.isSimple() && SubVT.isVector()) {
56298 EVT ConcatVT =
56299 EVT::getVectorVT(*DAG.getContext(), SubVT.getScalarType(),
56300 SubVT.getVectorElementCount() * Subs.size());
56301 for (SDValue &Sub : Subs)
56302 Sub = DAG.getBitcast(SubVT, Sub);
56303 return DAG.getBitcast(
56304 VT, DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Subs));
56305 }
56306 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
56307 };
56308 auto IsConcatFree = [](MVT VT, ArrayRef<SDValue> SubOps, unsigned Op) {
56309 bool AllConstants = true;
56310 bool AllSubs = true;
56311 unsigned VecSize = VT.getSizeInBits();
56312 for (unsigned I = 0, E = SubOps.size(); I != E; ++I) {
56313 SDValue BC = peekThroughBitcasts(SubOps[I].getOperand(Op));
56314 unsigned SubSize = BC.getValueSizeInBits();
56315 unsigned EltSize = BC.getScalarValueSizeInBits();
56316 AllConstants &= ISD::isBuildVectorOfConstantSDNodes(BC.getNode()) ||
56317 ISD::isBuildVectorOfConstantFPSDNodes(BC.getNode());
56318 AllSubs &= BC.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
56319 BC.getOperand(0).getValueSizeInBits() == VecSize &&
56320 (BC.getConstantOperandVal(1) * EltSize) == (I * SubSize);
56321 }
56322 return AllConstants || AllSubs;
56323 };
56324
56325 switch (Op0.getOpcode()) {
56326 case X86ISD::VBROADCAST: {
56327 if (!IsSplat && llvm::all_of(Ops, [](SDValue Op) {
56328 return Op.getOperand(0).getValueType().is128BitVector();
56329 })) {
56330 if (VT == MVT::v4f64 || VT == MVT::v4i64)
56331 return DAG.getNode(X86ISD::UNPCKL, DL, VT,
56332 ConcatSubOperand(VT, Ops, 0),
56333 ConcatSubOperand(VT, Ops, 0));
56334 // TODO: Add pseudo v8i32 PSHUFD handling to AVX1Only targets.
56335 if (VT == MVT::v8f32 || (VT == MVT::v8i32 && Subtarget.hasInt256()))
56336 return DAG.getNode(VT == MVT::v8f32 ? X86ISD::VPERMILPI
56337 : X86ISD::PSHUFD,
56338 DL, VT, ConcatSubOperand(VT, Ops, 0),
56339 getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG));
56340 }
56341 break;
56342 }
56343 case X86ISD::MOVDDUP:
56344 case X86ISD::MOVSHDUP:
56345 case X86ISD::MOVSLDUP: {
56346 if (!IsSplat)
56347 return DAG.getNode(Op0.getOpcode(), DL, VT,
56348 ConcatSubOperand(VT, Ops, 0));
56349 break;
56350 }
56351 case X86ISD::SHUFP: {
56352 // Add SHUFPD support if/when necessary.
56353 if (!IsSplat && VT.getScalarType() == MVT::f32 &&
56354 llvm::all_of(Ops, [Op0](SDValue Op) {
56355 return Op.getOperand(2) == Op0.getOperand(2);
56356 })) {
56357 return DAG.getNode(Op0.getOpcode(), DL, VT,
56358 ConcatSubOperand(VT, Ops, 0),
56359 ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
56360 }
56361 break;
56362 }
56363 case X86ISD::UNPCKH:
56364 case X86ISD::UNPCKL: {
56365 // Don't concatenate build_vector patterns.
56366 if (!IsSplat && EltSizeInBits >= 32 &&
56367 ((VT.is256BitVector() && Subtarget.hasInt256()) ||
56368 (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
56369 none_of(Ops, [](SDValue Op) {
56370 return peekThroughBitcasts(Op.getOperand(0)).getOpcode() ==
56371 ISD::SCALAR_TO_VECTOR ||
56372 peekThroughBitcasts(Op.getOperand(1)).getOpcode() ==
56373 ISD::SCALAR_TO_VECTOR;
56374 })) {
56375 return DAG.getNode(Op0.getOpcode(), DL, VT,
56376 ConcatSubOperand(VT, Ops, 0),
56377 ConcatSubOperand(VT, Ops, 1));
56378 }
56379 break;
56380 }
56381 case X86ISD::PSHUFHW:
56382 case X86ISD::PSHUFLW:
56383 case X86ISD::PSHUFD:
56384 if (!IsSplat && NumOps == 2 && VT.is256BitVector() &&
56385 Subtarget.hasInt256() && Op0.getOperand(1) == Ops[1].getOperand(1)) {
56386 return DAG.getNode(Op0.getOpcode(), DL, VT,
56387 ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
56388 }
56389 [[fallthrough]];
56390 case X86ISD::VPERMILPI:
56391 if (!IsSplat && EltSizeInBits == 32 &&
56392 (VT.is256BitVector() ||
56393 (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
56394 all_of(Ops, [&Op0](SDValue Op) {
56395 return Op0.getOperand(1) == Op.getOperand(1);
56396 })) {
56397 MVT FloatVT = VT.changeVectorElementType(MVT::f32);
56398 SDValue Res = DAG.getBitcast(FloatVT, ConcatSubOperand(VT, Ops, 0));
56399 Res =
56400 DAG.getNode(X86ISD::VPERMILPI, DL, FloatVT, Res, Op0.getOperand(1));
56401 return DAG.getBitcast(VT, Res);
56402 }
56403 if (!IsSplat && NumOps == 2 && VT == MVT::v4f64) {
56404 uint64_t Idx0 = Ops[0].getConstantOperandVal(1);
56405 uint64_t Idx1 = Ops[1].getConstantOperandVal(1);
56406 uint64_t Idx = ((Idx1 & 3) << 2) | (Idx0 & 3);
56407 return DAG.getNode(Op0.getOpcode(), DL, VT,
56408 ConcatSubOperand(VT, Ops, 0),
56409 DAG.getTargetConstant(Idx, DL, MVT::i8));
56410 }
56411 break;
56412 case X86ISD::PSHUFB:
56413 case X86ISD::PSADBW:
56414 case X86ISD::VPMADDUBSW:
56415 case X86ISD::VPMADDWD:
56416 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
56417 (VT.is512BitVector() && Subtarget.useBWIRegs()))) {
56418 MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
56419 SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
56420 NumOps * SrcVT.getVectorNumElements());
56421 return DAG.getNode(Op0.getOpcode(), DL, VT,
56422 ConcatSubOperand(SrcVT, Ops, 0),
56423 ConcatSubOperand(SrcVT, Ops, 1));
56424 }
56425 break;
56426 case X86ISD::VPERMV:
56427 if (!IsSplat && NumOps == 2 &&
56428 (VT.is512BitVector() && Subtarget.useAVX512Regs())) {
56429 MVT OpVT = Op0.getSimpleValueType();
56430 int NumSrcElts = OpVT.getVectorNumElements();
56431 SmallVector<int, 64> ConcatMask;
56432 for (unsigned i = 0; i != NumOps; ++i) {
56433 SmallVector<int, 64> SubMask;
56434 SmallVector<SDValue, 2> SubOps;
56435 if (!getTargetShuffleMask(Ops[i], false, SubOps, SubMask))
56436 break;
56437 for (int M : SubMask) {
56438 if (0 <= M)
56439 M += i * NumSrcElts;
56440 ConcatMask.push_back(M);
56441 }
56442 }
56443 if (ConcatMask.size() == (NumOps * NumSrcElts)) {
56444 SDValue Src = concatSubVectors(Ops[0].getOperand(1),
56445 Ops[1].getOperand(1), DAG, DL);
56446 MVT IntMaskSVT = MVT::getIntegerVT(EltSizeInBits);
56447 MVT IntMaskVT = MVT::getVectorVT(IntMaskSVT, NumOps * NumSrcElts);
56448 SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
56449 return DAG.getNode(X86ISD::VPERMV, DL, VT, Mask, Src);
56450 }
56451 }
56452 break;
56453 case X86ISD::VPERMV3:
56454 if (!IsSplat && NumOps == 2 && VT.is512BitVector()) {
56455 MVT OpVT = Op0.getSimpleValueType();
56456 int NumSrcElts = OpVT.getVectorNumElements();
56457 SmallVector<int, 64> ConcatMask;
56458 for (unsigned i = 0; i != NumOps; ++i) {
56459 SmallVector<int, 64> SubMask;
56460 SmallVector<SDValue, 2> SubOps;
56461 if (!getTargetShuffleMask(Ops[i], false, SubOps, SubMask))
56462 break;
56463 for (int M : SubMask) {
56464 if (0 <= M) {
56465 int Src = M < NumSrcElts ? 0 : 2;
56466 M += M < NumSrcElts ? 0 : NumSrcElts;
56467
56468 // Reference the lowest sub if they upper sub is the same.
56469 if (Ops[0].getOperand(Src) != Ops[i].getOperand(Src))
56470 M += i * NumSrcElts;
56471 }
56472 ConcatMask.push_back(M);
56473 }
56474 }
56475 if (ConcatMask.size() == (NumOps * NumSrcElts)) {
56476 SDValue Src0 = concatSubVectors(Ops[0].getOperand(0),
56477 Ops[1].getOperand(0), DAG, DL);
56478 SDValue Src1 = concatSubVectors(Ops[0].getOperand(2),
56479 Ops[1].getOperand(2), DAG, DL);
56480 MVT IntMaskSVT = MVT::getIntegerVT(EltSizeInBits);
56481 MVT IntMaskVT = MVT::getVectorVT(IntMaskSVT, NumOps * NumSrcElts);
56482 SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
56483 return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1);
56484 }
56485 }
56486 break;
56487 case X86ISD::VPERM2X128: {
56488 if (!IsSplat && VT.is512BitVector() && Subtarget.useAVX512Regs()) {
56489 assert(NumOps == 2 && "Bad concat_vectors operands");
56490 unsigned Imm0 = Ops[0].getConstantOperandVal(2);
56491 unsigned Imm1 = Ops[1].getConstantOperandVal(2);
56492 // TODO: Handle zero'd subvectors.
56493 if ((Imm0 & 0x88) == 0 && (Imm1 & 0x88) == 0) {
56494 int Mask[4] = {(int)(Imm0 & 0x03), (int)((Imm0 >> 4) & 0x3), (int)(Imm1 & 0x03),
56495 (int)((Imm1 >> 4) & 0x3)};
56496 MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64;
56497 SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
56498 Ops[0].getOperand(1), DAG, DL);
56499 SDValue RHS = concatSubVectors(Ops[1].getOperand(0),
56500 Ops[1].getOperand(1), DAG, DL);
56501 SDValue Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT,
56502 DAG.getBitcast(ShuffleVT, LHS),
56503 DAG.getBitcast(ShuffleVT, RHS),
56504 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
56505 return DAG.getBitcast(VT, Res);
56506 }
56507 }
56508 break;
56509 }
56510 case X86ISD::SHUF128: {
56511 if (!IsSplat && NumOps == 2 && VT.is512BitVector()) {
56512 unsigned Imm0 = Ops[0].getConstantOperandVal(2);
56513 unsigned Imm1 = Ops[1].getConstantOperandVal(2);
56514 unsigned Imm = ((Imm0 & 1) << 0) | ((Imm0 & 2) << 1) | 0x08 |
56515 ((Imm1 & 1) << 4) | ((Imm1 & 2) << 5) | 0x80;
56516 SDValue LHS = concatSubVectors(Ops[0].getOperand(0),
56517 Ops[0].getOperand(1), DAG, DL);
56518 SDValue RHS = concatSubVectors(Ops[1].getOperand(0),
56519 Ops[1].getOperand(1), DAG, DL);
56520 return DAG.getNode(X86ISD::SHUF128, DL, VT, LHS, RHS,
56521 DAG.getTargetConstant(Imm, DL, MVT::i8));
56522 }
56523 break;
56524 }
56525 case ISD::TRUNCATE:
56526 if (!IsSplat && NumOps == 2 && VT.is256BitVector()) {
56527 EVT SrcVT = Ops[0].getOperand(0).getValueType();
56528 if (SrcVT.is256BitVector() && SrcVT.isSimple() &&
56529 SrcVT == Ops[1].getOperand(0).getValueType() &&
56530 Subtarget.useAVX512Regs() &&
56531 Subtarget.getPreferVectorWidth() >= 512 &&
56532 (SrcVT.getScalarSizeInBits() > 16 || Subtarget.useBWIRegs())) {
56533 EVT NewSrcVT = SrcVT.getDoubleNumVectorElementsVT(Ctx);
56534 return DAG.getNode(ISD::TRUNCATE, DL, VT,
56535 ConcatSubOperand(NewSrcVT, Ops, 0));
56536 }
56537 }
56538 break;
56539 case X86ISD::VSHLI:
56540 case X86ISD::VSRLI:
56541 // Special case: SHL/SRL AVX1 V4i64 by 32-bits can lower as a shuffle.
56542 // TODO: Move this to LowerShiftByScalarImmediate?
56543 if (VT == MVT::v4i64 && !Subtarget.hasInt256() &&
56544 llvm::all_of(Ops, [](SDValue Op) {
56545 return Op.getConstantOperandAPInt(1) == 32;
56546 })) {
56547 SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0));
56548 SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL);
56549 if (Op0.getOpcode() == X86ISD::VSHLI) {
56550 Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
56551 {8, 0, 8, 2, 8, 4, 8, 6});
56552 } else {
56553 Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
56554 {1, 8, 3, 8, 5, 8, 7, 8});
56555 }
56556 return DAG.getBitcast(VT, Res);
56557 }
56558 [[fallthrough]];
56559 case X86ISD::VSRAI:
56560 case X86ISD::VSHL:
56561 case X86ISD::VSRL:
56562 case X86ISD::VSRA:
56563 if (((VT.is256BitVector() && Subtarget.hasInt256()) ||
56564 (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
56565 (EltSizeInBits >= 32 || Subtarget.useBWIRegs()))) &&
56566 llvm::all_of(Ops, [Op0](SDValue Op) {
56567 return Op0.getOperand(1) == Op.getOperand(1);
56568 })) {
56569 return DAG.getNode(Op0.getOpcode(), DL, VT,
56570 ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
56571 }
56572 break;
56573 case X86ISD::VPERMI:
56574 case X86ISD::VROTLI:
56575 case X86ISD::VROTRI:
56576 if (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
56577 llvm::all_of(Ops, [Op0](SDValue Op) {
56578 return Op0.getOperand(1) == Op.getOperand(1);
56579 })) {
56580 return DAG.getNode(Op0.getOpcode(), DL, VT,
56581 ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1));
56582 }
56583 break;
56584 case ISD::AND:
56585 case ISD::OR:
56586 case ISD::XOR:
56587 case X86ISD::ANDNP:
56588 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
56589 (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
56590 return DAG.getNode(Op0.getOpcode(), DL, VT,
56591 ConcatSubOperand(VT, Ops, 0),
56592 ConcatSubOperand(VT, Ops, 1));
56593 }
56594 break;
56595 case X86ISD::PCMPEQ:
56596 case X86ISD::PCMPGT:
56597 if (!IsSplat && VT.is256BitVector() &&
56598 (Subtarget.hasInt256() || VT == MVT::v8i32) &&
56599 (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1))) {
56600 if (Subtarget.hasInt256())
56601 return DAG.getNode(Op0.getOpcode(), DL, VT,
56602 ConcatSubOperand(VT, Ops, 0),
56603 ConcatSubOperand(VT, Ops, 1));
56604
56605 // Without AVX2, see if we can cast the values to v8f32 and use fcmp.
56606 // TODO: Handle v4f64 as well?
56607 unsigned MaxSigBitsLHS = 0, MaxSigBitsRHS = 0;
56608 for (unsigned I = 0; I != NumOps; ++I) {
56609 MaxSigBitsLHS =
56610 std::max(MaxSigBitsLHS,
56611 DAG.ComputeMaxSignificantBits(Ops[I].getOperand(0)));
56612 MaxSigBitsRHS =
56613 std::max(MaxSigBitsRHS,
56614 DAG.ComputeMaxSignificantBits(Ops[I].getOperand(1)));
56615 if (MaxSigBitsLHS == EltSizeInBits && MaxSigBitsRHS == EltSizeInBits)
56616 break;
56617 }
56618
56619 ISD::CondCode ICC =
56620 Op0.getOpcode() == X86ISD::PCMPEQ ? ISD::SETEQ : ISD::SETGT;
56621 ISD::CondCode FCC =
56622 Op0.getOpcode() == X86ISD::PCMPEQ ? ISD::SETOEQ : ISD::SETOGT;
56623
56624 MVT FpSVT = MVT::getFloatingPointVT(EltSizeInBits);
56625 MVT FpVT = VT.changeVectorElementType(FpSVT);
56626
56627 if (std::optional<unsigned> CastOpc =
56628 CastIntSETCCtoFP(FpVT, ICC, MaxSigBitsLHS, MaxSigBitsRHS)) {
56629 SDValue LHS = ConcatSubOperand(VT, Ops, 0);
56630 SDValue RHS = ConcatSubOperand(VT, Ops, 1);
56631 LHS = DAG.getNode(*CastOpc, DL, FpVT, LHS);
56632 RHS = DAG.getNode(*CastOpc, DL, FpVT, RHS);
56633
56634 bool IsAlwaysSignaling;
56635 unsigned FSETCC =
56636 translateX86FSETCC(FCC, LHS, RHS, IsAlwaysSignaling);
56637 return DAG.getBitcast(
56638 VT, DAG.getNode(X86ISD::CMPP, DL, FpVT, LHS, RHS,
56639 DAG.getTargetConstant(FSETCC, DL, MVT::i8)));
56640 }
56641 }
56642 break;
56643 case ISD::CTPOP:
56644 case ISD::CTTZ:
56645 case ISD::CTLZ:
56646 case ISD::CTTZ_ZERO_UNDEF:
56647 case ISD::CTLZ_ZERO_UNDEF:
56648 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
56649 (VT.is512BitVector() && Subtarget.useBWIRegs()))) {
56650 return DAG.getNode(Op0.getOpcode(), DL, VT,
56651 ConcatSubOperand(VT, Ops, 0));
56652 }
56653 break;
56654 case X86ISD::GF2P8AFFINEQB:
56655 if (!IsSplat &&
56656 (VT.is256BitVector() ||
56657 (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
56658 llvm::all_of(Ops, [Op0](SDValue Op) {
56659 return Op0.getOperand(2) == Op.getOperand(2);
56660 })) {
56661 return DAG.getNode(Op0.getOpcode(), DL, VT,
56662 ConcatSubOperand(VT, Ops, 0),
56663 ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
56664 }
56665 break;
56666 case ISD::ADD:
56667 case ISD::SUB:
56668 case ISD::MUL:
56669 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
56670 (VT.is512BitVector() && Subtarget.useAVX512Regs() &&
56671 (EltSizeInBits >= 32 || Subtarget.useBWIRegs())))) {
56672 return DAG.getNode(Op0.getOpcode(), DL, VT,
56673 ConcatSubOperand(VT, Ops, 0),
56674 ConcatSubOperand(VT, Ops, 1));
56675 }
56676 break;
56677 // Due to VADD, VSUB, VMUL can executed on more ports than VINSERT and
56678 // their latency are short, so here we don't replace them unless we won't
56679 // introduce extra VINSERT.
56680 case ISD::FADD:
56681 case ISD::FSUB:
56682 case ISD::FMUL:
56683 if (!IsSplat && (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1)) &&
56684 (VT.is256BitVector() ||
56685 (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
56686 return DAG.getNode(Op0.getOpcode(), DL, VT,
56687 ConcatSubOperand(VT, Ops, 0),
56688 ConcatSubOperand(VT, Ops, 1));
56689 }
56690 break;
56691 case ISD::FDIV:
56692 if (!IsSplat && (VT.is256BitVector() ||
56693 (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
56694 return DAG.getNode(Op0.getOpcode(), DL, VT,
56695 ConcatSubOperand(VT, Ops, 0),
56696 ConcatSubOperand(VT, Ops, 1));
56697 }
56698 break;
56699 case X86ISD::HADD:
56700 case X86ISD::HSUB:
56701 case X86ISD::FHADD:
56702 case X86ISD::FHSUB:
56703 if (!IsSplat && VT.is256BitVector() &&
56704 (VT.isFloatingPoint() || Subtarget.hasInt256())) {
56705 return DAG.getNode(Op0.getOpcode(), DL, VT,
56706 ConcatSubOperand(VT, Ops, 0),
56707 ConcatSubOperand(VT, Ops, 1));
56708 }
56709 break;
56710 case X86ISD::PACKSS:
56711 case X86ISD::PACKUS:
56712 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) ||
56713 (VT.is512BitVector() && Subtarget.useBWIRegs()))) {
56714 MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
56715 SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
56716 NumOps * SrcVT.getVectorNumElements());
56717 return DAG.getNode(Op0.getOpcode(), DL, VT,
56718 ConcatSubOperand(SrcVT, Ops, 0),
56719 ConcatSubOperand(SrcVT, Ops, 1));
56720 }
56721 break;
56722 case X86ISD::PALIGNR:
56723 if (!IsSplat &&
56724 ((VT.is256BitVector() && Subtarget.hasInt256()) ||
56725 (VT.is512BitVector() && Subtarget.useBWIRegs())) &&
56726 llvm::all_of(Ops, [Op0](SDValue Op) {
56727 return Op0.getOperand(2) == Op.getOperand(2);
56728 })) {
56729 return DAG.getNode(Op0.getOpcode(), DL, VT,
56730 ConcatSubOperand(VT, Ops, 0),
56731 ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2));
56732 }
56733 break;
56734 case X86ISD::BLENDI:
56735 if (NumOps == 2 && VT.is512BitVector() && Subtarget.useBWIRegs()) {
56736 uint64_t Mask0 = Ops[0].getConstantOperandVal(2);
56737 uint64_t Mask1 = Ops[1].getConstantOperandVal(2);
56738 // MVT::v16i16 has repeated blend mask.
56739 if (Op0.getSimpleValueType() == MVT::v16i16) {
56740 Mask0 = (Mask0 << 8) | Mask0;
56741 Mask1 = (Mask1 << 8) | Mask1;
56742 }
56743 uint64_t Mask = (Mask1 << (VT.getVectorNumElements() / 2)) | Mask0;
56744 MVT MaskSVT = MVT::getIntegerVT(VT.getVectorNumElements());
56745 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
56746 SDValue Sel =
56747 DAG.getBitcast(MaskVT, DAG.getConstant(Mask, DL, MaskSVT));
56748 return DAG.getSelect(DL, VT, Sel, ConcatSubOperand(VT, Ops, 1),
56749 ConcatSubOperand(VT, Ops, 0));
56750 }
56751 break;
56752 case ISD::VSELECT:
56753 if (!IsSplat && Subtarget.hasAVX512() &&
56754 (VT.is256BitVector() ||
56755 (VT.is512BitVector() && Subtarget.useAVX512Regs())) &&
56756 (EltSizeInBits >= 32 || Subtarget.hasBWI())) {
56757 EVT SelVT = Ops[0].getOperand(0).getValueType();
56758 if (SelVT.getVectorElementType() == MVT::i1) {
56759 SelVT = EVT::getVectorVT(Ctx, MVT::i1,
56760 NumOps * SelVT.getVectorNumElements());
56761 if (TLI.isTypeLegal(SelVT))
56762 return DAG.getNode(Op0.getOpcode(), DL, VT,
56763 ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0),
56764 ConcatSubOperand(VT, Ops, 1),
56765 ConcatSubOperand(VT, Ops, 2));
56766 }
56767 }
56768 [[fallthrough]];
56769 case X86ISD::BLENDV:
56770 if (!IsSplat && VT.is256BitVector() && NumOps == 2 &&
56771 (EltSizeInBits >= 32 || Subtarget.hasInt256()) &&
56772 IsConcatFree(VT, Ops, 1) && IsConcatFree(VT, Ops, 2)) {
56773 EVT SelVT = Ops[0].getOperand(0).getValueType();
56774 SelVT = SelVT.getDoubleNumVectorElementsVT(Ctx);
56775 if (TLI.isTypeLegal(SelVT))
56776 return DAG.getNode(Op0.getOpcode(), DL, VT,
56777 ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0),
56778 ConcatSubOperand(VT, Ops, 1),
56779 ConcatSubOperand(VT, Ops, 2));
56780 }
56781 break;
56782 }
56783 }
56784
56785 // Fold subvector loads into one.
56786 // If needed, look through bitcasts to get to the load.
56787 if (auto *FirstLd = dyn_cast<LoadSDNode>(peekThroughBitcasts(Op0))) {
56788 unsigned Fast;
56789 const X86TargetLowering *TLI = Subtarget.getTargetLowering();
56790 if (TLI->allowsMemoryAccess(Ctx, DAG.getDataLayout(), VT,
56791 *FirstLd->getMemOperand(), &Fast) &&
56792 Fast) {
56793 if (SDValue Ld =
56794 EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false))
56795 return Ld;
56796 }
56797 }
56798
56799 // Attempt to fold target constant loads.
56800 if (all_of(Ops, [](SDValue Op) { return getTargetConstantFromNode(Op); })) {
56801 SmallVector<APInt> EltBits;
56802 APInt UndefElts = APInt::getZero(VT.getVectorNumElements());
56803 for (unsigned I = 0; I != NumOps; ++I) {
56804 APInt OpUndefElts;
56805 SmallVector<APInt> OpEltBits;
56806 if (!getTargetConstantBitsFromNode(Ops[I], EltSizeInBits, OpUndefElts,
56807 OpEltBits, /*AllowWholeUndefs*/ true,
56808 /*AllowPartialUndefs*/ false))
56809 break;
56810 EltBits.append(OpEltBits);
56811 UndefElts.insertBits(OpUndefElts, I * OpUndefElts.getBitWidth());
56812 }
56813 if (EltBits.size() == VT.getVectorNumElements()) {
56814 Constant *C = getConstantVector(VT, EltBits, UndefElts, Ctx);
56815 MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
56816 SDValue CV = DAG.getConstantPool(C, PVT);
56817 MachineFunction &MF = DAG.getMachineFunction();
56818 MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(MF);
56819 SDValue Ld = DAG.getLoad(VT, DL, DAG.getEntryNode(), CV, MPI);
56820 SDValue Sub = extractSubVector(Ld, 0, DAG, DL, Op0.getValueSizeInBits());
56821 DAG.ReplaceAllUsesOfValueWith(Op0, Sub);
56822 return Ld;
56823 }
56824 }
56825
56826 // If this simple subvector or scalar/subvector broadcast_load is inserted
56827 // into both halves, use a larger broadcast_load. Update other uses to use
56828 // an extracted subvector.
56829 if (IsSplat &&
56830 (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) {
56831 if (ISD::isNormalLoad(Op0.getNode()) ||
56832 Op0.getOpcode() == X86ISD::VBROADCAST_LOAD ||
56833 Op0.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) {
56834 auto *Mem = cast<MemSDNode>(Op0);
56835 unsigned Opc = Op0.getOpcode() == X86ISD::VBROADCAST_LOAD
56836 ? X86ISD::VBROADCAST_LOAD
56837 : X86ISD::SUBV_BROADCAST_LOAD;
56838 if (SDValue BcastLd =
56839 getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) {
56840 SDValue BcastSrc =
56841 extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits());
56842 DAG.ReplaceAllUsesOfValueWith(Op0, BcastSrc);
56843 return BcastLd;
56844 }
56845 }
56846 }
56847
56848 // If we're splatting a 128-bit subvector to 512-bits, use SHUF128 directly.
56849 if (IsSplat && NumOps == 4 && VT.is512BitVector() &&
56850 Subtarget.useAVX512Regs()) {
56851 MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64;
56852 SDValue Res = widenSubVector(Op0, false, Subtarget, DAG, DL, 512);
56853 Res = DAG.getBitcast(ShuffleVT, Res);
56854 Res = DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, Res, Res,
56855 getV4X86ShuffleImm8ForMask({0, 0, 0, 0}, DL, DAG));
56856 return DAG.getBitcast(VT, Res);
56857 }
56858
56859 return SDValue();
56860}
56861
56862static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG,
56863 TargetLowering::DAGCombinerInfo &DCI,
56864 const X86Subtarget &Subtarget) {
56865 EVT VT = N->getValueType(0);
56866 EVT SrcVT = N->getOperand(0).getValueType();
56867 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
56868 SmallVector<SDValue, 4> Ops(N->ops());
56869
56870 if (VT.getVectorElementType() == MVT::i1) {
56871 // Attempt to constant fold.
56872 unsigned SubSizeInBits = SrcVT.getSizeInBits();
56873 APInt Constant = APInt::getZero(VT.getSizeInBits());
56874 for (unsigned I = 0, E = Ops.size(); I != E; ++I) {
56875 auto *C = dyn_cast<ConstantSDNode>(peekThroughBitcasts(Ops[I]));
56876 if (!C) break;
56877 Constant.insertBits(C->getAPIntValue(), I * SubSizeInBits);
56878 if (I == (E - 1)) {
56879 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
56880 if (TLI.isTypeLegal(IntVT))
56881 return DAG.getBitcast(VT, DAG.getConstant(Constant, SDLoc(N), IntVT));
56882 }
56883 }
56884
56885 // Don't do anything else for i1 vectors.
56886 return SDValue();
56887 }
56888
56889 if (Subtarget.hasAVX() && TLI.isTypeLegal(VT) && TLI.isTypeLegal(SrcVT)) {
56890 if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG,
56891 DCI, Subtarget))
56892 return R;
56893 }
56894
56895 return SDValue();
56896}
56897
56898static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
56899 TargetLowering::DAGCombinerInfo &DCI,
56900 const X86Subtarget &Subtarget) {
56901 if (DCI.isBeforeLegalizeOps())
56902 return SDValue();
56903
56904 MVT OpVT = N->getSimpleValueType(0);
56905
56906 bool IsI1Vector = OpVT.getVectorElementType() == MVT::i1;
56907
56908 SDLoc dl(N);
56909 SDValue Vec = N->getOperand(0);
56910 SDValue SubVec = N->getOperand(1);
56911
56912 uint64_t IdxVal = N->getConstantOperandVal(2);
56913 MVT SubVecVT = SubVec.getSimpleValueType();
56914
56915 if (Vec.isUndef() && SubVec.isUndef())
56916 return DAG.getUNDEF(OpVT);
56917
56918 // Inserting undefs/zeros into zeros/undefs is a zero vector.
56919 if ((Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())) &&
56920 (SubVec.isUndef() || ISD::isBuildVectorAllZeros(SubVec.getNode())))
56921 return getZeroVector(OpVT, Subtarget, DAG, dl);
56922
56923 if (ISD::isBuildVectorAllZeros(Vec.getNode())) {
56924 // If we're inserting into a zero vector and then into a larger zero vector,
56925 // just insert into the larger zero vector directly.
56926 if (SubVec.getOpcode() == ISD::INSERT_SUBVECTOR &&
56927 ISD::isBuildVectorAllZeros(SubVec.getOperand(0).getNode())) {
56928 uint64_t Idx2Val = SubVec.getConstantOperandVal(2);
56929 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
56930 getZeroVector(OpVT, Subtarget, DAG, dl),
56931 SubVec.getOperand(1),
56932 DAG.getIntPtrConstant(IdxVal + Idx2Val, dl));
56933 }
56934
56935 // If we're inserting into a zero vector and our input was extracted from an
56936 // insert into a zero vector of the same type and the extraction was at
56937 // least as large as the original insertion. Just insert the original
56938 // subvector into a zero vector.
56939 if (SubVec.getOpcode() == ISD::EXTRACT_SUBVECTOR && IdxVal == 0 &&
56940 isNullConstant(SubVec.getOperand(1)) &&
56941 SubVec.getOperand(0).getOpcode() == ISD::INSERT_SUBVECTOR) {
56942 SDValue Ins = SubVec.getOperand(0);
56943 if (isNullConstant(Ins.getOperand(2)) &&
56944 ISD::isBuildVectorAllZeros(Ins.getOperand(0).getNode()) &&
56945 Ins.getOperand(1).getValueSizeInBits().getFixedValue() <=
56946 SubVecVT.getFixedSizeInBits())
56947 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
56948 getZeroVector(OpVT, Subtarget, DAG, dl),
56949 Ins.getOperand(1), N->getOperand(2));
56950 }
56951 }
56952
56953 // Stop here if this is an i1 vector.
56954 if (IsI1Vector)
56955 return SDValue();
56956
56957 // Eliminate an intermediate vector widening:
56958 // insert_subvector X, (insert_subvector undef, Y, 0), Idx -->
56959 // insert_subvector X, Y, Idx
56960 // TODO: This is a more general version of a DAGCombiner fold, can we move it
56961 // there?
56962 if (SubVec.getOpcode() == ISD::INSERT_SUBVECTOR &&
56963 SubVec.getOperand(0).isUndef() && isNullConstant(SubVec.getOperand(2)))
56964 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, Vec,
56965 SubVec.getOperand(1), N->getOperand(2));
56966
56967 // If this is an insert of an extract, combine to a shuffle. Don't do this
56968 // if the insert or extract can be represented with a subregister operation.
56969 if (SubVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
56970 SubVec.getOperand(0).getSimpleValueType() == OpVT &&
56971 (IdxVal != 0 ||
56972 !(Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())))) {
56973 int ExtIdxVal = SubVec.getConstantOperandVal(1);
56974 if (ExtIdxVal != 0) {
56975 int VecNumElts = OpVT.getVectorNumElements();
56976 int SubVecNumElts = SubVecVT.getVectorNumElements();
56977 SmallVector<int, 64> Mask(VecNumElts);
56978 // First create an identity shuffle mask.
56979 for (int i = 0; i != VecNumElts; ++i)
56980 Mask[i] = i;
56981 // Now insert the extracted portion.
56982 for (int i = 0; i != SubVecNumElts; ++i)
56983 Mask[i + IdxVal] = i + ExtIdxVal + VecNumElts;
56984
56985 return DAG.getVectorShuffle(OpVT, dl, Vec, SubVec.getOperand(0), Mask);
56986 }
56987 }
56988
56989 // Match concat_vector style patterns.
56990 SmallVector<SDValue, 2> SubVectorOps;
56991 if (collectConcatOps(N, SubVectorOps, DAG)) {
56992 if (SDValue Fold =
56993 combineConcatVectorOps(dl, OpVT, SubVectorOps, DAG, DCI, Subtarget))
56994 return Fold;
56995
56996 // If we're inserting all zeros into the upper half, change this to
56997 // a concat with zero. We will match this to a move
56998 // with implicit upper bit zeroing during isel.
56999 // We do this here because we don't want combineConcatVectorOps to
57000 // create INSERT_SUBVECTOR from CONCAT_VECTORS.
57001 if (SubVectorOps.size() == 2 &&
57002 ISD::isBuildVectorAllZeros(SubVectorOps[1].getNode()))
57003 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
57004 getZeroVector(OpVT, Subtarget, DAG, dl),
57005 SubVectorOps[0], DAG.getIntPtrConstant(0, dl));
57006
57007 // Attempt to recursively combine to a shuffle.
57008 if (all_of(SubVectorOps, [](SDValue SubOp) {
57009 return isTargetShuffle(SubOp.getOpcode());
57010 })) {
57011 SDValue Op(N, 0);
57012 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
57013 return Res;
57014 }
57015 }
57016
57017 // If this is a broadcast insert into an upper undef, use a larger broadcast.
57018 if (Vec.isUndef() && IdxVal != 0 && SubVec.getOpcode() == X86ISD::VBROADCAST)
57019 return DAG.getNode(X86ISD::VBROADCAST, dl, OpVT, SubVec.getOperand(0));
57020
57021 // If this is a broadcast load inserted into an upper undef, use a larger
57022 // broadcast load.
57023 if (Vec.isUndef() && IdxVal != 0 && SubVec.hasOneUse() &&
57024 SubVec.getOpcode() == X86ISD::VBROADCAST_LOAD) {
57025 auto *MemIntr = cast<MemIntrinsicSDNode>(SubVec);
57026 SDVTList Tys = DAG.getVTList(OpVT, MVT::Other);
57027 SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
57028 SDValue BcastLd =
57029 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
57030 MemIntr->getMemoryVT(),
57031 MemIntr->getMemOperand());
57032 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
57033 return BcastLd;
57034 }
57035
57036 // If we're splatting the lower half subvector of a full vector load into the
57037 // upper half, attempt to create a subvector broadcast.
57038 if (IdxVal == (OpVT.getVectorNumElements() / 2) && SubVec.hasOneUse() &&
57039 Vec.getValueSizeInBits() == (2 * SubVec.getValueSizeInBits())) {
57040 auto *VecLd = dyn_cast<LoadSDNode>(Vec);
57041 auto *SubLd = dyn_cast<LoadSDNode>(SubVec);
57042 if (VecLd && SubLd &&
57043 DAG.areNonVolatileConsecutiveLoads(SubLd, VecLd,
57044 SubVec.getValueSizeInBits() / 8, 0))
57045 return getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, dl, OpVT, SubVecVT,
57046 SubLd, 0, DAG);
57047 }
57048
57049 return SDValue();
57050}
57051
57052/// If we are extracting a subvector of a vector select and the select condition
57053/// is composed of concatenated vectors, try to narrow the select width. This
57054/// is a common pattern for AVX1 integer code because 256-bit selects may be
57055/// legal, but there is almost no integer math/logic available for 256-bit.
57056/// This function should only be called with legal types (otherwise, the calls
57057/// to get simple value types will assert).
57058static SDValue narrowExtractedVectorSelect(SDNode *Ext, const SDLoc &DL,
57059 SelectionDAG &DAG) {
57060 SDValue Sel = Ext->getOperand(0);
57061 if (Sel.getOpcode() != ISD::VSELECT ||
57062 !isFreeToSplitVector(Sel.getOperand(0).getNode(), DAG))
57063 return SDValue();
57064
57065 // Note: We assume simple value types because this should only be called with
57066 // legal operations/types.
57067 // TODO: This can be extended to handle extraction to 256-bits.
57068 MVT VT = Ext->getSimpleValueType(0);
57069 if (!VT.is128BitVector())
57070 return SDValue();
57071
57072 MVT SelCondVT = Sel.getOperand(0).getSimpleValueType();
57073 if (!SelCondVT.is256BitVector() && !SelCondVT.is512BitVector())
57074 return SDValue();
57075
57076 MVT WideVT = Ext->getOperand(0).getSimpleValueType();
57077 MVT SelVT = Sel.getSimpleValueType();
57078 assert((SelVT.is256BitVector() || SelVT.is512BitVector()) &&
57079 "Unexpected vector type with legal operations");
57080
57081 unsigned SelElts = SelVT.getVectorNumElements();
57082 unsigned CastedElts = WideVT.getVectorNumElements();
57083 unsigned ExtIdx = Ext->getConstantOperandVal(1);
57084 if (SelElts % CastedElts == 0) {
57085 // The select has the same or more (narrower) elements than the extract
57086 // operand. The extraction index gets scaled by that factor.
57087 ExtIdx *= (SelElts / CastedElts);
57088 } else if (CastedElts % SelElts == 0) {
57089 // The select has less (wider) elements than the extract operand. Make sure
57090 // that the extraction index can be divided evenly.
57091 unsigned IndexDivisor = CastedElts / SelElts;
57092 if (ExtIdx % IndexDivisor != 0)
57093 return SDValue();
57094 ExtIdx /= IndexDivisor;
57095 } else {
57096 llvm_unreachable("Element count of simple vector types are not divisible?");
57097 }
57098
57099 unsigned NarrowingFactor = WideVT.getSizeInBits() / VT.getSizeInBits();
57100 unsigned NarrowElts = SelElts / NarrowingFactor;
57101 MVT NarrowSelVT = MVT::getVectorVT(SelVT.getVectorElementType(), NarrowElts);
57102 SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL);
57103 SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL);
57104 SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL);
57105 SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF);
57106 return DAG.getBitcast(VT, NarrowSel);
57107}
57108
57109static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
57110 TargetLowering::DAGCombinerInfo &DCI,
57111 const X86Subtarget &Subtarget) {
57112 // For AVX1 only, if we are extracting from a 256-bit and+not (which will
57113 // eventually get combined/lowered into ANDNP) with a concatenated operand,
57114 // split the 'and' into 128-bit ops to avoid the concatenate and extract.
57115 // We let generic combining take over from there to simplify the
57116 // insert/extract and 'not'.
57117 // This pattern emerges during AVX1 legalization. We handle it before lowering
57118 // to avoid complications like splitting constant vector loads.
57119
57120 // Capture the original wide type in the likely case that we need to bitcast
57121 // back to this type.
57122 if (!N->getValueType(0).isSimple())
57123 return SDValue();
57124
57125 MVT VT = N->getSimpleValueType(0);
57126 SDValue InVec = N->getOperand(0);
57127 unsigned IdxVal = N->getConstantOperandVal(1);
57128 SDValue InVecBC = peekThroughBitcasts(InVec);
57129 EVT InVecVT = InVec.getValueType();
57130 unsigned SizeInBits = VT.getSizeInBits();
57131 unsigned InSizeInBits = InVecVT.getSizeInBits();
57132 unsigned NumSubElts = VT.getVectorNumElements();
57133 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
57134 SDLoc DL(N);
57135
57136 if (Subtarget.hasAVX() && !Subtarget.hasAVX2() &&
57137 TLI.isTypeLegal(InVecVT) &&
57138 InSizeInBits == 256 && InVecBC.getOpcode() == ISD::AND) {
57139 auto isConcatenatedNot = [](SDValue V) {
57140 V = peekThroughBitcasts(V);
57141 if (!isBitwiseNot(V))
57142 return false;
57143 SDValue NotOp = V->getOperand(0);
57144 return peekThroughBitcasts(NotOp).getOpcode() == ISD::CONCAT_VECTORS;
57145 };
57146 if (isConcatenatedNot(InVecBC.getOperand(0)) ||
57147 isConcatenatedNot(InVecBC.getOperand(1))) {
57148 // extract (and v4i64 X, (not (concat Y1, Y2))), n -> andnp v2i64 X(n), Y1
57149 SDValue Concat = splitVectorIntBinary(InVecBC, DAG, SDLoc(InVecBC));
57150 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT,
57151 DAG.getBitcast(InVecVT, Concat), N->getOperand(1));
57152 }
57153 }
57154
57155 if (DCI.isBeforeLegalizeOps())
57156 return SDValue();
57157
57158 if (SDValue V = narrowExtractedVectorSelect(N, DL, DAG))
57159 return V;
57160
57161 if (ISD::isBuildVectorAllZeros(InVec.getNode()))
57162 return getZeroVector(VT, Subtarget, DAG, DL);
57163
57164 if (ISD::isBuildVectorAllOnes(InVec.getNode())) {
57165 if (VT.getScalarType() == MVT::i1)
57166 return DAG.getConstant(1, DL, VT);
57167 return getOnesVector(VT, DAG, DL);
57168 }
57169
57170 if (InVec.getOpcode() == ISD::BUILD_VECTOR)
57171 return DAG.getBuildVector(VT, DL, InVec->ops().slice(IdxVal, NumSubElts));
57172
57173 // If we are extracting from an insert into a larger vector, replace with a
57174 // smaller insert if we don't access less than the original subvector. Don't
57175 // do this for i1 vectors.
57176 // TODO: Relax the matching indices requirement?
57177 if (VT.getVectorElementType() != MVT::i1 &&
57178 InVec.getOpcode() == ISD::INSERT_SUBVECTOR && InVec.hasOneUse() &&
57179 IdxVal == InVec.getConstantOperandVal(2) &&
57180 InVec.getOperand(1).getValueSizeInBits() <= SizeInBits) {
57181 SDValue NewExt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT,
57182 InVec.getOperand(0), N->getOperand(1));
57183 unsigned NewIdxVal = InVec.getConstantOperandVal(2) - IdxVal;
57184 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, NewExt,
57185 InVec.getOperand(1),
57186 DAG.getVectorIdxConstant(NewIdxVal, DL));
57187 }
57188
57189 // If we're extracting an upper subvector from a broadcast we should just
57190 // extract the lowest subvector instead which should allow
57191 // SimplifyDemandedVectorElts do more simplifications.
57192 if (IdxVal != 0 && (InVec.getOpcode() == X86ISD::VBROADCAST ||
57193 InVec.getOpcode() == X86ISD::VBROADCAST_LOAD ||
57194 DAG.isSplatValue(InVec, /*AllowUndefs*/ false)))
57195 return extractSubVector(InVec, 0, DAG, DL, SizeInBits);
57196
57197 // If we're extracting a broadcasted subvector, just use the lowest subvector.
57198 if (IdxVal != 0 && InVec.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD &&
57199 cast<MemIntrinsicSDNode>(InVec)->getMemoryVT() == VT)
57200 return extractSubVector(InVec, 0, DAG, DL, SizeInBits);
57201
57202 // Attempt to extract from the source of a shuffle vector.
57203 if ((InSizeInBits % SizeInBits) == 0 && (IdxVal % NumSubElts) == 0) {
57204 SmallVector<int, 32> ShuffleMask;
57205 SmallVector<int, 32> ScaledMask;
57206 SmallVector<SDValue, 2> ShuffleInputs;
57207 unsigned NumSubVecs = InSizeInBits / SizeInBits;
57208 // Decode the shuffle mask and scale it so its shuffling subvectors.
57209 if (getTargetShuffleInputs(InVecBC, ShuffleInputs, ShuffleMask, DAG) &&
57210 scaleShuffleElements(ShuffleMask, NumSubVecs, ScaledMask)) {
57211 unsigned SubVecIdx = IdxVal / NumSubElts;
57212 if (ScaledMask[SubVecIdx] == SM_SentinelUndef)
57213 return DAG.getUNDEF(VT);
57214 if (ScaledMask[SubVecIdx] == SM_SentinelZero)
57215 return getZeroVector(VT, Subtarget, DAG, DL);
57216 SDValue Src = ShuffleInputs[ScaledMask[SubVecIdx] / NumSubVecs];
57217 if (Src.getValueSizeInBits() == InSizeInBits) {
57218 unsigned SrcSubVecIdx = ScaledMask[SubVecIdx] % NumSubVecs;
57219 unsigned SrcEltIdx = SrcSubVecIdx * NumSubElts;
57220 return extractSubVector(DAG.getBitcast(InVecVT, Src), SrcEltIdx, DAG,
57221 DL, SizeInBits);
57222 }
57223 }
57224 }
57225
57226 auto IsExtractFree = [](SDValue V) {
57227 if (V.hasOneUse()) {
57228 V = peekThroughOneUseBitcasts(V);
57229 if (V.getOpcode() == ISD::LOAD)
57230 return true;
57231 }
57232 V = peekThroughBitcasts(V);
57233 if (ISD::isBuildVectorOfConstantSDNodes(V.getNode()))
57234 return true;
57235 if (ISD::isBuildVectorOfConstantFPSDNodes(V.getNode()))
57236 return true;
57237 return V.isUndef();
57238 };
57239
57240 // If we're extracting the lowest subvector and we're the only user,
57241 // we may be able to perform this with a smaller vector width.
57242 unsigned InOpcode = InVec.getOpcode();
57243 if (InVec.hasOneUse()) {
57244 if (IdxVal == 0 && VT == MVT::v2f64 && InVecVT == MVT::v4f64) {
57245 // v2f64 CVTDQ2PD(v4i32).
57246 if (InOpcode == ISD::SINT_TO_FP &&
57247 InVec.getOperand(0).getValueType() == MVT::v4i32) {
57248 return DAG.getNode(X86ISD::CVTSI2P, DL, VT, InVec.getOperand(0));
57249 }
57250 // v2f64 CVTUDQ2PD(v4i32).
57251 if (InOpcode == ISD::UINT_TO_FP && Subtarget.hasVLX() &&
57252 InVec.getOperand(0).getValueType() == MVT::v4i32) {
57253 return DAG.getNode(X86ISD::CVTUI2P, DL, VT, InVec.getOperand(0));
57254 }
57255 // v2f64 CVTPS2PD(v4f32).
57256 if (InOpcode == ISD::FP_EXTEND &&
57257 InVec.getOperand(0).getValueType() == MVT::v4f32) {
57258 return DAG.getNode(X86ISD::VFPEXT, DL, VT, InVec.getOperand(0));
57259 }
57260 }
57261 // v4i32 CVTPS2DQ(v4f32) / CVTPS2UDQ(v4f32).
57262 // v4f32 CVTDQ2PS(v4i32) / CVTUDQ2PS(v4i32).
57263 if ((InOpcode == ISD::FP_TO_SINT || InOpcode == ISD::SINT_TO_FP ||
57264 ((InOpcode == ISD::FP_TO_UINT || InOpcode == ISD::UINT_TO_FP) &&
57265 Subtarget.hasVLX())) &&
57266 (VT == MVT::v4i32 || VT == MVT::v4f32)) {
57267 SDValue Src = InVec.getOperand(0);
57268 if (Src.getValueType().getScalarSizeInBits() == 32)
57269 return DAG.getNode(InOpcode, DL, VT,
57270 extractSubVector(Src, IdxVal, DAG, DL, SizeInBits));
57271 }
57272 if (IdxVal == 0 &&
57273 (ISD::isExtOpcode(InOpcode) || ISD::isExtVecInRegOpcode(InOpcode)) &&
57274 (SizeInBits == 128 || SizeInBits == 256) &&
57275 InVec.getOperand(0).getValueSizeInBits() >= SizeInBits) {
57276 SDValue Ext = InVec.getOperand(0);
57277 if (Ext.getValueSizeInBits() > SizeInBits)
57278 Ext = extractSubVector(Ext, 0, DAG, DL, SizeInBits);
57279 unsigned ExtOp = DAG.getOpcode_EXTEND_VECTOR_INREG(InOpcode);
57280 return DAG.getNode(ExtOp, DL, VT, Ext);
57281 }
57282 if (IdxVal == 0 && InOpcode == ISD::VSELECT &&
57283 InVec.getOperand(0).getValueType().is256BitVector() &&
57284 InVec.getOperand(1).getValueType().is256BitVector() &&
57285 InVec.getOperand(2).getValueType().is256BitVector()) {
57286 SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128);
57287 SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128);
57288 SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
57289 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2);
57290 }
57291 if (IdxVal == 0 && InOpcode == ISD::TRUNCATE && Subtarget.hasVLX() &&
57292 (SizeInBits == 128 || SizeInBits == 256)) {
57293 SDValue InVecSrc = InVec.getOperand(0);
57294 unsigned Scale = InVecSrc.getValueSizeInBits() / InSizeInBits;
57295 SDValue Ext = extractSubVector(InVecSrc, 0, DAG, DL, Scale * SizeInBits);
57296 return DAG.getNode(InOpcode, DL, VT, Ext);
57297 }
57298
57299 if (SizeInBits == 128 || SizeInBits == 256) {
57300 switch (InOpcode) {
57301 case X86ISD::MOVDDUP:
57302 return DAG.getNode(
57303 InOpcode, DL, VT,
57304 extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits));
57305 case X86ISD::PCMPEQ:
57306 case X86ISD::PCMPGT:
57307 case X86ISD::UNPCKH:
57308 case X86ISD::UNPCKL:
57309 if (IsExtractFree(InVec.getOperand(0)) ||
57310 IsExtractFree(InVec.getOperand(1)))
57311 return DAG.getNode(InOpcode, DL, VT,
57312 extractSubVector(InVec.getOperand(0), IdxVal, DAG,
57313 DL, SizeInBits),
57314 extractSubVector(InVec.getOperand(1), IdxVal, DAG,
57315 DL, SizeInBits));
57316 break;
57317 case X86ISD::CMPP:
57318 if (IsExtractFree(InVec.getOperand(0)) ||
57319 IsExtractFree(InVec.getOperand(1)))
57320 return DAG.getNode(InOpcode, DL, VT,
57321 extractSubVector(InVec.getOperand(0), IdxVal, DAG,
57322 DL, SizeInBits),
57323 extractSubVector(InVec.getOperand(1), IdxVal, DAG,
57324 DL, SizeInBits),
57325 InVec.getOperand(2));
57326 break;
57327 case X86ISD::BLENDI:
57328 if (IsExtractFree(InVec.getOperand(0)) ||
57329 IsExtractFree(InVec.getOperand(1))) {
57330 uint64_t M = InVec.getConstantOperandVal(2) & 255;
57331 M = VT.getScalarType() == MVT::i16 ? M : (M >> IdxVal);
57332 return DAG.getNode(InOpcode, DL, VT,
57333 extractSubVector(InVec.getOperand(0), IdxVal, DAG,
57334 DL, SizeInBits),
57335 extractSubVector(InVec.getOperand(1), IdxVal, DAG,
57336 DL, SizeInBits),
57337 DAG.getTargetConstant(M, DL, MVT::i8));
57338 }
57339 break;
57340 }
57341 }
57342 }
57343
57344 // Always split vXi64 logical shifts where we're extracting the upper 32-bits
57345 // as this is very likely to fold into a shuffle/truncation.
57346 if ((InOpcode == X86ISD::VSHLI || InOpcode == X86ISD::VSRLI) &&
57347 InVecVT.getScalarSizeInBits() == 64 &&
57348 InVec.getConstantOperandAPInt(1) == 32) {
57349 SDValue Ext =
57350 extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits);
57351 return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1));
57352 }
57353
57354 return SDValue();
57355}
57356
57357static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
57358 EVT VT = N->getValueType(0);
57359 SDValue Src = N->getOperand(0);
57360 SDLoc DL(N);
57361
57362 // If this is a scalar to vector to v1i1 from an AND with 1, bypass the and.
57363 // This occurs frequently in our masked scalar intrinsic code and our
57364 // floating point select lowering with AVX512.
57365 // TODO: SimplifyDemandedBits instead?
57366 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse() &&
57367 isOneConstant(Src.getOperand(1)))
57368 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Src.getOperand(0));
57369
57370 // Combine scalar_to_vector of an extract_vector_elt into an extract_subvec.
57371 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
57372 Src.hasOneUse() && Src.getOperand(0).getValueType().isVector() &&
57373 Src.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
57374 isNullConstant(Src.getOperand(1)))
57375 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0),
57376 Src.getOperand(1));
57377
57378 // Reduce v2i64 to v4i32 if we don't need the upper bits or are known zero.
57379 // TODO: Move to DAGCombine/SimplifyDemandedBits?
57380 if ((VT == MVT::v2i64 || VT == MVT::v2f64) && Src.hasOneUse()) {
57381 auto IsExt64 = [&DAG](SDValue Op, bool IsZeroExt) {
57382 if (Op.getValueType() != MVT::i64)
57383 return SDValue();
57384 unsigned Opc = IsZeroExt ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND;
57385 if (Op.getOpcode() == Opc &&
57386 Op.getOperand(0).getScalarValueSizeInBits() <= 32)
57387 return Op.getOperand(0);
57388 unsigned Ext = IsZeroExt ? ISD::ZEXTLOAD : ISD::EXTLOAD;
57389 if (auto *Ld = dyn_cast<LoadSDNode>(Op))
57390 if (Ld->getExtensionType() == Ext &&
57391 Ld->getMemoryVT().getScalarSizeInBits() <= 32)
57392 return Op;
57393 if (IsZeroExt) {
57394 KnownBits Known = DAG.computeKnownBits(Op);
57395 if (!Known.isConstant() && Known.countMinLeadingZeros() >= 32)
57396 return Op;
57397 }
57398 return SDValue();
57399 };
57400
57401 if (SDValue AnyExt = IsExt64(peekThroughOneUseBitcasts(Src), false))
57402 return DAG.getBitcast(
57403 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
57404 DAG.getAnyExtOrTrunc(AnyExt, DL, MVT::i32)));
57405
57406 if (SDValue ZeroExt = IsExt64(peekThroughOneUseBitcasts(Src), true))
57407 return DAG.getBitcast(
57408 VT,
57409 DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v4i32,
57410 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
57411 DAG.getZExtOrTrunc(ZeroExt, DL, MVT::i32))));
57412 }
57413
57414 // Combine (v2i64 (scalar_to_vector (i64 (bitconvert (mmx))))) to MOVQ2DQ.
57415 if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST &&
57416 Src.getOperand(0).getValueType() == MVT::x86mmx)
57417 return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0));
57418
57419 // See if we're broadcasting the scalar value, in which case just reuse that.
57420 // Ensure the same SDValue from the SDNode use is being used.
57421 if (VT.getScalarType() == Src.getValueType())
57422 for (SDNode *User : Src->uses())
57423 if (User->getOpcode() == X86ISD::VBROADCAST &&
57424 Src == User->getOperand(0)) {
57425 unsigned SizeInBits = VT.getFixedSizeInBits();
57426 unsigned BroadcastSizeInBits =
57427 User->getValueSizeInBits(0).getFixedValue();
57428 if (BroadcastSizeInBits == SizeInBits)
57429 return SDValue(User, 0);
57430 if (BroadcastSizeInBits > SizeInBits)
57431 return extractSubVector(SDValue(User, 0), 0, DAG, DL, SizeInBits);
57432 // TODO: Handle BroadcastSizeInBits < SizeInBits when we have test
57433 // coverage.
57434 }
57435
57436 return SDValue();
57437}
57438
57439// Simplify PMULDQ and PMULUDQ operations.
57440static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
57441 TargetLowering::DAGCombinerInfo &DCI,
57442 const X86Subtarget &Subtarget) {
57443 SDValue LHS = N->getOperand(0);
57444 SDValue RHS = N->getOperand(1);
57445
57446 // Canonicalize constant to RHS.
57447 if (DAG.isConstantIntBuildVectorOrConstantInt(LHS) &&
57448 !DAG.isConstantIntBuildVectorOrConstantInt(RHS))
57449 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), RHS, LHS);
57450
57451 // Multiply by zero.
57452 // Don't return RHS as it may contain UNDEFs.
57453 if (ISD::isBuildVectorAllZeros(RHS.getNode()))
57454 return DAG.getConstant(0, SDLoc(N), N->getValueType(0));
57455
57456 // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
57457 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
57458 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(64), DCI))
57459 return SDValue(N, 0);
57460
57461 // If the input is an extend_invec and the SimplifyDemandedBits call didn't
57462 // convert it to any_extend_invec, due to the LegalOperations check, do the
57463 // conversion directly to a vector shuffle manually. This exposes combine
57464 // opportunities missed by combineEXTEND_VECTOR_INREG not calling
57465 // combineX86ShufflesRecursively on SSE4.1 targets.
57466 // FIXME: This is basically a hack around several other issues related to
57467 // ANY_EXTEND_VECTOR_INREG.
57468 if (N->getValueType(0) == MVT::v2i64 && LHS.hasOneUse() &&
57469 (LHS.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
57470 LHS.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) &&
57471 LHS.getOperand(0).getValueType() == MVT::v4i32) {
57472 SDLoc dl(N);
57473 LHS = DAG.getVectorShuffle(MVT::v4i32, dl, LHS.getOperand(0),
57474 LHS.getOperand(0), { 0, -1, 1, -1 });
57475 LHS = DAG.getBitcast(MVT::v2i64, LHS);
57476 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
57477 }
57478 if (N->getValueType(0) == MVT::v2i64 && RHS.hasOneUse() &&
57479 (RHS.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
57480 RHS.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) &&
57481 RHS.getOperand(0).getValueType() == MVT::v4i32) {
57482 SDLoc dl(N);
57483 RHS = DAG.getVectorShuffle(MVT::v4i32, dl, RHS.getOperand(0),
57484 RHS.getOperand(0), { 0, -1, 1, -1 });
57485 RHS = DAG.getBitcast(MVT::v2i64, RHS);
57486 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
57487 }
57488
57489 return SDValue();
57490}
57491
57492// Simplify VPMADDUBSW/VPMADDWD operations.
57493static SDValue combineVPMADD(SDNode *N, SelectionDAG &DAG,
57494 TargetLowering::DAGCombinerInfo &DCI) {
57495 MVT VT = N->getSimpleValueType(0);
57496 SDValue LHS = N->getOperand(0);
57497 SDValue RHS = N->getOperand(1);
57498 unsigned Opc = N->getOpcode();
57499 bool IsPMADDWD = Opc == X86ISD::VPMADDWD;
57500 assert((Opc == X86ISD::VPMADDWD || Opc == X86ISD::VPMADDUBSW) &&
57501 "Unexpected PMADD opcode");
57502
57503 // Multiply by zero.
57504 // Don't return LHS/RHS as it may contain UNDEFs.
57505 if (ISD::isBuildVectorAllZeros(LHS.getNode()) ||
57506 ISD::isBuildVectorAllZeros(RHS.getNode()))
57507 return DAG.getConstant(0, SDLoc(N), VT);
57508
57509 // Constant folding.
57510 APInt LHSUndefs, RHSUndefs;
57511 SmallVector<APInt> LHSBits, RHSBits;
57512 unsigned SrcEltBits = LHS.getScalarValueSizeInBits();
57513 unsigned DstEltBits = VT.getScalarSizeInBits();
57514 if (getTargetConstantBitsFromNode(LHS, SrcEltBits, LHSUndefs, LHSBits) &&
57515 getTargetConstantBitsFromNode(RHS, SrcEltBits, RHSUndefs, RHSBits)) {
57516 SmallVector<APInt> Result;
57517 for (unsigned I = 0, E = LHSBits.size(); I != E; I += 2) {
57518 APInt LHSLo = LHSBits[I + 0], LHSHi = LHSBits[I + 1];
57519 APInt RHSLo = RHSBits[I + 0], RHSHi = RHSBits[I + 1];
57520 LHSLo = IsPMADDWD ? LHSLo.sext(DstEltBits) : LHSLo.zext(DstEltBits);
57521 LHSHi = IsPMADDWD ? LHSHi.sext(DstEltBits) : LHSHi.zext(DstEltBits);
57522 APInt Lo = LHSLo * RHSLo.sext(DstEltBits);
57523 APInt Hi = LHSHi * RHSHi.sext(DstEltBits);
57524 APInt Res = IsPMADDWD ? (Lo + Hi) : Lo.sadd_sat(Hi);
57525 Result.push_back(Res);
57526 }
57527 return getConstVector(Result, VT, DAG, SDLoc(N));
57528 }
57529
57530 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
57531 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
57532 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
57533 return SDValue(N, 0);
57534
57535 return SDValue();
57536}
57537
57538static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG,
57539 TargetLowering::DAGCombinerInfo &DCI,
57540 const X86Subtarget &Subtarget) {
57541 EVT VT = N->getValueType(0);
57542 SDValue In = N->getOperand(0);
57543 unsigned Opcode = N->getOpcode();
57544 unsigned InOpcode = In.getOpcode();
57545 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
57546 SDLoc DL(N);
57547
57548 // Try to merge vector loads and extend_inreg to an extload.
57549 if (!DCI.isBeforeLegalizeOps() && ISD::isNormalLoad(In.getNode()) &&
57550 In.hasOneUse()) {
57551 auto *Ld = cast<LoadSDNode>(In);
57552 if (Ld->isSimple()) {
57553 MVT SVT = In.getSimpleValueType().getVectorElementType();
57554 ISD::LoadExtType Ext = Opcode == ISD::SIGN_EXTEND_VECTOR_INREG
57555 ? ISD::SEXTLOAD
57556 : ISD::ZEXTLOAD;
57557 EVT MemVT = VT.changeVectorElementType(SVT);
57558 if (TLI.isLoadExtLegal(Ext, VT, MemVT)) {
57559 SDValue Load = DAG.getExtLoad(
57560 Ext, DL, VT, Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(),
57561 MemVT, Ld->getOriginalAlign(), Ld->getMemOperand()->getFlags());
57562 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
57563 return Load;
57564 }
57565 }
57566 }
57567
57568 // Fold EXTEND_VECTOR_INREG(EXTEND_VECTOR_INREG(X)) -> EXTEND_VECTOR_INREG(X).
57569 if (Opcode == InOpcode)
57570 return DAG.getNode(Opcode, DL, VT, In.getOperand(0));
57571
57572 // Fold EXTEND_VECTOR_INREG(EXTRACT_SUBVECTOR(EXTEND(X),0))
57573 // -> EXTEND_VECTOR_INREG(X).
57574 // TODO: Handle non-zero subvector indices.
57575 if (InOpcode == ISD::EXTRACT_SUBVECTOR && In.getConstantOperandVal(1) == 0 &&
57576 In.getOperand(0).getOpcode() == DAG.getOpcode_EXTEND(Opcode) &&
57577 In.getOperand(0).getOperand(0).getValueSizeInBits() ==
57578 In.getValueSizeInBits())
57579 return DAG.getNode(Opcode, DL, VT, In.getOperand(0).getOperand(0));
57580
57581 // Fold EXTEND_VECTOR_INREG(BUILD_VECTOR(X,Y,?,?)) -> BUILD_VECTOR(X,0,Y,0).
57582 // TODO: Move to DAGCombine?
57583 if (!DCI.isBeforeLegalizeOps() && Opcode == ISD::ZERO_EXTEND_VECTOR_INREG &&
57584 In.getOpcode() == ISD::BUILD_VECTOR && In.hasOneUse() &&
57585 In.getValueSizeInBits() == VT.getSizeInBits()) {
57586 unsigned NumElts = VT.getVectorNumElements();
57587 unsigned Scale = VT.getScalarSizeInBits() / In.getScalarValueSizeInBits();
57588 EVT EltVT = In.getOperand(0).getValueType();
57589 SmallVector<SDValue> Elts(Scale * NumElts, DAG.getConstant(0, DL, EltVT));
57590 for (unsigned I = 0; I != NumElts; ++I)
57591 Elts[I * Scale] = In.getOperand(I);
57592 return DAG.getBitcast(VT, DAG.getBuildVector(In.getValueType(), DL, Elts));
57593 }
57594
57595 // Attempt to combine as a shuffle on SSE41+ targets.
57596 if (Subtarget.hasSSE41()) {
57597 SDValue Op(N, 0);
57598 if (TLI.isTypeLegal(VT) && TLI.isTypeLegal(In.getValueType()))
57599 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
57600 return Res;
57601 }
57602
57603 return SDValue();
57604}
57605
57606static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
57607 TargetLowering::DAGCombinerInfo &DCI) {
57608 EVT VT = N->getValueType(0);
57609
57610 if (ISD::isBuildVectorAllZeros(N->getOperand(0).getNode()))
57611 return DAG.getConstant(0, SDLoc(N), VT);
57612
57613 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
57614 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements());
57615 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, DCI))
57616 return SDValue(N, 0);
57617
57618 return SDValue();
57619}
57620
57621// Optimize (fp16_to_fp (fp_to_fp16 X)) to VCVTPS2PH followed by VCVTPH2PS.
57622// Done as a combine because the lowering for fp16_to_fp and fp_to_fp16 produce
57623// extra instructions between the conversion due to going to scalar and back.
57624static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG,
57625 const X86Subtarget &Subtarget) {
57626 if (Subtarget.useSoftFloat() || !Subtarget.hasF16C())
57627 return SDValue();
57628
57629 if (N->getOperand(0).getOpcode() != ISD::FP_TO_FP16)
57630 return SDValue();
57631
57632 if (N->getValueType(0) != MVT::f32 ||
57633 N->getOperand(0).getOperand(0).getValueType() != MVT::f32)
57634 return SDValue();
57635
57636 SDLoc dl(N);
57637 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32,
57638 N->getOperand(0).getOperand(0));
57639 Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
57640 DAG.getTargetConstant(4, dl, MVT::i32));
57641 Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
57642 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
57643 DAG.getIntPtrConstant(0, dl));
57644}
57645
57646static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
57647 TargetLowering::DAGCombinerInfo &DCI,
57648 const X86Subtarget &Subtarget) {
57649 EVT VT = N->getValueType(0);
57650 bool IsStrict = N->isStrictFPOpcode();
57651 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
57652 EVT SrcVT = Src.getValueType();
57653
57654 SDLoc dl(N);
57655 if (SrcVT.getScalarType() == MVT::bf16) {
57656 if (DCI.isAfterLegalizeDAG() && Src.getOpcode() == ISD::FP_ROUND &&
57657 !IsStrict && Src.getOperand(0).getValueType() == VT)
57658 return Src.getOperand(0);
57659
57660 if (!SrcVT.isVector())
57661 return SDValue();
57662
57663 assert(!IsStrict && "Strict FP doesn't support BF16");
57664 if (VT.getVectorElementType() == MVT::f64) {
57665 EVT TmpVT = VT.changeVectorElementType(MVT::f32);
57666 return DAG.getNode(ISD::FP_EXTEND, dl, VT,
57667 DAG.getNode(ISD::FP_EXTEND, dl, TmpVT, Src));
57668 }
57669 assert(VT.getVectorElementType() == MVT::f32 && "Unexpected fpext");
57670 EVT NVT = SrcVT.changeVectorElementType(MVT::i32);
57671 Src = DAG.getBitcast(SrcVT.changeTypeToInteger(), Src);
57672 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Src);
57673 Src = DAG.getNode(ISD::SHL, dl, NVT, Src, DAG.getConstant(16, dl, NVT));
57674 return DAG.getBitcast(VT, Src);
57675 }
57676
57677 if (!Subtarget.hasF16C() || Subtarget.useSoftFloat())
57678 return SDValue();
57679
57680 if (Subtarget.hasFP16())
57681 return SDValue();
57682
57683 if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::f16)
57684 return SDValue();
57685
57686 if (VT.getVectorElementType() != MVT::f32 &&
57687 VT.getVectorElementType() != MVT::f64)
57688 return SDValue();
57689
57690 unsigned NumElts = VT.getVectorNumElements();
57691 if (NumElts == 1 || !isPowerOf2_32(NumElts))
57692 return SDValue();
57693
57694 // Convert the input to vXi16.
57695 EVT IntVT = SrcVT.changeVectorElementTypeToInteger();
57696 Src = DAG.getBitcast(IntVT, Src);
57697
57698 // Widen to at least 8 input elements.
57699 if (NumElts < 8) {
57700 unsigned NumConcats = 8 / NumElts;
57701 SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT)
57702 : DAG.getConstant(0, dl, IntVT);
57703 SmallVector<SDValue, 4> Ops(NumConcats, Fill);
57704 Ops[0] = Src;
57705 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, Ops);
57706 }
57707
57708 // Destination is vXf32 with at least 4 elements.
57709 EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::f32,
57710 std::max(4U, NumElts));
57711 SDValue Cvt, Chain;
57712 if (IsStrict) {
57713 Cvt = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {CvtVT, MVT::Other},
57714 {N->getOperand(0), Src});
57715 Chain = Cvt.getValue(1);
57716 } else {
57717 Cvt = DAG.getNode(X86ISD::CVTPH2PS, dl, CvtVT, Src);
57718 }
57719
57720 if (NumElts < 4) {
57721 assert(NumElts == 2 && "Unexpected size");
57722 Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Cvt,
57723 DAG.getIntPtrConstant(0, dl));
57724 }
57725
57726 if (IsStrict) {
57727 // Extend to the original VT if necessary.
57728 if (Cvt.getValueType() != VT) {
57729 Cvt = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {VT, MVT::Other},
57730 {Chain, Cvt});
57731 Chain = Cvt.getValue(1);
57732 }
57733 return DAG.getMergeValues({Cvt, Chain}, dl);
57734 }
57735
57736 // Extend to the original VT if necessary.
57737 return DAG.getNode(ISD::FP_EXTEND, dl, VT, Cvt);
57738}
57739
57740// Try to find a larger VBROADCAST_LOAD/SUBV_BROADCAST_LOAD that we can extract
57741// from. Limit this to cases where the loads have the same input chain and the
57742// output chains are unused. This avoids any memory ordering issues.
57743static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG,
57744 TargetLowering::DAGCombinerInfo &DCI) {
57745 assert((N->getOpcode() == X86ISD::VBROADCAST_LOAD ||
57746 N->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) &&
57747 "Unknown broadcast load type");
57748
57749 // Only do this if the chain result is unused.
57750 if (N->hasAnyUseOfValue(1))
57751 return SDValue();
57752
57753 auto *MemIntrin = cast<MemIntrinsicSDNode>(N);
57754
57755 SDValue Ptr = MemIntrin->getBasePtr();
57756 SDValue Chain = MemIntrin->getChain();
57757 EVT VT = N->getSimpleValueType(0);
57758 EVT MemVT = MemIntrin->getMemoryVT();
57759
57760 // Look at other users of our base pointer and try to find a wider broadcast.
57761 // The input chain and the size of the memory VT must match.
57762 for (SDNode *User : Ptr->uses())
57763 if (User != N && User->getOpcode() == N->getOpcode() &&
57764 cast<MemIntrinsicSDNode>(User)->getBasePtr() == Ptr &&
57765 cast<MemIntrinsicSDNode>(User)->getChain() == Chain &&
57766 cast<MemIntrinsicSDNode>(User)->getMemoryVT().getSizeInBits() ==
57767 MemVT.getSizeInBits() &&
57768 !User->hasAnyUseOfValue(1) &&
57769 User->getValueSizeInBits(0).getFixedValue() > VT.getFixedSizeInBits()) {
57770 SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N),
57771 VT.getSizeInBits());
57772 Extract = DAG.getBitcast(VT, Extract);
57773 return DCI.CombineTo(N, Extract, SDValue(User, 1));
57774 }
57775
57776 return SDValue();
57777}
57778
57779static SDValue combineFP_ROUND(SDNode *N, SelectionDAG &DAG,
57780 const X86Subtarget &Subtarget) {
57781 if (!Subtarget.hasF16C() || Subtarget.useSoftFloat())
57782 return SDValue();
57783
57784 bool IsStrict = N->isStrictFPOpcode();
57785 EVT VT = N->getValueType(0);
57786 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
57787 EVT SrcVT = Src.getValueType();
57788
57789 if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
57790 SrcVT.getVectorElementType() != MVT::f32)
57791 return SDValue();
57792
57793 SDLoc dl(N);
57794
57795 SDValue Cvt, Chain;
57796 unsigned NumElts = VT.getVectorNumElements();
57797 if (Subtarget.hasFP16()) {
57798 // Combine (v8f16 fp_round(concat_vectors(v4f32 (xint_to_fp v4i64),
57799 // v4f32 (xint_to_fp v4i64))))
57800 // into (v8f16 vector_shuffle(v8f16 (CVTXI2P v4i64),
57801 // v8f16 (CVTXI2P v4i64)))
57802 if (NumElts == 8 && Src.getOpcode() == ISD::CONCAT_VECTORS &&
57803 Src.getNumOperands() == 2) {
57804 SDValue Cvt0, Cvt1;
57805 SDValue Op0 = Src.getOperand(0);
57806 SDValue Op1 = Src.getOperand(1);
57807 bool IsOp0Strict = Op0->isStrictFPOpcode();
57808 if (Op0.getOpcode() != Op1.getOpcode() ||
57809 Op0.getOperand(IsOp0Strict ? 1 : 0).getValueType() != MVT::v4i64 ||
57810 Op1.getOperand(IsOp0Strict ? 1 : 0).getValueType() != MVT::v4i64) {
57811 return SDValue();
57812 }
57813 int Mask[8] = {0, 1, 2, 3, 8, 9, 10, 11};
57814 if (IsStrict) {
57815 assert(IsOp0Strict && "Op0 must be strict node");
57816 unsigned Opc = Op0.getOpcode() == ISD::STRICT_SINT_TO_FP
57817 ? X86ISD::STRICT_CVTSI2P
57818 : X86ISD::STRICT_CVTUI2P;
57819 Cvt0 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
57820 {Op0.getOperand(0), Op0.getOperand(1)});
57821 Cvt1 = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},
57822 {Op1.getOperand(0), Op1.getOperand(1)});
57823 Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
57824 return DAG.getMergeValues({Cvt, Cvt0.getValue(1)}, dl);
57825 }
57826 unsigned Opc = Op0.getOpcode() == ISD::SINT_TO_FP ? X86ISD::CVTSI2P
57827 : X86ISD::CVTUI2P;
57828 Cvt0 = DAG.getNode(Opc, dl, MVT::v8f16, Op0.getOperand(0));
57829 Cvt1 = DAG.getNode(Opc, dl, MVT::v8f16, Op1.getOperand(0));
57830 return Cvt = DAG.getVectorShuffle(MVT::v8f16, dl, Cvt0, Cvt1, Mask);
57831 }
57832 return SDValue();
57833 }
57834
57835 if (NumElts == 1 || !isPowerOf2_32(NumElts))
57836 return SDValue();
57837
57838 // Widen to at least 4 input elements.
57839 if (NumElts < 4)
57840 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
57841 DAG.getConstantFP(0.0, dl, SrcVT));
57842
57843 // Destination is v8i16 with at least 8 elements.
57844 EVT CvtVT =
57845 EVT::getVectorVT(*DAG.getContext(), MVT::i16, std::max(8U, NumElts));
57846 SDValue Rnd = DAG.getTargetConstant(4, dl, MVT::i32);
57847 if (IsStrict) {
57848 Cvt = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {CvtVT, MVT::Other},
57849 {N->getOperand(0), Src, Rnd});
57850 Chain = Cvt.getValue(1);
57851 } else {
57852 Cvt = DAG.getNode(X86ISD::CVTPS2PH, dl, CvtVT, Src, Rnd);
57853 }
57854
57855 // Extract down to real number of elements.
57856 if (NumElts < 8) {
57857 EVT IntVT = VT.changeVectorElementTypeToInteger();
57858 Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, IntVT, Cvt,
57859 DAG.getIntPtrConstant(0, dl));
57860 }
57861
57862 Cvt = DAG.getBitcast(VT, Cvt);
57863
57864 if (IsStrict)
57865 return DAG.getMergeValues({Cvt, Chain}, dl);
57866
57867 return Cvt;
57868}
57869
57870static SDValue combineMOVDQ2Q(SDNode *N, SelectionDAG &DAG) {
57871 SDValue Src = N->getOperand(0);
57872
57873 // Turn MOVDQ2Q+simple_load into an mmx load.
57874 if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse()) {
57875 LoadSDNode *LN = cast<LoadSDNode>(Src.getNode());
57876
57877 if (LN->isSimple()) {
57878 SDValue NewLd = DAG.getLoad(MVT::x86mmx, SDLoc(N), LN->getChain(),
57879 LN->getBasePtr(),
57880 LN->getPointerInfo(),
57881 LN->getOriginalAlign(),
57882 LN->getMemOperand()->getFlags());
57883 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), NewLd.getValue(1));
57884 return NewLd;
57885 }
57886 }
57887
57888 return SDValue();
57889}
57890
57891static SDValue combinePDEP(SDNode *N, SelectionDAG &DAG,
57892 TargetLowering::DAGCombinerInfo &DCI) {
57893 unsigned NumBits = N->getSimpleValueType(0).getSizeInBits();
57894 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
57895 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnes(NumBits), DCI))
57896 return SDValue(N, 0);
57897
57898 return SDValue();
57899}
57900
57901// Fixup the MMX intrinsics' types: in IR they are expressed with <1 x i64>,
57902// and so SelectionDAGBuilder creates them with v1i64 types, but they need to
57903// use x86mmx instead.
57904static SDValue FixupMMXIntrinsicTypes(SDNode *N, SelectionDAG &DAG) {
57905 SDLoc dl(N);
57906
57907 bool MadeChange = false, CastReturnVal = false;
57908 SmallVector<SDValue, 8> Args;
57909 for (const SDValue &Arg : N->op_values()) {
57910 if (Arg.getValueType() == MVT::v1i64) {
57911 MadeChange = true;
57912 Args.push_back(DAG.getBitcast(MVT::x86mmx, Arg));
57913 } else
57914 Args.push_back(Arg);
57915 }
57916 SDVTList VTs = N->getVTList();
57917 SDVTList NewVTs = VTs;
57918 if (VTs.NumVTs > 0 && VTs.VTs[0] == MVT::v1i64) {
57919 SmallVector<EVT> NewVTArr(ArrayRef<EVT>(VTs.VTs, VTs.NumVTs));
57920 NewVTArr[0] = MVT::x86mmx;
57921 NewVTs = DAG.getVTList(NewVTArr);
57922 MadeChange = true;
57923 CastReturnVal = true;
57924 }
57925
57926 if (MadeChange) {
57927 SDValue Result = DAG.getNode(N->getOpcode(), dl, NewVTs, Args);
57928 if (CastReturnVal) {
57929 SmallVector<SDValue, 2> Returns;
57930 for (unsigned i = 0, e = Result->getNumValues(); i != e; ++i)
57931 Returns.push_back(Result.getValue(i));
57932 Returns[0] = DAG.getBitcast(MVT::v1i64, Returns[0]);
57933 return DAG.getMergeValues(Returns, dl);
57934 }
57935 return Result;
57936 }
57937 return SDValue();
57938}
57939static SDValue combineINTRINSIC_WO_CHAIN(SDNode *N, SelectionDAG &DAG,
57940 TargetLowering::DAGCombinerInfo &DCI) {
57941 if (!DCI.isBeforeLegalize())
57942 return SDValue();
57943
57944 unsigned IntNo = N->getConstantOperandVal(0);
57945 const IntrinsicData *IntrData = getIntrinsicWithoutChain(IntNo);
57946
57947 if (IntrData && IntrData->Type == INTR_TYPE_CAST_MMX)
57948 return FixupMMXIntrinsicTypes(N, DAG);
57949
57950 return SDValue();
57951}
57952
57953static SDValue combineINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
57954 TargetLowering::DAGCombinerInfo &DCI) {
57955 if (!DCI.isBeforeLegalize())
57956 return SDValue();
57957
57958 unsigned IntNo = N->getConstantOperandVal(1);
57959 const IntrinsicData *IntrData = getIntrinsicWithChain(IntNo);
57960
57961 if (IntrData && IntrData->Type == INTR_TYPE_CAST_MMX)
57962 return FixupMMXIntrinsicTypes(N, DAG);
57963
57964 return SDValue();
57965}
57966
57967static SDValue combineINTRINSIC_VOID(SDNode *N, SelectionDAG &DAG,
57968 TargetLowering::DAGCombinerInfo &DCI) {
57969 if (!DCI.isBeforeLegalize())
57970 return SDValue();
57971
57972 unsigned IntNo = N->getConstantOperandVal(1);
57973 const IntrinsicData *IntrData = getIntrinsicWithChain(IntNo);
57974
57975 if (IntrData && IntrData->Type == INTR_TYPE_CAST_MMX)
57976 return FixupMMXIntrinsicTypes(N, DAG);
57977
57978 return SDValue();
57979}
57980
57981SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
57982 DAGCombinerInfo &DCI) const {
57983 SelectionDAG &DAG = DCI.DAG;
57984 switch (N->getOpcode()) {
57985 // clang-format off
57986 default: break;
57987 case ISD::SCALAR_TO_VECTOR:
57988 return combineScalarToVector(N, DAG);
57989 case ISD::EXTRACT_VECTOR_ELT:
57990 case X86ISD::PEXTRW:
57991 case X86ISD::PEXTRB:
57992 return combineExtractVectorElt(N, DAG, DCI, Subtarget);
57993 case ISD::CONCAT_VECTORS:
57994 return combineCONCAT_VECTORS(N, DAG, DCI, Subtarget);
57995 case ISD::INSERT_SUBVECTOR:
57996 return combineINSERT_SUBVECTOR(N, DAG, DCI, Subtarget);
57997 case ISD::EXTRACT_SUBVECTOR:
57998 return combineEXTRACT_SUBVECTOR(N, DAG, DCI, Subtarget);
57999 case ISD::VSELECT:
58000 case ISD::SELECT:
58001 case X86ISD::BLENDV: return combineSelect(N, DAG, DCI, Subtarget);
58002 case ISD::BITCAST: return combineBitcast(N, DAG, DCI, Subtarget);
58003 case X86ISD::CMOV: return combineCMov(N, DAG, DCI, Subtarget);
58004 case X86ISD::CMP: return combineCMP(N, DAG, DCI, Subtarget);
58005 case ISD::ADD: return combineAdd(N, DAG, DCI, Subtarget);
58006 case ISD::SUB: return combineSub(N, DAG, DCI, Subtarget);
58007 case X86ISD::ADD:
58008 case X86ISD::SUB: return combineX86AddSub(N, DAG, DCI, Subtarget);
58009 case X86ISD::CLOAD:
58010 case X86ISD::CSTORE: return combineX86CloadCstore(N, DAG);
58011 case X86ISD::SBB: return combineSBB(N, DAG);
58012 case X86ISD::ADC: return combineADC(N, DAG, DCI);
58013 case ISD::MUL: return combineMul(N, DAG, DCI, Subtarget);
58014 case ISD::SHL: return combineShiftLeft(N, DAG, Subtarget);
58015 case ISD::SRA: return combineShiftRightArithmetic(N, DAG, Subtarget);
58016 case ISD::SRL: return combineShiftRightLogical(N, DAG, DCI, Subtarget);
58017 case ISD::AND: return combineAnd(N, DAG, DCI, Subtarget);
58018 case ISD::OR: return combineOr(N, DAG, DCI, Subtarget);
58019 case ISD::XOR: return combineXor(N, DAG, DCI, Subtarget);
58020 case ISD::BITREVERSE: return combineBITREVERSE(N, DAG, DCI, Subtarget);
58021 case ISD::AVGCEILS:
58022 case ISD::AVGCEILU:
58023 case ISD::AVGFLOORS:
58024 case ISD::AVGFLOORU: return combineAVG(N, DAG, DCI, Subtarget);
58025 case X86ISD::BEXTR:
58026 case X86ISD::BEXTRI: return combineBEXTR(N, DAG, DCI, Subtarget);
58027 case ISD::LOAD: return combineLoad(N, DAG, DCI, Subtarget);
58028 case ISD::MLOAD: return combineMaskedLoad(N, DAG, DCI, Subtarget);
58029 case ISD::STORE: return combineStore(N, DAG, DCI, Subtarget);
58030 case ISD::MSTORE: return combineMaskedStore(N, DAG, DCI, Subtarget);
58031 case X86ISD::VEXTRACT_STORE:
58032 return combineVEXTRACT_STORE(N, DAG, DCI, Subtarget);
58033 case ISD::SINT_TO_FP:
58034 case ISD::STRICT_SINT_TO_FP:
58035 return combineSIntToFP(N, DAG, DCI, Subtarget);
58036 case ISD::UINT_TO_FP:
58037 case ISD::STRICT_UINT_TO_FP:
58038 return combineUIntToFP(N, DAG, Subtarget);
58039 case ISD::LRINT:
58040 case ISD::LLRINT: return combineLRINT_LLRINT(N, DAG, Subtarget);
58041 case ISD::FADD:
58042 case ISD::FSUB: return combineFaddFsub(N, DAG, Subtarget);
58043 case X86ISD::VFCMULC:
58044 case X86ISD::VFMULC: return combineFMulcFCMulc(N, DAG, Subtarget);
58045 case ISD::FNEG: return combineFneg(N, DAG, DCI, Subtarget);
58046 case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget);
58047 case X86ISD::VTRUNC: return combineVTRUNC(N, DAG, DCI);
58048 case X86ISD::ANDNP: return combineAndnp(N, DAG, DCI, Subtarget);
58049 case X86ISD::FAND: return combineFAnd(N, DAG, Subtarget);
58050 case X86ISD::FANDN: return combineFAndn(N, DAG, Subtarget);
58051 case X86ISD::FXOR:
58052 case X86ISD::FOR: return combineFOr(N, DAG, DCI, Subtarget);
58053 case X86ISD::FMIN:
58054 case X86ISD::FMAX: return combineFMinFMax(N, DAG);
58055 case ISD::FMINNUM:
58056 case ISD::FMAXNUM: return combineFMinNumFMaxNum(N, DAG, Subtarget);
58057 case X86ISD::CVTSI2P:
58058 case X86ISD::CVTUI2P: return combineX86INT_TO_FP(N, DAG, DCI);
58059 case X86ISD::CVTP2SI:
58060 case X86ISD::CVTP2UI:
58061 case X86ISD::STRICT_CVTTP2SI:
58062 case X86ISD::CVTTP2SI:
58063 case X86ISD::STRICT_CVTTP2UI:
58064 case X86ISD::CVTTP2UI:
58065 return combineCVTP2I_CVTTP2I(N, DAG, DCI);
58066 case X86ISD::STRICT_CVTPH2PS:
58067 case X86ISD::CVTPH2PS: return combineCVTPH2PS(N, DAG, DCI);
58068 case X86ISD::BT: return combineBT(N, DAG, DCI);
58069 case ISD::ANY_EXTEND:
58070 case ISD::ZERO_EXTEND: return combineZext(N, DAG, DCI, Subtarget);
58071 case ISD::SIGN_EXTEND: return combineSext(N, DAG, DCI, Subtarget);
58072 case ISD::SIGN_EXTEND_INREG: return combineSignExtendInReg(N, DAG, Subtarget);
58073 case ISD::ANY_EXTEND_VECTOR_INREG:
58074 case ISD::SIGN_EXTEND_VECTOR_INREG:
58075 case ISD::ZERO_EXTEND_VECTOR_INREG:
58076 return combineEXTEND_VECTOR_INREG(N, DAG, DCI, Subtarget);
58077 case ISD::SETCC: return combineSetCC(N, DAG, DCI, Subtarget);
58078 case X86ISD::SETCC: return combineX86SetCC(N, DAG, Subtarget);
58079 case X86ISD::BRCOND: return combineBrCond(N, DAG, Subtarget);
58080 case X86ISD::PACKSS:
58081 case X86ISD::PACKUS: return combineVectorPack(N, DAG, DCI, Subtarget);
58082 case X86ISD::HADD:
58083 case X86ISD::HSUB:
58084 case X86ISD::FHADD:
58085 case X86ISD::FHSUB: return combineVectorHADDSUB(N, DAG, DCI, Subtarget);
58086 case X86ISD::VSHL:
58087 case X86ISD::VSRA:
58088 case X86ISD::VSRL:
58089 return combineVectorShiftVar(N, DAG, DCI, Subtarget);
58090 case X86ISD::VSHLI:
58091 case X86ISD::VSRAI:
58092 case X86ISD::VSRLI:
58093 return combineVectorShiftImm(N, DAG, DCI, Subtarget);
58094 case ISD::INSERT_VECTOR_ELT:
58095 case X86ISD::PINSRB:
58096 case X86ISD::PINSRW: return combineVectorInsert(N, DAG, DCI, Subtarget);
58097 case X86ISD::SHUFP: // Handle all target specific shuffles
58098 case X86ISD::INSERTPS:
58099 case X86ISD::EXTRQI:
58100 case X86ISD::INSERTQI:
58101 case X86ISD::VALIGN:
58102 case X86ISD::PALIGNR:
58103 case X86ISD::VSHLDQ:
58104 case X86ISD::VSRLDQ:
58105 case X86ISD::BLENDI:
58106 case X86ISD::UNPCKH:
58107 case X86ISD::UNPCKL:
58108 case X86ISD::MOVHLPS:
58109 case X86ISD::MOVLHPS:
58110 case X86ISD::PSHUFB:
58111 case X86ISD::PSHUFD:
58112 case X86ISD::PSHUFHW:
58113 case X86ISD::PSHUFLW:
58114 case X86ISD::MOVSHDUP:
58115 case X86ISD::MOVSLDUP:
58116 case X86ISD::MOVDDUP:
58117 case X86ISD::MOVSS:
58118 case X86ISD::MOVSD:
58119 case X86ISD::MOVSH:
58120 case X86ISD::VBROADCAST:
58121 case X86ISD::VPPERM:
58122 case X86ISD::VPERMI:
58123 case X86ISD::VPERMV:
58124 case X86ISD::VPERMV3:
58125 case X86ISD::VPERMIL2:
58126 case X86ISD::VPERMILPI:
58127 case X86ISD::VPERMILPV:
58128 case X86ISD::VPERM2X128:
58129 case X86ISD::SHUF128:
58130 case X86ISD::VZEXT_MOVL:
58131 case ISD::VECTOR_SHUFFLE: return combineShuffle(N, DAG, DCI,Subtarget);
58132 case X86ISD::FMADD_RND:
58133 case X86ISD::FMSUB:
58134 case X86ISD::STRICT_FMSUB:
58135 case X86ISD::FMSUB_RND:
58136 case X86ISD::FNMADD:
58137 case X86ISD::STRICT_FNMADD:
58138 case X86ISD::FNMADD_RND:
58139 case X86ISD::FNMSUB:
58140 case X86ISD::STRICT_FNMSUB:
58141 case X86ISD::FNMSUB_RND:
58142 case ISD::FMA:
58143 case ISD::STRICT_FMA: return combineFMA(N, DAG, DCI, Subtarget);
58144 case X86ISD::FMADDSUB_RND:
58145 case X86ISD::FMSUBADD_RND:
58146 case X86ISD::FMADDSUB:
58147 case X86ISD::FMSUBADD: return combineFMADDSUB(N, DAG, DCI);
58148 case X86ISD::MOVMSK: return combineMOVMSK(N, DAG, DCI, Subtarget);
58149 case X86ISD::TESTP: return combineTESTP(N, DAG, DCI, Subtarget);
58150 case X86ISD::MGATHER:
58151 case X86ISD::MSCATTER: return combineX86GatherScatter(N, DAG, DCI);
58152 case ISD::MGATHER:
58153 case ISD::MSCATTER: return combineGatherScatter(N, DAG, DCI);
58154 case X86ISD::PCMPEQ:
58155 case X86ISD::PCMPGT: return combineVectorCompare(N, DAG, Subtarget);
58156 case X86ISD::PMULDQ:
58157 case X86ISD::PMULUDQ: return combinePMULDQ(N, DAG, DCI, Subtarget);
58158 case X86ISD::VPMADDUBSW:
58159 case X86ISD::VPMADDWD: return combineVPMADD(N, DAG, DCI);
58160 case X86ISD::KSHIFTL:
58161 case X86ISD::KSHIFTR: return combineKSHIFT(N, DAG, DCI);
58162 case ISD::FP16_TO_FP: return combineFP16_TO_FP(N, DAG, Subtarget);
58163 case ISD::STRICT_FP_EXTEND:
58164 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DAG, DCI, Subtarget);
58165 case ISD::STRICT_FP_ROUND:
58166 case ISD::FP_ROUND: return combineFP_ROUND(N, DAG, Subtarget);
58167 case X86ISD::VBROADCAST_LOAD:
58168 case X86ISD::SUBV_BROADCAST_LOAD: return combineBROADCAST_LOAD(N, DAG, DCI);
58169 case X86ISD::MOVDQ2Q: return combineMOVDQ2Q(N, DAG);
58170 case X86ISD::PDEP: return combinePDEP(N, DAG, DCI);
58171 case ISD::INTRINSIC_WO_CHAIN: return combineINTRINSIC_WO_CHAIN(N, DAG, DCI);
58172 case ISD::INTRINSIC_W_CHAIN: return combineINTRINSIC_W_CHAIN(N, DAG, DCI);
58173 case ISD::INTRINSIC_VOID: return combineINTRINSIC_VOID(N, DAG, DCI);
58174 // clang-format on
58175 }
58176
58177 return SDValue();
58178}
58179
58180bool X86TargetLowering::preferABDSToABSWithNSW(EVT VT) const {
58181 return false;
58182}
58183
58184// Prefer (non-AVX512) vector TRUNCATE(SIGN_EXTEND_INREG(X)) to use of PACKSS.
58185bool X86TargetLowering::preferSextInRegOfTruncate(EVT TruncVT, EVT VT,
58186 EVT ExtVT) const {
58187 return Subtarget.hasAVX512() || !VT.isVector();
58188}
58189
58190bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
58191 if (!isTypeLegal(VT))
58192 return false;
58193
58194 // There are no vXi8 shifts.
58195 if (Opc == ISD::SHL && VT.isVector() && VT.getVectorElementType() == MVT::i8)
58196 return false;
58197
58198 // TODO: Almost no 8-bit ops are desirable because they have no actual
58199 // size/speed advantages vs. 32-bit ops, but they do have a major
58200 // potential disadvantage by causing partial register stalls.
58201 //
58202 // 8-bit multiply/shl is probably not cheaper than 32-bit multiply/shl, and
58203 // we have specializations to turn 32-bit multiply/shl into LEA or other ops.
58204 // Also, see the comment in "IsDesirableToPromoteOp" - where we additionally
58205 // check for a constant operand to the multiply.
58206 if ((Opc == ISD::MUL || Opc == ISD::SHL) && VT == MVT::i8)
58207 return false;
58208
58209 // i16 instruction encodings are longer and some i16 instructions are slow,
58210 // so those are not desirable.
58211 if (VT == MVT::i16) {
58212 switch (Opc) {
58213 default:
58214 break;
58215 case ISD::LOAD:
58216 case ISD::SIGN_EXTEND:
58217 case ISD::ZERO_EXTEND:
58218 case ISD::ANY_EXTEND:
58219 case ISD::MUL:
58220 return false;
58221 case ISD::SHL:
58222 case ISD::SRA:
58223 case ISD::SRL:
58224 case ISD::SUB:
58225 case ISD::ADD:
58226 case ISD::AND:
58227 case ISD::OR:
58228 case ISD::XOR:
58229 // NDD instruction never has "partial register write" issue b/c it has
58230 // destination register's upper bits [63:OSIZE]) zeroed even when
58231 // OSIZE=8/16.
58232 return Subtarget.hasNDD();
58233 }
58234 }
58235
58236 // Any legal type not explicitly accounted for above here is desirable.
58237 return true;
58238}
58239
58240SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc &dl,
58241 SDValue Value, SDValue Addr,
58242 int JTI,
58243 SelectionDAG &DAG) const {
58244 const Module *M = DAG.getMachineFunction().getFunction().getParent();
58245 Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch");
58246 if (IsCFProtectionSupported) {
58247 // In case control-flow branch protection is enabled, we need to add
58248 // notrack prefix to the indirect branch.
58249 // In order to do that we create NT_BRIND SDNode.
58250 // Upon ISEL, the pattern will convert it to jmp with NoTrack prefix.
58251 SDValue JTInfo = DAG.getJumpTableDebugInfo(JTI, Value, dl);
58252 return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, JTInfo, Addr);
58253 }
58254
58255 return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG);
58256}
58257
58258TargetLowering::AndOrSETCCFoldKind
58259X86TargetLowering::isDesirableToCombineLogicOpOfSETCC(
58260 const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const {
58261 using AndOrSETCCFoldKind = TargetLowering::AndOrSETCCFoldKind;
58262 EVT VT = LogicOp->getValueType(0);
58263 EVT OpVT = SETCC0->getOperand(0).getValueType();
58264 if (!VT.isInteger())
58265 return AndOrSETCCFoldKind::None;
58266
58267 if (VT.isVector())
58268 return AndOrSETCCFoldKind(AndOrSETCCFoldKind::NotAnd |
58269 (isOperationLegal(ISD::ABS, OpVT)
58270 ? AndOrSETCCFoldKind::ABS
58271 : AndOrSETCCFoldKind::None));
58272
58273 // Don't use `NotAnd` as even though `not` is generally shorter code size than
58274 // `add`, `add` can lower to LEA which can save moves / spills. Any case where
58275 // `NotAnd` applies, `AddAnd` does as well.
58276 // TODO: Currently we lower (icmp eq/ne (and ~X, Y), 0) -> `test (not X), Y`,
58277 // if we change that to `andn Y, X` it may be worth prefering `NotAnd` here.
58278 return AndOrSETCCFoldKind::AddAnd;
58279}
58280
58281bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
58282 EVT VT = Op.getValueType();
58283 bool Is8BitMulByConstant = VT == MVT::i8 && Op.getOpcode() == ISD::MUL &&
58284 isa<ConstantSDNode>(Op.getOperand(1));
58285
58286 // i16 is legal, but undesirable since i16 instruction encodings are longer
58287 // and some i16 instructions are slow.
58288 // 8-bit multiply-by-constant can usually be expanded to something cheaper
58289 // using LEA and/or other ALU ops.
58290 if (VT != MVT::i16 && !Is8BitMulByConstant)
58291 return false;
58292
58293 auto IsFoldableRMW = [](SDValue Load, SDValue Op) {
58294 if (!Op.hasOneUse())
58295 return false;
58296 SDNode *User = *Op->use_begin();
58297 if (!ISD::isNormalStore(User))
58298 return false;
58299 auto *Ld = cast<LoadSDNode>(Load);
58300 auto *St = cast<StoreSDNode>(User);
58301 return Ld->getBasePtr() == St->getBasePtr();
58302 };
58303
58304 auto IsFoldableAtomicRMW = [](SDValue Load, SDValue Op) {
58305 if (!Load.hasOneUse() || Load.getOpcode() != ISD::ATOMIC_LOAD)
58306 return false;
58307 if (!Op.hasOneUse())
58308 return false;
58309 SDNode *User = *Op->use_begin();
58310 if (User->getOpcode() != ISD::ATOMIC_STORE)
58311 return false;
58312 auto *Ld = cast<AtomicSDNode>(Load);
58313 auto *St = cast<AtomicSDNode>(User);
58314 return Ld->getBasePtr() == St->getBasePtr();
58315 };
58316
58317 bool Commute = false;
58318 switch (Op.getOpcode()) {
58319 default: return false;
58320 case ISD::SIGN_EXTEND:
58321 case ISD::ZERO_EXTEND:
58322 case ISD::ANY_EXTEND:
58323 break;
58324 case ISD::SHL:
58325 case ISD::SRA:
58326 case ISD::SRL: {
58327 SDValue N0 = Op.getOperand(0);
58328 // Look out for (store (shl (load), x)).
58329 if (X86::mayFoldLoad(N0, Subtarget) && IsFoldableRMW(N0, Op))
58330 return false;
58331 break;
58332 }
58333 case ISD::ADD:
58334 case ISD::MUL:
58335 case ISD::AND:
58336 case ISD::OR:
58337 case ISD::XOR:
58338 Commute = true;
58339 [[fallthrough]];
58340 case ISD::SUB: {
58341 SDValue N0 = Op.getOperand(0);
58342 SDValue N1 = Op.getOperand(1);
58343 // Avoid disabling potential load folding opportunities.
58344 if (X86::mayFoldLoad(N1, Subtarget) &&
58345 (!Commute || !isa<ConstantSDNode>(N0) ||
58346 (Op.getOpcode() != ISD::MUL && IsFoldableRMW(N1, Op))))
58347 return false;
58348 if (X86::mayFoldLoad(N0, Subtarget) &&
58349 ((Commute && !isa<ConstantSDNode>(N1)) ||
58350 (Op.getOpcode() != ISD::MUL && IsFoldableRMW(N0, Op))))
58351 return false;
58352 if (IsFoldableAtomicRMW(N0, Op) ||
58353 (Commute && IsFoldableAtomicRMW(N1, Op)))
58354 return false;
58355 }
58356 }
58357
58358 PVT = MVT::i32;
58359 return true;
58360}
58361
58362//===----------------------------------------------------------------------===//
58363// X86 Inline Assembly Support
58364//===----------------------------------------------------------------------===//
58365
58366// Helper to match a string separated by whitespace.
58367static bool matchAsm(StringRef S, ArrayRef<const char *> Pieces) {
58368 S = S.substr(S.find_first_not_of(" \t")); // Skip leading whitespace.
58369
58370 for (StringRef Piece : Pieces) {
58371 if (!S.starts_with(Piece)) // Check if the piece matches.
58372 return false;
58373
58374 S = S.substr(Piece.size());
58375 StringRef::size_type Pos = S.find_first_not_of(" \t");
58376 if (Pos == 0) // We matched a prefix.
58377 return false;
58378
58379 S = S.substr(Pos);
58380 }
58381
58382 return S.empty();
58383}
58384
58385static bool clobbersFlagRegisters(const SmallVector<StringRef, 4> &AsmPieces) {
58386
58387 if (AsmPieces.size() == 3 || AsmPieces.size() == 4) {
58388 if (llvm::is_contained(AsmPieces, "~{cc}") &&
58389 llvm::is_contained(AsmPieces, "~{flags}") &&
58390 llvm::is_contained(AsmPieces, "~{fpsr}")) {
58391
58392 if (AsmPieces.size() == 3)
58393 return true;
58394 else if (llvm::is_contained(AsmPieces, "~{dirflag}"))
58395 return true;
58396 }
58397 }
58398 return false;
58399}
58400
58401bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
58402 InlineAsm *IA = cast<InlineAsm>(CI->getCalledOperand());
58403
58404 const std::string &AsmStr = IA->getAsmString();
58405
58406 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
58407 if (!Ty || Ty->getBitWidth() % 16 != 0)
58408 return false;
58409
58410 // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a"
58411 SmallVector<StringRef, 4> AsmPieces;
58412 SplitString(AsmStr, AsmPieces, ";\n");
58413
58414 switch (AsmPieces.size()) {
58415 default: return false;
58416 case 1:
58417 // FIXME: this should verify that we are targeting a 486 or better. If not,
58418 // we will turn this bswap into something that will be lowered to logical
58419 // ops instead of emitting the bswap asm. For now, we don't support 486 or
58420 // lower so don't worry about this.
58421 // bswap $0
58422 if (matchAsm(AsmPieces[0], {"bswap", "$0"}) ||
58423 matchAsm(AsmPieces[0], {"bswapl", "$0"}) ||
58424 matchAsm(AsmPieces[0], {"bswapq", "$0"}) ||
58425 matchAsm(AsmPieces[0], {"bswap", "${0:q}"}) ||
58426 matchAsm(AsmPieces[0], {"bswapl", "${0:q}"}) ||
58427 matchAsm(AsmPieces[0], {"bswapq", "${0:q}"})) {
58428 // No need to check constraints, nothing other than the equivalent of
58429 // "=r,0" would be valid here.
58430 return IntrinsicLowering::LowerToByteSwap(CI);
58431 }
58432
58433 // rorw $$8, ${0:w} --> llvm.bswap.i16
58434 if (CI->getType()->isIntegerTy(16) &&
58435 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
58436 (matchAsm(AsmPieces[0], {"rorw", "$$8,", "${0:w}"}) ||
58437 matchAsm(AsmPieces[0], {"rolw", "$$8,", "${0:w}"}))) {
58438 AsmPieces.clear();
58439 StringRef ConstraintsStr = IA->getConstraintString();
58440 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
58441 array_pod_sort(AsmPieces.begin(), AsmPieces.end());
58442 if (clobbersFlagRegisters(AsmPieces))
58443 return IntrinsicLowering::LowerToByteSwap(CI);
58444 }
58445 break;
58446 case 3:
58447 if (CI->getType()->isIntegerTy(32) &&
58448 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
58449 matchAsm(AsmPieces[0], {"rorw", "$$8,", "${0:w}"}) &&
58450 matchAsm(AsmPieces[1], {"rorl", "$$16,", "$0"}) &&
58451 matchAsm(AsmPieces[2], {"rorw", "$$8,", "${0:w}"})) {
58452 AsmPieces.clear();
58453 StringRef ConstraintsStr = IA->getConstraintString();
58454 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
58455 array_pod_sort(AsmPieces.begin(), AsmPieces.end());
58456 if (clobbersFlagRegisters(AsmPieces))
58457 return IntrinsicLowering::LowerToByteSwap(CI);
58458 }
58459
58460 if (CI->getType()->isIntegerTy(64)) {
58461 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
58462 if (Constraints.size() >= 2 &&
58463 Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" &&
58464 Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") {
58465 // bswap %eax / bswap %edx / xchgl %eax, %edx -> llvm.bswap.i64
58466 if (matchAsm(AsmPieces[0], {"bswap", "%eax"}) &&
58467 matchAsm(AsmPieces[1], {"bswap", "%edx"}) &&
58468 matchAsm(AsmPieces[2], {"xchgl", "%eax,", "%edx"}))
58469 return IntrinsicLowering::LowerToByteSwap(CI);
58470 }
58471 }
58472 break;
58473 }
58474 return false;
58475}
58476
58477static X86::CondCode parseConstraintCode(llvm::StringRef Constraint) {
58478 X86::CondCode Cond = StringSwitch<X86::CondCode>(Constraint)
58479 .Case("{@cca}", X86::COND_A)
58480 .Case("{@ccae}", X86::COND_AE)
58481 .Case("{@ccb}", X86::COND_B)
58482 .Case("{@ccbe}", X86::COND_BE)
58483 .Case("{@ccc}", X86::COND_B)
58484 .Case("{@cce}", X86::COND_E)
58485 .Case("{@ccz}", X86::COND_E)
58486 .Case("{@ccg}", X86::COND_G)
58487 .Case("{@ccge}", X86::COND_GE)
58488 .Case("{@ccl}", X86::COND_L)
58489 .Case("{@ccle}", X86::COND_LE)
58490 .Case("{@ccna}", X86::COND_BE)
58491 .Case("{@ccnae}", X86::COND_B)
58492 .Case("{@ccnb}", X86::COND_AE)
58493 .Case("{@ccnbe}", X86::COND_A)
58494 .Case("{@ccnc}", X86::COND_AE)
58495 .Case("{@ccne}", X86::COND_NE)
58496 .Case("{@ccnz}", X86::COND_NE)
58497 .Case("{@ccng}", X86::COND_LE)
58498 .Case("{@ccnge}", X86::COND_L)
58499 .Case("{@ccnl}", X86::COND_GE)
58500 .Case("{@ccnle}", X86::COND_G)
58501 .Case("{@ccno}", X86::COND_NO)
58502 .Case("{@ccnp}", X86::COND_NP)
58503 .Case("{@ccns}", X86::COND_NS)
58504 .Case("{@cco}", X86::COND_O)
58505 .Case("{@ccp}", X86::COND_P)
58506 .Case("{@ccs}", X86::COND_S)
58507 .Default(X86::COND_INVALID);
58508 return Cond;
58509}
58510
58511/// Given a constraint letter, return the type of constraint for this target.
58512X86TargetLowering::ConstraintType
58513X86TargetLowering::getConstraintType(StringRef Constraint) const {
58514 if (Constraint.size() == 1) {
58515 switch (Constraint[0]) {
58516 case 'R':
58517 case 'q':
58518 case 'Q':
58519 case 'f':
58520 case 't':
58521 case 'u':
58522 case 'y':
58523 case 'x':
58524 case 'v':
58525 case 'l':
58526 case 'k': // AVX512 masking registers.
58527 return C_RegisterClass;
58528 case 'a':
58529 case 'b':
58530 case 'c':
58531 case 'd':
58532 case 'S':
58533 case 'D':
58534 case 'A':
58535 return C_Register;
58536 case 'I':
58537 case 'J':
58538 case 'K':
58539 case 'N':
58540 case 'G':
58541 case 'L':
58542 case 'M':
58543 return C_Immediate;
58544 case 'C':
58545 case 'e':
58546 case 'Z':
58547 return C_Other;
58548 default:
58549 break;
58550 }
58551 }
58552 else if (Constraint.size() == 2) {
58553 switch (Constraint[0]) {
58554 default:
58555 break;
58556 case 'W':
58557 if (Constraint[1] != 's')
58558 break;
58559 return C_Other;
58560 case 'Y':
58561 switch (Constraint[1]) {
58562 default:
58563 break;
58564 case 'z':
58565 return C_Register;
58566 case 'i':
58567 case 'm':
58568 case 'k':
58569 case 't':
58570 case '2':
58571 return C_RegisterClass;
58572 }
58573 break;
58574 case 'j':
58575 switch (Constraint[1]) {
58576 default:
58577 break;
58578 case 'r':
58579 case 'R':
58580 return C_RegisterClass;
58581 }
58582 }
58583 } else if (parseConstraintCode(Constraint) != X86::COND_INVALID)
58584 return C_Other;
58585 return TargetLowering::getConstraintType(Constraint);
58586}
58587
58588/// Examine constraint type and operand type and determine a weight value.
58589/// This object must already have been set up with the operand type
58590/// and the current alternative constraint selected.
58591TargetLowering::ConstraintWeight
58592X86TargetLowering::getSingleConstraintMatchWeight(
58593 AsmOperandInfo &Info, const char *Constraint) const {
58594 ConstraintWeight Wt = CW_Invalid;
58595 Value *CallOperandVal = Info.CallOperandVal;
58596 // If we don't have a value, we can't do a match,
58597 // but allow it at the lowest weight.
58598 if (!CallOperandVal)
58599 return CW_Default;
58600 Type *Ty = CallOperandVal->getType();
58601 // Look at the constraint type.
58602 switch (*Constraint) {
58603 default:
58604 Wt = TargetLowering::getSingleConstraintMatchWeight(Info, Constraint);
58605 [[fallthrough]];
58606 case 'R':
58607 case 'q':
58608 case 'Q':
58609 case 'a':
58610 case 'b':
58611 case 'c':
58612 case 'd':
58613 case 'S':
58614 case 'D':
58615 case 'A':
58616 if (CallOperandVal->getType()->isIntegerTy())
58617 Wt = CW_SpecificReg;
58618 break;
58619 case 'f':
58620 case 't':
58621 case 'u':
58622 if (Ty->isFloatingPointTy())
58623 Wt = CW_SpecificReg;
58624 break;
58625 case 'y':
58626 if (Ty->getPrimitiveSizeInBits() == 64 && Subtarget.hasMMX())
58627 Wt = CW_SpecificReg;
58628 break;
58629 case 'Y':
58630 if (StringRef(Constraint).size() != 2)
58631 break;
58632 switch (Constraint[1]) {
58633 default:
58634 return CW_Invalid;
58635 // XMM0
58636 case 'z':
58637 if (((Ty->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) ||
58638 ((Ty->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX()) ||
58639 ((Ty->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512()))
58640 return CW_SpecificReg;
58641 return CW_Invalid;
58642 // Conditional OpMask regs (AVX512)
58643 case 'k':
58644 if ((Ty->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512())
58645 return CW_Register;
58646 return CW_Invalid;
58647 // Any MMX reg
58648 case 'm':
58649 if (Ty->getPrimitiveSizeInBits() == 64 && Subtarget.hasMMX())
58650 return CW_SpecificReg;
58651 return CW_Invalid;
58652 // Any SSE reg when ISA >= SSE2, same as 'x'
58653 case 'i':
58654 case 't':
58655 case '2':
58656 if (!Subtarget.hasSSE2())
58657 return CW_Invalid;
58658 break;
58659 }
58660 break;
58661 case 'j':
58662 if (StringRef(Constraint).size() != 2)
58663 break;
58664 switch (Constraint[1]) {
58665 default:
58666 return CW_Invalid;
58667 case 'r':
58668 case 'R':
58669 if (CallOperandVal->getType()->isIntegerTy())
58670 Wt = CW_SpecificReg;
58671 break;
58672 }
58673 break;
58674 case 'v':
58675 if ((Ty->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512())
58676 Wt = CW_Register;
58677 [[fallthrough]];
58678 case 'x':
58679 if (((Ty->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) ||
58680 ((Ty->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX()))
58681 Wt = CW_Register;
58682 break;
58683 case 'k':
58684 // Enable conditional vector operations using %k<#> registers.
58685 if ((Ty->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512())
58686 Wt = CW_Register;
58687 break;
58688 case 'I':
58689 if (auto *C = dyn_cast<ConstantInt>(Info.CallOperandVal))
58690 if (C->getZExtValue() <= 31)
58691 Wt = CW_Constant;
58692 break;
58693 case 'J':
58694 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
58695 if (C->getZExtValue() <= 63)
58696 Wt = CW_Constant;
58697 break;
58698 case 'K':
58699 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
58700 if ((C->getSExtValue() >= -0x80) && (C->getSExtValue() <= 0x7f))
58701 Wt = CW_Constant;
58702 break;
58703 case 'L':
58704 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
58705 if ((C->getZExtValue() == 0xff) || (C->getZExtValue() == 0xffff))
58706 Wt = CW_Constant;
58707 break;
58708 case 'M':
58709 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
58710 if (C->getZExtValue() <= 3)
58711 Wt = CW_Constant;
58712 break;
58713 case 'N':
58714 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
58715 if (C->getZExtValue() <= 0xff)
58716 Wt = CW_Constant;
58717 break;
58718 case 'G':
58719 case 'C':
58720 if (isa<ConstantFP>(CallOperandVal))
58721 Wt = CW_Constant;
58722 break;
58723 case 'e':
58724 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
58725 if ((C->getSExtValue() >= -0x80000000LL) &&
58726 (C->getSExtValue() <= 0x7fffffffLL))
58727 Wt = CW_Constant;
58728 break;
58729 case 'Z':
58730 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
58731 if (C->getZExtValue() <= 0xffffffff)
58732 Wt = CW_Constant;
58733 break;
58734 }
58735 return Wt;
58736}
58737
58738/// Try to replace an X constraint, which matches anything, with another that
58739/// has more specific requirements based on the type of the corresponding
58740/// operand.
58741const char *X86TargetLowering::
58742LowerXConstraint(EVT ConstraintVT) const {
58743 // FP X constraints get lowered to SSE1/2 registers if available, otherwise
58744 // 'f' like normal targets.
58745 if (ConstraintVT.isFloatingPoint()) {
58746 if (Subtarget.hasSSE1())
58747 return "x";
58748 }
58749
58750 return TargetLowering::LowerXConstraint(ConstraintVT);
58751}
58752
58753// Lower @cc targets via setcc.
58754SDValue X86TargetLowering::LowerAsmOutputForConstraint(
58755 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
58756 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
58757 X86::CondCode Cond = parseConstraintCode(OpInfo.ConstraintCode);
58758 if (Cond == X86::COND_INVALID)
58759 return SDValue();
58760 // Check that return type is valid.
58761 if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
58762 OpInfo.ConstraintVT.getSizeInBits() < 8)
58763 report_fatal_error("Glue output operand is of invalid type");
58764
58765 // Get EFLAGS register. Only update chain when copyfrom is glued.
58766 if (Glue.getNode()) {
58767 Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, Glue);
58768 Chain = Glue.getValue(1);
58769 } else
58770 Glue = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32);
58771 // Extract CC code.
58772 SDValue CC = getSETCC(Cond, Glue, DL, DAG);
58773 // Extend to 32-bits
58774 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
58775
58776 return Result;
58777}
58778
58779/// Lower the specified operand into the Ops vector.
58780/// If it is invalid, don't add anything to Ops.
58781void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
58782 StringRef Constraint,
58783 std::vector<SDValue> &Ops,
58784 SelectionDAG &DAG) const {
58785 SDValue Result;
58786 char ConstraintLetter = Constraint[0];
58787 switch (ConstraintLetter) {
58788 default: break;
58789 case 'I':
58790 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
58791 if (C->getZExtValue() <= 31) {
58792 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
58793 Op.getValueType());
58794 break;
58795 }
58796 }
58797 return;
58798 case 'J':
58799 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
58800 if (C->getZExtValue() <= 63) {
58801 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
58802 Op.getValueType());
58803 break;
58804 }
58805 }
58806 return;
58807 case 'K':
58808 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
58809 if (isInt<8>(C->getSExtValue())) {
58810 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
58811 Op.getValueType());
58812 break;
58813 }
58814 }
58815 return;
58816 case 'L':
58817 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
58818 if (C->getZExtValue() == 0xff || C->getZExtValue() == 0xffff ||
58819 (Subtarget.is64Bit() && C->getZExtValue() == 0xffffffff)) {
58820 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
58821 Op.getValueType());
58822 break;
58823 }
58824 }
58825 return;
58826 case 'M':
58827 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
58828 if (C->getZExtValue() <= 3) {
58829 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
58830 Op.getValueType());
58831 break;
58832 }
58833 }
58834 return;
58835 case 'N':
58836 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
58837 if (C->getZExtValue() <= 255) {
58838 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
58839 Op.getValueType());
58840 break;
58841 }
58842 }
58843 return;
58844 case 'O':
58845 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
58846 if (C->getZExtValue() <= 127) {
58847 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
58848 Op.getValueType());
58849 break;
58850 }
58851 }
58852 return;
58853 case 'e': {
58854 // 32-bit signed value
58855 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
58856 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
58857 C->getSExtValue())) {
58858 // Widen to 64 bits here to get it sign extended.
58859 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);
58860 break;
58861 }
58862 // FIXME gcc accepts some relocatable values here too, but only in certain
58863 // memory models; it's complicated.
58864 }
58865 return;
58866 }
58867 case 'W': {
58868 assert(Constraint[1] == 's');
58869 // Op is a BlockAddressSDNode or a GlobalAddressSDNode with an optional
58870 // offset.
58871 if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
58872 Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
58873 BA->getValueType(0)));
58874 } else {
58875 int64_t Offset = 0;
58876 if (Op->getOpcode() == ISD::ADD &&
58877 isa<ConstantSDNode>(Op->getOperand(1))) {
58878 Offset = cast<ConstantSDNode>(Op->getOperand(1))->getSExtValue();
58879 Op = Op->getOperand(0);
58880 }
58881 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op))
58882 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
58883 GA->getValueType(0), Offset));
58884 }
58885 return;
58886 }
58887 case 'Z': {
58888 // 32-bit unsigned value
58889 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
58890 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
58891 C->getZExtValue())) {
58892 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
58893 Op.getValueType());
58894 break;
58895 }
58896 }
58897 // FIXME gcc accepts some relocatable values here too, but only in certain
58898 // memory models; it's complicated.
58899 return;
58900 }
58901 case 'i': {
58902 // Literal immediates are always ok.
58903 if (auto *CST = dyn_cast<ConstantSDNode>(Op)) {
58904 bool IsBool = CST->getConstantIntValue()->getBitWidth() == 1;
58905 BooleanContent BCont = getBooleanContents(MVT::i64);
58906 ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
58907 : ISD::SIGN_EXTEND;
58908 int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? CST->getZExtValue()
58909 : CST->getSExtValue();
58910 Result = DAG.getTargetConstant(ExtVal, SDLoc(Op), MVT::i64);
58911 break;
58912 }
58913
58914 // In any sort of PIC mode addresses need to be computed at runtime by
58915 // adding in a register or some sort of table lookup. These can't
58916 // be used as immediates. BlockAddresses and BasicBlocks are fine though.
58917 if ((Subtarget.isPICStyleGOT() || Subtarget.isPICStyleStubPIC()) &&
58918 !(isa<BlockAddressSDNode>(Op) || isa<BasicBlockSDNode>(Op)))
58919 return;
58920
58921 // If we are in non-pic codegen mode, we allow the address of a global (with
58922 // an optional displacement) to be used with 'i'.
58923 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op))
58924 // If we require an extra load to get this address, as in PIC mode, we
58925 // can't accept it.
58926 if (isGlobalStubReference(
58927 Subtarget.classifyGlobalReference(GA->getGlobal())))
58928 return;
58929 break;
58930 }
58931 }
58932
58933 if (Result.getNode()) {
58934 Ops.push_back(Result);
58935 return;
58936 }
58937 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
58938}
58939
58940/// Check if \p RC is a general purpose register class.
58941/// I.e., GR* or one of their variant.
58942static bool isGRClass(const TargetRegisterClass &RC) {
58943 return RC.hasSuperClassEq(&X86::GR8RegClass) ||
58944 RC.hasSuperClassEq(&X86::GR16RegClass) ||
58945 RC.hasSuperClassEq(&X86::GR32RegClass) ||
58946 RC.hasSuperClassEq(&X86::GR64RegClass) ||
58947 RC.hasSuperClassEq(&X86::LOW32_ADDR_ACCESS_RBPRegClass);
58948}
58949
58950/// Check if \p RC is a vector register class.
58951/// I.e., FR* / VR* or one of their variant.
58952static bool isFRClass(const TargetRegisterClass &RC) {
58953 return RC.hasSuperClassEq(&X86::FR16XRegClass) ||
58954 RC.hasSuperClassEq(&X86::FR32XRegClass) ||
58955 RC.hasSuperClassEq(&X86::FR64XRegClass) ||
58956 RC.hasSuperClassEq(&X86::VR128XRegClass) ||
58957 RC.hasSuperClassEq(&X86::VR256XRegClass) ||
58958 RC.hasSuperClassEq(&X86::VR512RegClass);
58959}
58960
58961/// Check if \p RC is a mask register class.
58962/// I.e., VK* or one of their variant.
58963static bool isVKClass(const TargetRegisterClass &RC) {
58964 return RC.hasSuperClassEq(&X86::VK1RegClass) ||
58965 RC.hasSuperClassEq(&X86::VK2RegClass) ||
58966 RC.hasSuperClassEq(&X86::VK4RegClass) ||
58967 RC.hasSuperClassEq(&X86::VK8RegClass) ||
58968 RC.hasSuperClassEq(&X86::VK16RegClass) ||
58969 RC.hasSuperClassEq(&X86::VK32RegClass) ||
58970 RC.hasSuperClassEq(&X86::VK64RegClass);
58971}
58972
58973static bool useEGPRInlineAsm(const X86Subtarget &Subtarget) {
58974 return Subtarget.hasEGPR() && Subtarget.useInlineAsmGPR32();
58975}
58976
58977std::pair<unsigned, const TargetRegisterClass *>
58978X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
58979 StringRef Constraint,
58980 MVT VT) const {
58981 // First, see if this is a constraint that directly corresponds to an LLVM
58982 // register class.
58983 if (Constraint.size() == 1) {
58984 // GCC Constraint Letters
58985 switch (Constraint[0]) {
58986 default: break;
58987 // 'A' means [ER]AX + [ER]DX.
58988 case 'A':
58989 if (Subtarget.is64Bit())
58990 return std::make_pair(X86::RAX, &X86::GR64_ADRegClass);
58991 assert((Subtarget.is32Bit() || Subtarget.is16Bit()) &&
58992 "Expecting 64, 32 or 16 bit subtarget");
58993 return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
58994
58995 // TODO: Slight differences here in allocation order and leaving
58996 // RIP in the class. Do they matter any more here than they do
58997 // in the normal allocation?
58998 case 'k':
58999 if (Subtarget.hasAVX512()) {
59000 if (VT == MVT::v1i1 || VT == MVT::i1)
59001 return std::make_pair(0U, &X86::VK1RegClass);
59002 if (VT == MVT::v8i1 || VT == MVT::i8)
59003 return std::make_pair(0U, &X86::VK8RegClass);
59004 if (VT == MVT::v16i1 || VT == MVT::i16)
59005 return std::make_pair(0U, &X86::VK16RegClass);
59006 }
59007 if (Subtarget.hasBWI()) {
59008 if (VT == MVT::v32i1 || VT == MVT::i32)
59009 return std::make_pair(0U, &X86::VK32RegClass);
59010 if (VT == MVT::v64i1 || VT == MVT::i64)
59011 return std::make_pair(0U, &X86::VK64RegClass);
59012 }
59013 break;
59014 case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
59015 if (Subtarget.is64Bit()) {
59016 if (VT == MVT::i8 || VT == MVT::i1)
59017 return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
59018 ? &X86::GR8RegClass
59019 : &X86::GR8_NOREX2RegClass);
59020 if (VT == MVT::i16)
59021 return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
59022 ? &X86::GR16RegClass
59023 : &X86::GR16_NOREX2RegClass);
59024 if (VT == MVT::i32 || VT == MVT::f32)
59025 return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
59026 ? &X86::GR32RegClass
59027 : &X86::GR32_NOREX2RegClass);
59028 if (VT != MVT::f80 && !VT.isVector())
59029 return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
59030 ? &X86::GR64RegClass
59031 : &X86::GR64_NOREX2RegClass);
59032 break;
59033 }
59034 [[fallthrough]];
59035 // 32-bit fallthrough
59036 case 'Q': // Q_REGS
59037 if (VT == MVT::i8 || VT == MVT::i1)
59038 return std::make_pair(0U, &X86::GR8_ABCD_LRegClass);
59039 if (VT == MVT::i16)
59040 return std::make_pair(0U, &X86::GR16_ABCDRegClass);
59041 if (VT == MVT::i32 || VT == MVT::f32 ||
59042 (!VT.isVector() && !Subtarget.is64Bit()))
59043 return std::make_pair(0U, &X86::GR32_ABCDRegClass);
59044 if (VT != MVT::f80 && !VT.isVector())
59045 return std::make_pair(0U, &X86::GR64_ABCDRegClass);
59046 break;
59047 case 'r': // GENERAL_REGS
59048 case 'l': // INDEX_REGS
59049 if (VT == MVT::i8 || VT == MVT::i1)
59050 return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
59051 ? &X86::GR8RegClass
59052 : &X86::GR8_NOREX2RegClass);
59053 if (VT == MVT::i16)
59054 return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
59055 ? &X86::GR16RegClass
59056 : &X86::GR16_NOREX2RegClass);
59057 if (VT == MVT::i32 || VT == MVT::f32 ||
59058 (!VT.isVector() && !Subtarget.is64Bit()))
59059 return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
59060 ? &X86::GR32RegClass
59061 : &X86::GR32_NOREX2RegClass);
59062 if (VT != MVT::f80 && !VT.isVector())
59063 return std::make_pair(0U, useEGPRInlineAsm(Subtarget)
59064 ? &X86::GR64RegClass
59065 : &X86::GR64_NOREX2RegClass);
59066 break;
59067 case 'R': // LEGACY_REGS
59068 if (VT == MVT::i8 || VT == MVT::i1)
59069 return std::make_pair(0U, &X86::GR8_NOREXRegClass);
59070 if (VT == MVT::i16)
59071 return std::make_pair(0U, &X86::GR16_NOREXRegClass);
59072 if (VT == MVT::i32 || VT == MVT::f32 ||
59073 (!VT.isVector() && !Subtarget.is64Bit()))
59074 return std::make_pair(0U, &X86::GR32_NOREXRegClass);
59075 if (VT != MVT::f80 && !VT.isVector())
59076 return std::make_pair(0U, &X86::GR64_NOREXRegClass);
59077 break;
59078 case 'f': // FP Stack registers.
59079 // If SSE is enabled for this VT, use f80 to ensure the isel moves the
59080 // value to the correct fpstack register class.
59081 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
59082 return std::make_pair(0U, &X86::RFP32RegClass);
59083 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))
59084 return std::make_pair(0U, &X86::RFP64RegClass);
59085 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80)
59086 return std::make_pair(0U, &X86::RFP80RegClass);
59087 break;
59088 case 'y': // MMX_REGS if MMX allowed.
59089 if (!Subtarget.hasMMX()) break;
59090 return std::make_pair(0U, &X86::VR64RegClass);
59091 case 'v':
59092 case 'x': // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
59093 if (!Subtarget.hasSSE1()) break;
59094 bool VConstraint = (Constraint[0] == 'v');
59095
59096 switch (VT.SimpleTy) {
59097 default: break;
59098 // Scalar SSE types.
59099 case MVT::f16:
59100 if (VConstraint && Subtarget.hasFP16())
59101 return std::make_pair(0U, &X86::FR16XRegClass);
59102 break;
59103 case MVT::f32:
59104 case MVT::i32:
59105 if (VConstraint && Subtarget.hasVLX())
59106 return std::make_pair(0U, &X86::FR32XRegClass);
59107 return std::make_pair(0U, &X86::FR32RegClass);
59108 case MVT::f64:
59109 case MVT::i64:
59110 if (VConstraint && Subtarget.hasVLX())
59111 return std::make_pair(0U, &X86::FR64XRegClass);
59112 return std::make_pair(0U, &X86::FR64RegClass);
59113 case MVT::i128:
59114 if (Subtarget.is64Bit()) {
59115 if (VConstraint && Subtarget.hasVLX())
59116 return std::make_pair(0U, &X86::VR128XRegClass);
59117 return std::make_pair(0U, &X86::VR128RegClass);
59118 }
59119 break;
59120 // Vector types and fp128.
59121 case MVT::v8f16:
59122 if (!Subtarget.hasFP16())
59123 break;
59124 if (VConstraint)
59125 return std::make_pair(0U, &X86::VR128XRegClass);
59126 return std::make_pair(0U, &X86::VR128RegClass);
59127 case MVT::v8bf16:
59128 if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
59129 break;
59130 if (VConstraint)
59131 return std::make_pair(0U, &X86::VR128XRegClass);
59132 return std::make_pair(0U, &X86::VR128RegClass);
59133 case MVT::f128:
59134 case MVT::v16i8:
59135 case MVT::v8i16:
59136 case MVT::v4i32:
59137 case MVT::v2i64:
59138 case MVT::v4f32:
59139 case MVT::v2f64:
59140 if (VConstraint && Subtarget.hasVLX())
59141 return std::make_pair(0U, &X86::VR128XRegClass);
59142 return std::make_pair(0U, &X86::VR128RegClass);
59143 // AVX types.
59144 case MVT::v16f16:
59145 if (!Subtarget.hasFP16())
59146 break;
59147 if (VConstraint)
59148 return std::make_pair(0U, &X86::VR256XRegClass);
59149 return std::make_pair(0U, &X86::VR256RegClass);
59150 case MVT::v16bf16:
59151 if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
59152 break;
59153 if (VConstraint)
59154 return std::make_pair(0U, &X86::VR256XRegClass);
59155 return std::make_pair(0U, &X86::VR256RegClass);
59156 case MVT::v32i8:
59157 case MVT::v16i16:
59158 case MVT::v8i32:
59159 case MVT::v4i64:
59160 case MVT::v8f32:
59161 case MVT::v4f64:
59162 if (VConstraint && Subtarget.hasVLX())
59163 return std::make_pair(0U, &X86::VR256XRegClass);
59164 if (Subtarget.hasAVX())
59165 return std::make_pair(0U, &X86::VR256RegClass);
59166 break;
59167 case MVT::v32f16:
59168 if (!Subtarget.hasFP16())
59169 break;
59170 if (VConstraint)
59171 return std::make_pair(0U, &X86::VR512RegClass);
59172 return std::make_pair(0U, &X86::VR512_0_15RegClass);
59173 case MVT::v32bf16:
59174 if (!Subtarget.hasBF16())
59175 break;
59176 if (VConstraint)
59177 return std::make_pair(0U, &X86::VR512RegClass);
59178 return std::make_pair(0U, &X86::VR512_0_15RegClass);
59179 case MVT::v64i8:
59180 case MVT::v32i16:
59181 case MVT::v8f64:
59182 case MVT::v16f32:
59183 case MVT::v16i32:
59184 case MVT::v8i64:
59185 if (!Subtarget.hasAVX512()) break;
59186 if (VConstraint)
59187 return std::make_pair(0U, &X86::VR512RegClass);
59188 return std::make_pair(0U, &X86::VR512_0_15RegClass);
59189 }
59190 break;
59191 }
59192 } else if (Constraint.size() == 2 && Constraint[0] == 'Y') {
59193 switch (Constraint[1]) {
59194 default:
59195 break;
59196 case 'i':
59197 case 't':
59198 case '2':
59199 return getRegForInlineAsmConstraint(TRI, "x", VT);
59200 case 'm':
59201 if (!Subtarget.hasMMX()) break;
59202 return std::make_pair(0U, &X86::VR64RegClass);
59203 case 'z':
59204 if (!Subtarget.hasSSE1()) break;
59205 switch (VT.SimpleTy) {
59206 default: break;
59207 // Scalar SSE types.
59208 case MVT::f16:
59209 if (!Subtarget.hasFP16())
59210 break;
59211 return std::make_pair(X86::XMM0, &X86::FR16XRegClass);
59212 case MVT::f32:
59213 case MVT::i32:
59214 return std::make_pair(X86::XMM0, &X86::FR32RegClass);
59215 case MVT::f64:
59216 case MVT::i64:
59217 return std::make_pair(X86::XMM0, &X86::FR64RegClass);
59218 case MVT::v8f16:
59219 if (!Subtarget.hasFP16())
59220 break;
59221 return std::make_pair(X86::XMM0, &X86::VR128RegClass);
59222 case MVT::v8bf16:
59223 if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
59224 break;
59225 return std::make_pair(X86::XMM0, &X86::VR128RegClass);
59226 case MVT::f128:
59227 case MVT::v16i8:
59228 case MVT::v8i16:
59229 case MVT::v4i32:
59230 case MVT::v2i64:
59231 case MVT::v4f32:
59232 case MVT::v2f64:
59233 return std::make_pair(X86::XMM0, &X86::VR128RegClass);
59234 // AVX types.
59235 case MVT::v16f16:
59236 if (!Subtarget.hasFP16())
59237 break;
59238 return std::make_pair(X86::YMM0, &X86::VR256RegClass);
59239 case MVT::v16bf16:
59240 if (!Subtarget.hasBF16() || !Subtarget.hasVLX())
59241 break;
59242 return std::make_pair(X86::YMM0, &X86::VR256RegClass);
59243 case MVT::v32i8:
59244 case MVT::v16i16:
59245 case MVT::v8i32:
59246 case MVT::v4i64:
59247 case MVT::v8f32:
59248 case MVT::v4f64:
59249 if (Subtarget.hasAVX())
59250 return std::make_pair(X86::YMM0, &X86::VR256RegClass);
59251 break;
59252 case MVT::v32f16:
59253 if (!Subtarget.hasFP16())
59254 break;
59255 return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
59256 case MVT::v32bf16:
59257 if (!Subtarget.hasBF16())
59258 break;
59259 return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
59260 case MVT::v64i8:
59261 case MVT::v32i16:
59262 case MVT::v8f64:
59263 case MVT::v16f32:
59264 case MVT::v16i32:
59265 case MVT::v8i64:
59266 if (Subtarget.hasAVX512())
59267 return std::make_pair(X86::ZMM0, &X86::VR512_0_15RegClass);
59268 break;
59269 }
59270 break;
59271 case 'k':
59272 // This register class doesn't allocate k0 for masked vector operation.
59273 if (Subtarget.hasAVX512()) {
59274 if (VT == MVT::v1i1 || VT == MVT::i1)
59275 return std::make_pair(0U, &X86::VK1WMRegClass);
59276 if (VT == MVT::v8i1 || VT == MVT::i8)
59277 return std::make_pair(0U, &X86::VK8WMRegClass);
59278 if (VT == MVT::v16i1 || VT == MVT::i16)
59279 return std::make_pair(0U, &X86::VK16WMRegClass);
59280 }
59281 if (Subtarget.hasBWI()) {
59282 if (VT == MVT::v32i1 || VT == MVT::i32)
59283 return std::make_pair(0U, &X86::VK32WMRegClass);
59284 if (VT == MVT::v64i1 || VT == MVT::i64)
59285 return std::make_pair(0U, &X86::VK64WMRegClass);
59286 }
59287 break;
59288 }
59289 } else if (Constraint.size() == 2 && Constraint[0] == 'j') {
59290 switch (Constraint[1]) {
59291 default:
59292 break;
59293 case 'r':
59294 if (VT == MVT::i8 || VT == MVT::i1)
59295 return std::make_pair(0U, &X86::GR8_NOREX2RegClass);
59296 if (VT == MVT::i16)
59297 return std::make_pair(0U, &X86::GR16_NOREX2RegClass);
59298 if (VT == MVT::i32 || VT == MVT::f32)
59299 return std::make_pair(0U, &X86::GR32_NOREX2RegClass);
59300 if (VT != MVT::f80 && !VT.isVector())
59301 return std::make_pair(0U, &X86::GR64_NOREX2RegClass);
59302 break;
59303 case 'R':
59304 if (VT == MVT::i8 || VT == MVT::i1)
59305 return std::make_pair(0U, &X86::GR8RegClass);
59306 if (VT == MVT::i16)
59307 return std::make_pair(0U, &X86::GR16RegClass);
59308 if (VT == MVT::i32 || VT == MVT::f32)
59309 return std::make_pair(0U, &X86::GR32RegClass);
59310 if (VT != MVT::f80 && !VT.isVector())
59311 return std::make_pair(0U, &X86::GR64RegClass);
59312 break;
59313 }
59314 }
59315
59316 if (parseConstraintCode(Constraint) != X86::COND_INVALID)
59317 return std::make_pair(0U, &X86::GR32RegClass);
59318
59319 // Use the default implementation in TargetLowering to convert the register
59320 // constraint into a member of a register class.
59321 std::pair<Register, const TargetRegisterClass*> Res;
59322 Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
59323
59324 // Not found as a standard register?
59325 if (!Res.second) {
59326 // Only match x87 registers if the VT is one SelectionDAGBuilder can convert
59327 // to/from f80.
59328 if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) {
59329 // Map st(0) -> st(7) -> ST0
59330 if (Constraint.size() == 7 && Constraint[0] == '{' &&
59331 tolower(Constraint[1]) == 's' && tolower(Constraint[2]) == 't' &&
59332 Constraint[3] == '(' &&
59333 (Constraint[4] >= '0' && Constraint[4] <= '7') &&
59334 Constraint[5] == ')' && Constraint[6] == '}') {
59335 // st(7) is not allocatable and thus not a member of RFP80. Return
59336 // singleton class in cases where we have a reference to it.
59337 if (Constraint[4] == '7')
59338 return std::make_pair(X86::FP7, &X86::RFP80_7RegClass);
59339 return std::make_pair(X86::FP0 + Constraint[4] - '0',
59340 &X86::RFP80RegClass);
59341 }
59342
59343 // GCC allows "st(0)" to be called just plain "st".
59344 if (StringRef("{st}").equals_insensitive(Constraint))
59345 return std::make_pair(X86::FP0, &X86::RFP80RegClass);
59346 }
59347
59348 // flags -> EFLAGS
59349 if (StringRef("{flags}").equals_insensitive(Constraint))
59350 return std::make_pair(X86::EFLAGS, &X86::CCRRegClass);
59351
59352 // dirflag -> DF
59353 // Only allow for clobber.
59354 if (StringRef("{dirflag}").equals_insensitive(Constraint) &&
59355 VT == MVT::Other)
59356 return std::make_pair(X86::DF, &X86::DFCCRRegClass);
59357
59358 // fpsr -> FPSW
59359 // Only allow for clobber.
59360 if (StringRef("{fpsr}").equals_insensitive(Constraint) && VT == MVT::Other)
59361 return std::make_pair(X86::FPSW, &X86::FPCCRRegClass);
59362
59363 return Res;
59364 }
59365
59366 // Make sure it isn't a register that requires 64-bit mode.
59367 if (!Subtarget.is64Bit() &&
59368 (isFRClass(*Res.second) || isGRClass(*Res.second)) &&
59369 TRI->getEncodingValue(Res.first) >= 8) {
59370 // Register requires REX prefix, but we're in 32-bit mode.
59371 return std::make_pair(0, nullptr);
59372 }
59373
59374 // Make sure it isn't a register that requires AVX512.
59375 if (!Subtarget.hasAVX512() && isFRClass(*Res.second) &&
59376 TRI->getEncodingValue(Res.first) & 0x10) {
59377 // Register requires EVEX prefix.
59378 return std::make_pair(0, nullptr);
59379 }
59380
59381 // Otherwise, check to see if this is a register class of the wrong value
59382 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to
59383 // turn into {ax},{dx}.
59384 // MVT::Other is used to specify clobber names.
59385 if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other)
59386 return Res; // Correct type already, nothing to do.
59387
59388 // Get a matching integer of the correct size. i.e. "ax" with MVT::32 should
59389 // return "eax". This should even work for things like getting 64bit integer
59390 // registers when given an f64 type.
59391 const TargetRegisterClass *Class = Res.second;
59392 // The generic code will match the first register class that contains the
59393 // given register. Thus, based on the ordering of the tablegened file,
59394 // the "plain" GR classes might not come first.
59395 // Therefore, use a helper method.
59396 if (isGRClass(*Class)) {
59397 unsigned Size = VT.getSizeInBits();
59398 if (Size == 1) Size = 8;
59399 if (Size != 8 && Size != 16 && Size != 32 && Size != 64)
59400 return std::make_pair(0, nullptr);
59401 Register DestReg = getX86SubSuperRegister(Res.first, Size);
59402 if (DestReg.isValid()) {
59403 bool is64Bit = Subtarget.is64Bit();
59404 const TargetRegisterClass *RC =
59405 Size == 8 ? (is64Bit ? &X86::GR8RegClass : &X86::GR8_NOREXRegClass)
59406 : Size == 16 ? (is64Bit ? &X86::GR16RegClass : &X86::GR16_NOREXRegClass)
59407 : Size == 32 ? (is64Bit ? &X86::GR32RegClass : &X86::GR32_NOREXRegClass)
59408 : /*Size == 64*/ (is64Bit ? &X86::GR64RegClass : nullptr);
59409 if (Size == 64 && !is64Bit) {
59410 // Model GCC's behavior here and select a fixed pair of 32-bit
59411 // registers.
59412 switch (DestReg) {
59413 case X86::RAX:
59414 return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
59415 case X86::RDX:
59416 return std::make_pair(X86::EDX, &X86::GR32_DCRegClass);
59417 case X86::RCX:
59418 return std::make_pair(X86::ECX, &X86::GR32_CBRegClass);
59419 case X86::RBX:
59420 return std::make_pair(X86::EBX, &X86::GR32_BSIRegClass);
59421 case X86::RSI:
59422 return std::make_pair(X86::ESI, &X86::GR32_SIDIRegClass);
59423 case X86::RDI:
59424 return std::make_pair(X86::EDI, &X86::GR32_DIBPRegClass);
59425 case X86::RBP:
59426 return std::make_pair(X86::EBP, &X86::GR32_BPSPRegClass);
59427 default:
59428 return std::make_pair(0, nullptr);
59429 }
59430 }
59431 if (RC && RC->contains(DestReg))
59432 return std::make_pair(DestReg, RC);
59433 return Res;
59434 }
59435 // No register found/type mismatch.
59436 return std::make_pair(0, nullptr);
59437 } else if (isFRClass(*Class)) {
59438 // Handle references to XMM physical registers that got mapped into the
59439 // wrong class. This can happen with constraints like {xmm0} where the
59440 // target independent register mapper will just pick the first match it can
59441 // find, ignoring the required type.
59442
59443 // TODO: Handle f128 and i128 in FR128RegClass after it is tested well.
59444 if (VT == MVT::f16)
59445 Res.second = &X86::FR16XRegClass;
59446 else if (VT == MVT::f32 || VT == MVT::i32)
59447 Res.second = &X86::FR32XRegClass;
59448 else if (VT == MVT::f64 || VT == MVT::i64)
59449 Res.second = &X86::FR64XRegClass;
59450 else if (TRI->isTypeLegalForClass(X86::VR128XRegClass, VT))
59451 Res.second = &X86::VR128XRegClass;
59452 else if (TRI->isTypeLegalForClass(X86::VR256XRegClass, VT))
59453 Res.second = &X86::VR256XRegClass;
59454 else if (TRI->isTypeLegalForClass(X86::VR512RegClass, VT))
59455 Res.second = &X86::VR512RegClass;
59456 else {
59457 // Type mismatch and not a clobber: Return an error;
59458 Res.first = 0;
59459 Res.second = nullptr;
59460 }
59461 } else if (isVKClass(*Class)) {
59462 if (VT == MVT::v1i1 || VT == MVT::i1)
59463 Res.second = &X86::VK1RegClass;
59464 else if (VT == MVT::v8i1 || VT == MVT::i8)
59465 Res.second = &X86::VK8RegClass;
59466 else if (VT == MVT::v16i1 || VT == MVT::i16)
59467 Res.second = &X86::VK16RegClass;
59468 else if (VT == MVT::v32i1 || VT == MVT::i32)
59469 Res.second = &X86::VK32RegClass;
59470 else if (VT == MVT::v64i1 || VT == MVT::i64)
59471 Res.second = &X86::VK64RegClass;
59472 else {
59473 // Type mismatch and not a clobber: Return an error;
59474 Res.first = 0;
59475 Res.second = nullptr;
59476 }
59477 }
59478
59479 return Res;
59480}
59481
59482bool X86TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
59483 // Integer division on x86 is expensive. However, when aggressively optimizing
59484 // for code size, we prefer to use a div instruction, as it is usually smaller
59485 // than the alternative sequence.
59486 // The exception to this is vector division. Since x86 doesn't have vector
59487 // integer division, leaving the division as-is is a loss even in terms of
59488 // size, because it will have to be scalarized, while the alternative code
59489 // sequence can be performed in vector form.
59490 bool OptSize = Attr.hasFnAttr(Attribute::MinSize);
59491 return OptSize && !VT.isVector();
59492}
59493
59494void X86TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
59495 if (!Subtarget.is64Bit())
59496 return;
59497
59498 // Update IsSplitCSR in X86MachineFunctionInfo.
59499 X86MachineFunctionInfo *AFI =
59500 Entry->getParent()->getInfo<X86MachineFunctionInfo>();
59501 AFI->setIsSplitCSR(true);
59502}
59503
59504void X86TargetLowering::insertCopiesSplitCSR(
59505 MachineBasicBlock *Entry,
59506 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
59507 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
59508 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
59509 if (!IStart)
59510 return;
59511
59512 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
59513 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
59514 MachineBasicBlock::iterator MBBI = Entry->begin();
59515 for (const MCPhysReg *I = IStart; *I; ++I) {
59516 const TargetRegisterClass *RC = nullptr;
59517 if (X86::GR64RegClass.contains(*I))
59518 RC = &X86::GR64RegClass;
59519 else
59520 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
59521
59522 Register NewVR = MRI->createVirtualRegister(RC);
59523 // Create copy from CSR to a virtual register.
59524 // FIXME: this currently does not emit CFI pseudo-instructions, it works
59525 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
59526 // nounwind. If we want to generalize this later, we may need to emit
59527 // CFI pseudo-instructions.
59528 assert(
59529 Entry->getParent()->getFunction().hasFnAttribute(Attribute::NoUnwind) &&
59530 "Function should be nounwind in insertCopiesSplitCSR!");
59531 Entry->addLiveIn(*I);
59532 BuildMI(*Entry, MBBI, MIMetadata(), TII->get(TargetOpcode::COPY), NewVR)
59533 .addReg(*I);
59534
59535 // Insert the copy-back instructions right before the terminator.
59536 for (auto *Exit : Exits)
59537 BuildMI(*Exit, Exit->getFirstTerminator(), MIMetadata(),
59538 TII->get(TargetOpcode::COPY), *I)
59539 .addReg(NewVR);
59540 }
59541}
59542
59543bool X86TargetLowering::supportSwiftError() const {
59544 return Subtarget.is64Bit();
59545}
59546
59547MachineInstr *
59548X86TargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
59549 MachineBasicBlock::instr_iterator &MBBI,
59550 const TargetInstrInfo *TII) const {
59551 assert(MBBI->isCall() && MBBI->getCFIType() &&
59552 "Invalid call instruction for a KCFI check");
59553
59554 MachineFunction &MF = *MBB.getParent();
59555 // If the call target is a memory operand, unfold it and use R11 for the
59556 // call, so KCFI_CHECK won't have to recompute the address.
59557 switch (MBBI->getOpcode()) {
59558 case X86::CALL64m:
59559 case X86::CALL64m_NT:
59560 case X86::TAILJMPm64:
59561 case X86::TAILJMPm64_REX: {
59562 MachineBasicBlock::instr_iterator OrigCall = MBBI;
59563 SmallVector<MachineInstr *, 2> NewMIs;
59564 if (!TII->unfoldMemoryOperand(MF, *OrigCall, X86::R11, /*UnfoldLoad=*/true,
59565 /*UnfoldStore=*/false, NewMIs))
59566 report_fatal_error("Failed to unfold memory operand for a KCFI check");
59567 for (auto *NewMI : NewMIs)
59568 MBBI = MBB.insert(OrigCall, NewMI);
59569 assert(MBBI->isCall() &&
59570 "Unexpected instruction after memory operand unfolding");
59571 if (OrigCall->shouldUpdateCallSiteInfo())
59572 MF.moveCallSiteInfo(&*OrigCall, &*MBBI);
59573 MBBI->setCFIType(MF, OrigCall->getCFIType());
59574 OrigCall->eraseFromParent();
59575 break;
59576 }
59577 default:
59578 break;
59579 }
59580
59581 MachineOperand &Target = MBBI->getOperand(0);
59582 Register TargetReg;
59583 switch (MBBI->getOpcode()) {
59584 case X86::CALL64r:
59585 case X86::CALL64r_NT:
59586 case X86::TAILJMPr64:
59587 case X86::TAILJMPr64_REX:
59588 assert(Target.isReg() && "Unexpected target operand for an indirect call");
59589 Target.setIsRenamable(false);
59590 TargetReg = Target.getReg();
59591 break;
59592 case X86::CALL64pcrel32:
59593 case X86::TAILJMPd64:
59594 assert(Target.isSymbol() && "Unexpected target operand for a direct call");
59595 // X86TargetLowering::EmitLoweredIndirectThunk always uses r11 for
59596 // 64-bit indirect thunk calls.
59597 assert(StringRef(Target.getSymbolName()).ends_with("_r11") &&
59598 "Unexpected register for an indirect thunk call");
59599 TargetReg = X86::R11;
59600 break;
59601 default:
59602 llvm_unreachable("Unexpected CFI call opcode");
59603 break;
59604 }
59605
59606 return BuildMI(MBB, MBBI, MIMetadata(*MBBI), TII->get(X86::KCFI_CHECK))
59607 .addReg(TargetReg)
59608 .addImm(MBBI->getCFIType())
59609 .getInstr();
59610}
59611
59612/// Returns true if stack probing through a function call is requested.
59613bool X86TargetLowering::hasStackProbeSymbol(const MachineFunction &MF) const {
59614 return !getStackProbeSymbolName(MF).empty();
59615}
59616
59617/// Returns true if stack probing through inline assembly is requested.
59618bool X86TargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
59619
59620 // No inline stack probe for Windows, they have their own mechanism.
59621 if (Subtarget.isOSWindows() ||
59622 MF.getFunction().hasFnAttribute("no-stack-arg-probe"))
59623 return false;
59624
59625 // If the function specifically requests inline stack probes, emit them.
59626 if (MF.getFunction().hasFnAttribute("probe-stack"))
59627 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
59628 "inline-asm";
59629
59630 return false;
59631}
59632
59633/// Returns the name of the symbol used to emit stack probes or the empty
59634/// string if not applicable.
59635StringRef
59636X86TargetLowering::getStackProbeSymbolName(const MachineFunction &MF) const {
59637 // Inline Stack probes disable stack probe call
59638 if (hasInlineStackProbe(MF))
59639 return "";
59640
59641 // If the function specifically requests stack probes, emit them.
59642 if (MF.getFunction().hasFnAttribute("probe-stack"))
59643 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString();
59644
59645 // Generally, if we aren't on Windows, the platform ABI does not include
59646 // support for stack probes, so don't emit them.
59647 if (!Subtarget.isOSWindows() || Subtarget.isTargetMachO() ||
59648 MF.getFunction().hasFnAttribute("no-stack-arg-probe"))
59649 return "";
59650
59651 // We need a stack probe to conform to the Windows ABI. Choose the right
59652 // symbol.
59653 if (Subtarget.is64Bit())
59654 return Subtarget.isTargetCygMing() ? "___chkstk_ms" : "__chkstk";
59655 return Subtarget.isTargetCygMing() ? "_alloca" : "_chkstk";
59656}
59657
59658unsigned
59659X86TargetLowering::getStackProbeSize(const MachineFunction &MF) const {
59660 // The default stack probe size is 4096 if the function has no stackprobesize
59661 // attribute.
59662 return MF.getFunction().getFnAttributeAsParsedInteger("stack-probe-size",
59663 4096);
59664}
59665
59666Align X86TargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
59667 if (ML && ML->isInnermost() &&
59668 ExperimentalPrefInnermostLoopAlignment.getNumOccurrences())
59669 return Align(1ULL << ExperimentalPrefInnermostLoopAlignment);
59670 return TargetLowering::getPrefLoopAlignment();
59671}
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
Success
#define Success
Definition: AArch64Disassembler.cpp:213
Widen
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
Definition: AArch64ISelDAGToDAG.cpp:1180
parseConstraintCode
static AArch64CC::CondCode parseConstraintCode(llvm::StringRef Constraint)
Definition: AArch64ISelLowering.cpp:11677
LowerFunnelShift
static SDValue LowerFunnelShift(SDValue Op, SelectionDAG &DAG)
Definition: AArch64ISelLowering.cpp:6775
getSETCC
static SDValue getSETCC(AArch64CC::CondCode CC, SDValue NZCV, const SDLoc &DL, SelectionDAG &DAG)
Helper function to create 'CSET', which is equivalent to 'CSINC <Wd>, WZR, WZR, invert(<cond>)'.
Definition: AArch64ISelLowering.cpp:11701
LowerPREFETCH
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
Definition: AArch64ISelLowering.cpp:4265
LowerXALUO
static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG)
Definition: AArch64ISelLowering.cpp:4234
foldVectorXorShiftIntoCmp
static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X,...
Definition: AArch64ISelLowering.cpp:18057
RegSize
unsigned RegSize
Definition: AArch64MIPeepholeOpt.cpp:157
Wrapper
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
Definition: AMDGPUAliasAnalysis.cpp:31
getNode
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
Definition: AMDGPUDelayedMCExpr.cpp:15
NODE_NAME_CASE
#define NODE_NAME_CASE(node)
Definition: AMDGPUISelLowering.cpp:5399
S1
static const LLT S1
Definition: AMDGPULegalizerInfo.cpp:282
F64
static const LLT F64
Definition: AMDGPULegalizerInfo.cpp:288
Select
amdgpu AMDGPU Register Bank Select
Definition: AMDGPURegBankSelect.cpp:46
LowerShift
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
Definition: ARMISelLowering.cpp:6670
LowerCONCAT_VECTORS
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
Definition: ARMISelLowering.cpp:9235
LowerEXTRACT_SUBVECTOR
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
Definition: ARMISelLowering.cpp:9260
LowerATOMIC_FENCE
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
Definition: ARMISelLowering.cpp:4276
LowerCTTZ
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST)
Definition: ARMISelLowering.cpp:6528
getZeroVector
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
getZeroVector - Returns a vector of specified type with all zero elements.
Definition: ARMISelLowering.cpp:6320
LowerMUL
static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG)
Definition: ARMISelLowering.cpp:9636
LowerMLOAD
static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG)
Definition: ARMISelLowering.cpp:10273
LowerVSETCC
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
Definition: ARMISelLowering.cpp:6790
MBB
MachineBasicBlock & MBB
Definition: ARMSLSHardening.cpp:71
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: ARMSLSHardening.cpp:73
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: ARMSLSHardening.cpp:72
EXPAND
#define EXPAND(Op)
Results
Function Alias Analysis Results
Definition: AliasAnalysis.cpp:769
BT
BitTracker BT
Definition: BitTracker.cpp:73
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:57
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
D
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
LLVM_ATTRIBUTE_UNUSED
#define LLVM_ATTRIBUTE_UNUSED
Definition: Compiler.h:199
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
RetTy
return RetTy
Definition: DeadArgumentElimination.cpp:360
Idx
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Definition: DeadArgumentElimination.cpp:352
ignored
Looks at all the uses of the given value Returns the Liveness deduced from the uses of this value Adds all uses that cause the result to be MaybeLive to MaybeLiveRetUses If the result is MaybeLiveUses might be modified but its content should be ignored(since it might not be complete). DeadArgumentEliminationPass
Definition: DeadArgumentElimination.cpp:472
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:79
Name
std::string Name
Definition: ELFObjHandler.cpp:77
Size
uint64_t Size
Definition: ELFObjHandler.cpp:81
Sym
Symbol * Sym
Definition: ELF_riscv.cpp:479
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
isSigned
static bool isSigned(unsigned int Opcode)
Definition: ExpandLargeDivRem.cpp:53
extractBits
static KnownBits extractBits(unsigned BitWidth, const KnownBits &SrcOpKnown, const KnownBits &OffsetKnown, const KnownBits &WidthKnown)
Definition: GISelKnownBits.cpp:133
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
isUndef
static bool isUndef(ArrayRef< int > Mask)
Definition: HexagonISelDAGToDAGHVX.cpp:1032
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:113
MaxDepth
static const unsigned MaxDepth
Definition: InstCombineMulDivRem.cpp:1395
ShuffleOps
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
Definition: InstCombineVectorOps.cpp:779
matchShuffleAsBitRotate
static int matchShuffleAsBitRotate(ArrayRef< int > Mask, int NumSubElts)
Try to lower a vector shuffle as a bit rotation.
Definition: Instructions.cpp:2404
LowerCTLZ
static Value * LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctlz of V before the specified instruction IP.
Definition: IntrinsicLowering.cpp:187
LowerCTPOP
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
Definition: IntrinsicLowering.cpp:148
RegName
#define RegName(no)
Options
static LVOptions Options
Definition: LVOptions.cpp:25
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:512
Matrix
Live Register Matrix
Definition: LiveRegMatrix.cpp:44
lower256BitShuffle
static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Dispatching routine to lower various 256-bit LoongArch vector shuffles.
Definition: LoongArchISelLowering.cpp:1255
lower128BitShuffle
static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Dispatching routine to lower various 128-bit LoongArch vector shuffles.
Definition: LoongArchISelLowering.cpp:830
isAndOrOfSetCCs
static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc)
Return true if node is an ISD::AND or ISD::OR of two M68k::SETcc nodes each of which has no other use...
Definition: M68kISelLowering.cpp:2429
hasNonFlagsUse
static bool hasNonFlagsUse(SDValue Op)
return true if Op has a use that doesn't just read flags.
Definition: M68kISelLowering.cpp:1849
isCMOVPseudo
static bool isCMOVPseudo(MachineInstr &MI)
Definition: M68kISelLowering.cpp:3054
combineCarryThroughADD
static SDValue combineCarryThroughADD(SDValue CCR)
Definition: M68kISelLowering.cpp:3538
isTruncWithZeroHighBitsInput
static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG)
Definition: M68kISelLowering.cpp:2249
F
#define F(x, y, z)
Definition: MD5.cpp:55
I
#define I(x, y, z)
Definition: MD5.cpp:58
G
#define G(x, y, z)
Definition: MD5.cpp:56
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1928
R2
#define R2(n)
T1
#define T1
Definition: Mips16ISelLowering.cpp:340
High
uint64_t High
Definition: NVVMIntrRange.cpp:52
II
uint64_t IntrinsicInst * II
Definition: NVVMIntrRange.cpp:52
ObjCARCUtil.h
This file defines ARC utility functions which are used by various parts of the compiler.
Y
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
P
#define P(N)
Operation
PowerPC Reduce CR logical Operation
Definition: PPCReduceCRLogicals.cpp:735
Fixup
PowerPC TLS Dynamic Call Fixup
Definition: PPCTLSDynamicCall.cpp:340
if
if(PassOpts->AAPipeline)
Definition: PassBuilderBindings.cpp:64
lowerVECTOR_SHUFFLE
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Definition: RISCVISelLowering.cpp:5001
negateFMAOpcode
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
Definition: RISCVISelLowering.cpp:15153
Cond
const SmallVectorImpl< MachineOperand > & Cond
Definition: RISCVRedundantCopyElimination.cpp:75
isValid
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
Definition: RustDemangle.cpp:181
SDPatternMatch.h
Contains matchers for matching SelectionDAG nodes and values.
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
isSimple
static bool isSimple(Instruction *I)
Definition: SLPVectorizer.cpp:1056
OpIndex
unsigned OpIndex
Definition: SPIRVModuleAnalysis.cpp:64
substr
static StringRef substr(StringRef Str, uint64_t Len)
Definition: SimplifyLibCalls.cpp:350
SmallBitVector.h
This file implements the SmallBitVector class.
SmallSet.h
This file defines the SmallSet class.
Enabled
static bool Enabled
Definition: Statistic.cpp:46
Statistic.h
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
StringExtras.h
This file contains some functions that are useful when dealing with strings.
StringSwitch.h
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
TargetLowering.h
This file describes how to lower LLVM code to machine code.
getOpcode
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:191
getBitWidth
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
Definition: ValueTracking.cpp:94
computeKnownBitsForHorizontalOperation
static KnownBits computeKnownBitsForHorizontalOperation(const Operator *I, const APInt &DemandedElts, unsigned Depth, const SimplifyQuery &Q, const function_ref< KnownBits(const KnownBits &, const KnownBits &)> KnownBitsFunc)
Definition: ValueTracking.cpp:968
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition: Value.cpp:469
extractSubVector
static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &DL, unsigned VectorWidth)
Definition: WebAssemblyISelLowering.cpp:2703
is64Bit
static bool is64Bit(const char *name)
Definition: X86Disassembler.cpp:1078
GET_EGPR_IF_ENABLED
#define GET_EGPR_IF_ENABLED(OPC)
getSUBriOpcode
static unsigned getSUBriOpcode(bool IsLP64)
Definition: X86FrameLowering.cpp:110
isNoopOrBroadcastShuffleMask
static bool isNoopOrBroadcastShuffleMask(ArrayRef< int > Mask)
Definition: X86ISelLowering.cpp:11129
matchLogicBlend
static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask)
Definition: X86ISelLowering.cpp:50297
widenMaskVectorType
static MVT widenMaskVectorType(MVT VT, const X86Subtarget &Subtarget)
Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer t...
Definition: X86ISelLowering.cpp:4033
combineAndLoadToBZHI
static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:49598
combineOrCmpEqZeroToCtlzSrl
static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:50412
combineAndnp
static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Do target-specific dag combines on X86ISD::ANDNP nodes.
Definition: X86ISelLowering.cpp:53563
combineExtractWithShuffle
static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:44799
LowerStore
static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:24675
createPSADBW
static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0, const SDValue &Zext1, const SDLoc &DL, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:44365
combineAddOrSubToADCOrSBB
static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT, SDValue X, SDValue Y, SelectionDAG &DAG, bool ZeroSecondOpOnly=false)
If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to...
Definition: X86ISelLowering.cpp:50544
matchScalarReduction
static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp, SmallVectorImpl< SDValue > &SrcOps, SmallVectorImpl< APInt > *SrcMask=nullptr)
Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...)) style scalarized (associative) ...
Definition: X86ISelLowering.cpp:22226
combineSubABS
static SDValue combineSubABS(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:55965
translateX86FSETCC
static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1, bool &IsAlwaysSignaling)
Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs.
Definition: X86ISelLowering.cpp:23053
detectPMADDUBSW
static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
Definition: X86ISelLowering.cpp:52584
checkBoolTestSetCCCombine
static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC)
Definition: X86ISelLowering.cpp:46715
useEGPRInlineAsm
static bool useEGPRInlineAsm(const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:58973
getNullFPConstForNullVal
static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements),...
Definition: X86ISelLowering.cpp:53312
matchBinaryPermuteShuffle
static bool matchBinaryPermuteShuffle(MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
Definition: X86ISelLowering.cpp:38407
combineSub
static SDValue combineSub(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:56055
isGRClass
static bool isGRClass(const TargetRegisterClass &RC)
Check if RC is a general purpose register class.
Definition: X86ISelLowering.cpp:58942
getTargetShuffleMask
static bool getTargetShuffleMask(SDValue N, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary)
Calculates the shuffle mask corresponding to the target-specific opcode.
Definition: X86ISelLowering.cpp:5228
vectorizeExtractedCast
static SDValue vectorizeExtractedCast(SDValue Cast, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Given a scalar cast operation that is extracted from a vector, try to vectorize the cast op followed ...
Definition: X86ISelLowering.cpp:19127
combineShiftRightLogical
static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48352
lowerShuffleAsInsertPS
static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:12772
combineSubSetcc
static SDValue combineSubSetcc(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:56002
match1BitShuffleAsKSHIFT
static int match1BitShuffleAsKSHIFT(unsigned &Opcode, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable)
Definition: X86ISelLowering.cpp:17321
isHorizontalBinOpPart
static bool isHorizontalBinOpPart(const BuildVectorSDNode *N, unsigned Opcode, const SDLoc &DL, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1)
This is a helper function of LowerToHorizontalOp().
Definition: X86ISelLowering.cpp:7758
SplitAndExtendv16i1
static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In, const SDLoc &dl, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:20218
getShuffleHalfVectors
static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > HalfMask, int HalfIdx1, int HalfIdx2, bool UndefLower, SelectionDAG &DAG, bool UseConcat=false)
Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors f...
Definition: X86ISelLowering.cpp:15290
combineSignExtendInReg
static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53814
getTargetVShiftNode
static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, int ShAmtIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle vector element shifts by a splat shift amount.
Definition: X86ISelLowering.cpp:25241
ConstantBit
@ ConstantBit
Definition: X86ISelLowering.cpp:30560
NotConstantBit
@ NotConstantBit
Definition: X86ISelLowering.cpp:30561
NotShiftBit
@ NotShiftBit
Definition: X86ISelLowering.cpp:30563
ShiftBit
@ ShiftBit
Definition: X86ISelLowering.cpp:30562
UndefBit
@ UndefBit
Definition: X86ISelLowering.cpp:30559
combineZext
static SDValue combineZext(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:54242
incDecVectorConstant
static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc, bool NSW)
Given a buildvector constant, return a new vector constant with each element incremented or decrement...
Definition: X86ISelLowering.cpp:23164
isAddSubOrSubAdd
static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, unsigned &NumExtracts, bool &IsSubAdd)
Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operati...
Definition: X86ISelLowering.cpp:7913
cheapX86FSETCC_SSE
static bool cheapX86FSETCC_SSE(ISD::CondCode SetCCOpcode)
Definition: X86ISelLowering.cpp:23047
lowerV4F32Shuffle
static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Lower 4-lane 32-bit floating point shuffles.
Definition: X86ISelLowering.cpp:13071
emitXBegin
static MachineBasicBlock * emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII)
Utility function to emit xbegin specifying the start of an RTM region.
Definition: X86ISelLowering.cpp:34545
EltsFromConsecutiveLoads
static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef< SDValue > Elts, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize)
Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced ...
Definition: X86ISelLowering.cpp:6898
scaleShuffleElements
static bool scaleShuffleElements(ArrayRef< int > Mask, unsigned NumDstElts, SmallVectorImpl< int > &ScaledMask)
Definition: X86ISelLowering.cpp:3747
LowerANY_EXTEND
static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:24414
LowerTruncateVecPackWithSignBits
static SDValue LowerTruncateVecPackWithSignBits(MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
This function lowers a vector truncation of 'extended sign-bits' or 'extended zero-bits' values.
Definition: X86ISelLowering.cpp:20538
BrMergingCcmpBias
static cl::opt< int > BrMergingCcmpBias("x86-br-merging-ccmp-bias", cl::init(6), cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target " "supports conditional compare instructions."), cl::Hidden)
getExtractedDemandedElts
static APInt getExtractedDemandedElts(SDNode *N)
Definition: X86ISelLowering.cpp:17993
combineBitOpWithPACK
static SDValue combineBitOpWithPACK(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:49460
LowerADJUST_TRAMPOLINE
static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:27496
lowerV8I32Shuffle
static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 8-lane 32-bit integer shuffles.
Definition: X86ISelLowering.cpp:16188
LowerVectorCTLZInRegLUT
static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:27975
combinePTESTCC
static SDValue combinePTESTCC(SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
If we are inverting an PTEST/TESTP operand, attempt to adjust the CC to avoid the inversion.
Definition: X86ISelLowering.cpp:46933
getAltBitOpcode
static unsigned getAltBitOpcode(unsigned Opcode)
Definition: X86ISelLowering.cpp:43508
getConstantVector
static Constant * getConstantVector(MVT VT, ArrayRef< APInt > Bits, const APInt &Undefs, LLVMContext &C)
Definition: X86ISelLowering.cpp:7197
LowerABD
static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:28446
LowerREADCYCLECOUNTER
static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:26725
combineBitcast
static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:43977
promoteXINT_TO_FP
static SDValue promoteXINT_TO_FP(SDValue Op, const SDLoc &dl, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:19305
combineCastedMaskArithmetic
static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:43752
insert1BitVector
static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Insert i1-subvector to i1-vector.
Definition: X86ISelLowering.cpp:4327
materializeVectorConstant
static SDValue materializeVectorConstant(SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Create a vector constant without a load.
Definition: X86ISelLowering.cpp:8436
lowerShuffleWithPSHUFB
static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower a shuffle with a single PSHUFB of V1 or V2.
Definition: X86ISelLowering.cpp:9838
combineFP16_TO_FP
static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:57624
combineBMILogicOp
static SDValue combineBMILogicOp(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:49805
LowerUINT_TO_FP_i64
static SDValue LowerUINT_TO_FP_i64(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
64-bit unsigned integer to double expansion.
Definition: X86ISelLowering.cpp:19494
useVectorCast
static bool useVectorCast(unsigned Opcode, MVT FromVT, MVT ToVT, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:19102
isX86CCSigned
static bool isX86CCSigned(unsigned X86CC)
Return true if the condition is an signed comparison operation.
Definition: X86ISelLowering.cpp:2802
LowerFP_TO_FP16
static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:21676
lowerV4X128Shuffle
static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower a vector shuffle as a 128-bit shuffles.
Definition: X86ISelLowering.cpp:16647
combineBitOpWithShift
static SDValue combineBitOpWithShift(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:49417
combineSelect
static SDValue combineSelect(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Do target-specific dag combines on SELECT and VSELECT nodes.
Definition: X86ISelLowering.cpp:45969
isUndefOrZeroInRange
static bool isUndefOrZeroInRange(ArrayRef< int > Mask, unsigned Pos, unsigned Size)
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or ...
Definition: X86ISelLowering.cpp:3649
combineToConsecutiveLoads
static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize)
Definition: X86ISelLowering.cpp:7180
LowerCONCAT_VECTORSvXi1
static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:9283
getConstVector
static SDValue getConstVector(ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false)
Definition: X86ISelLowering.cpp:3788
commuteSelect
static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:45938
createPHIsForCMOVsInSinkBB
static MachineInstrBuilder createPHIsForCMOVsInSinkBB(MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB)
Definition: X86ISelLowering.cpp:34966
insert128BitVector
static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
Generate a DAG to put 128-bits into a vector > 128 bits.
Definition: X86ISelLowering.cpp:3997
onlyZeroFlagUsed
static bool onlyZeroFlagUsed(SDValue Flags)
Definition: X86ISelLowering.cpp:55238
extract256BitVector
static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
Generate a DAG to grab 256-bits from a 512-bit vector.
Definition: X86ISelLowering.cpp:3961
combineFAndFNotToFAndn
static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53323
combineGatherScatter
static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:54785
combineMulToPMADDWD
static SDValue combineMulToPMADDWD(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:47827
LowerINTRINSIC_W_CHAIN
static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:26802
lowerShuffleAsLanePermuteAndShuffle
static SDValue lowerShuffleAsLanePermuteAndShuffle(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one source with a lane permutatio...
Definition: X86ISelLowering.cpp:14887
checkSignTestSetCCCombine
static SDValue checkSignTestSetCCCombine(SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:46646
isFoldableUseOfShuffle
static bool isFoldableUseOfShuffle(SDNode *N)
Definition: X86ISelLowering.cpp:7250
getTargetShuffleInputs
static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
Definition: X86ISelLowering.cpp:6349
getVectorMaskingNode
static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along w...
Definition: X86ISelLowering.cpp:25372
lowerAddSub
static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:28143
truncateVectorWithPACKSS
static SDValue truncateVectorWithPACKSS(EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Truncate using inreg sign extension and X86ISD::PACKSS.
Definition: X86ISelLowering.cpp:20436
combineMaskedLoadConstantMask
static SDValue combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:51528
isShuffleMaskInputInPlace
static bool isShuffleMaskInputInPlace(int Input, ArrayRef< int > Mask)
Test whether the specified input (0 or 1) is in-place blended by the given mask.
Definition: X86ISelLowering.cpp:12423
isMultiLaneShuffleMask
static bool isMultiLaneShuffleMask(unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
Test whether elements in each LaneSizeInBits lane in this shuffle mask come from multiple lanes - thi...
Definition: X86ISelLowering.cpp:9431
LowerVSETCCWithSUBUS
static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT, ISD::CondCode Cond, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
As another special case, use PSUBUS[BW] when it's profitable.
Definition: X86ISelLowering.cpp:23198
combineFP_EXTEND
static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:57646
is128BitLaneRepeatedShuffleMask
static bool is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
Test whether a shuffle mask is equivalent within each 128-bit lane.
Definition: X86ISelLowering.cpp:9498
getPMOVMSKB
static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:31155
combineUIntToFP
static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:55018
getPackDemandedElts
static void getPackDemandedElts(EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
Definition: X86ISelLowering.cpp:5188
LowerVectorCTPOPInRegLUT
static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:31315
combineADC
static SDValue combineADC(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:55477
CastIntSETCCtoFP
static std::optional< unsigned > CastIntSETCCtoFP(MVT VT, ISD::CondCode CC, unsigned NumSignificantBitsLHS, unsigned NumSignificantBitsRHS)
Definition: X86ISelLowering.cpp:56170
lowerShuffleAsVTRUNCAndUnpack
static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:15712
isShuffleFoldableLoad
static bool isShuffleFoldableLoad(SDValue V)
Helper to test for a load that can be folded with x86 shuffles.
Definition: X86ISelLowering.cpp:12206
narrowVectorSelect
static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
If both arms of a vector select are concatenated vectors, split the select, and concatenate the resul...
Definition: X86ISelLowering.cpp:45679
lowerShuffleAsElementInsertion
static SDValue lowerShuffleAsElementInsertion(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower insertion of a single element into a zero vector.
Definition: X86ISelLowering.cpp:12221
combineXor
static SDValue combineXor(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53143
isUnpackWdShuffleMask
static bool isUnpackWdShuffleMask(ArrayRef< int > Mask, MVT VT, const SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:9717
LowerTruncateVecPack
static SDValue LowerTruncateVecPack(MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::P...
Definition: X86ISelLowering.cpp:20572
lowerShuffleAsRepeatedMaskAndLanePermute
static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represe...
Definition: X86ISelLowering.cpp:15441
computeKnownBitsForPSADBW
static void computeKnownBitsForPSADBW(SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
Definition: X86ISelLowering.cpp:37188
getSEHRegistrationNodeSize
static int getSEHRegistrationNodeSize(const Function *Fn)
Definition: X86ISelLowering.cpp:25424
combineShuffleOfConcatUndef
static SDValue combineShuffleOfConcatUndef(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:41761
getScalarMaskingNode
static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Creates an SDNode for a predicated scalar operation.
Definition: X86ISelLowering.cpp:25398
buildFromShuffleMostly
static SDValue buildFromShuffleMostly(SDValue Op, const SDLoc &DL, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:7548
lowerBuildVectorToBitOp
static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is con...
Definition: X86ISelLowering.cpp:8359
LowerZERO_EXTEND
static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:20288
isFNEG
static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth=0)
Returns the negated value if the node N flips sign of FP value.
Definition: X86ISelLowering.cpp:52792
lowerV16I16Shuffle
static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 16-lane 16-bit integer shuffles.
Definition: X86ISelLowering.cpp:16330
lowerShuffleWithUndefHalf
static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
Definition: X86ISelLowering.cpp:15330
lowerAtomicArith
static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Lower atomic_load_ops into LOCK-prefixed operations.
Definition: X86ISelLowering.cpp:31713
convertShiftLeftToScale
static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:29447
lowerV32I8Shuffle
static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 32-lane 8-bit integer shuffles.
Definition: X86ISelLowering.cpp:16453
checkAndUpdateEFLAGSKill
static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
Definition: X86ISelLowering.cpp:34912
computeKnownBitsForPMADDWD
static void computeKnownBitsForPMADDWD(SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
Definition: X86ISelLowering.cpp:37205
LowerBITCAST
static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:31183
LowerEXTRACT_VECTOR_ELT_SSE4
static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:17896
lowerShuffleAsTruncBroadcast
static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/buil...
Definition: X86ISelLowering.cpp:12347
ExpandHorizontalBinOp
static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI)
Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.
Definition: X86ISelLowering.cpp:7872
LowerToHorizontalOp
static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
Definition: X86ISelLowering.cpp:8235
getGFNICtrlMask
SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT VT, unsigned Amt=0)
Definition: X86ISelLowering.cpp:29144
combineFP_ROUND
static SDValue combineFP_ROUND(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:57779
GetTLSADDR
static SDValue GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InGlue, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false)
Definition: X86ISelLowering.cpp:18669
combineAndShuffleNot
static SDValue combineAndShuffleNot(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Try to fold: and (vector_shuffle<Z,...,Z> (insert_vector_elt undef, (xor X, -1), Z),...
Definition: X86ISelLowering.cpp:49134
lowerShuffleAsBitMask
static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to emit a bitmask instruction for a shuffle.
Definition: X86ISelLowering.cpp:10485
lowerShuffleWithUNPCK256
static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit) followed by unpack 256-bit.
Definition: X86ISelLowering.cpp:10035
is256BitLaneRepeatedShuffleMask
static bool is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
Test whether a shuffle mask is equivalent within each 256-bit lane.
Definition: X86ISelLowering.cpp:9511
LowerMSCATTER
static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:32000
LowerShiftByScalarVariable
static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:29383
LowerSIGN_EXTEND_Mask
static SDValue LowerSIGN_EXTEND_Mask(SDValue Op, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:24359
getVectorShuffle
static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl, SDValue V1, SDValue V2, ArrayRef< int > Mask)
Definition: X86ISelLowering.cpp:4580
matchPMADDWD
static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:55535
combineMaskedStore
static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:51655
combineCommutableSHUFP
static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:40406
LowerUINT_TO_FP_i32
static SDValue LowerUINT_TO_FP_i32(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
32-bit unsigned integer to float expansion.
Definition: X86ISelLowering.cpp:19562
combineAdd
static SDValue combineAdd(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:55892
LowerTruncateVecI1
static SDValue LowerTruncateVecI1(SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:20623
combineVTRUNC
static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:52764
LowerLoad
static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:24762
ExperimentalPrefInnermostLoopAlignment
static cl::opt< int > ExperimentalPrefInnermostLoopAlignment("x86-experimental-pref-innermost-loop-alignment", cl::init(4), cl::desc("Sets the preferable loop alignment for experiments (as log2 bytes) " "for innermost loops only. If specified, this option overrides " "alignment set by x86-experimental-pref-loop-alignment."), cl::Hidden)
createVariablePermute
static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source v...
Definition: X86ISelLowering.cpp:8461
getHopForBuildVector
static SDValue getHopForBuildVector(const BuildVectorSDNode *BV, const SDLoc &DL, SelectionDAG &DAG, unsigned HOpcode, SDValue V0, SDValue V1)
Definition: X86ISelLowering.cpp:8197
LowerRotate
static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:30189
lowerShuffleAsSpecificZeroOrAnyExtend
static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Lower a vector shuffle as a zero or any extension.
Definition: X86ISelLowering.cpp:11895
needCarryOrOverflowFlag
static bool needCarryOrOverflowFlag(SDValue Flags)
Definition: X86ISelLowering.cpp:55203
combineCVTPH2PS
static SDValue combineCVTPH2PS(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:53712
getOnesVector
static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl)
Returns a vector of specified type with all bits set.
Definition: X86ISelLowering.cpp:4507
combineMaskedLoad
static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:51576
isUndefLowerHalf
static bool isUndefLowerHalf(ArrayRef< int > Mask)
Return true if the mask creates a vector whose lower half is undefined.
Definition: X86ISelLowering.cpp:3561
LowerMULH
static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:28746
combineRedundantDWordShuffle
static SDValue combineRedundantDWordShuffle(SDValue N, MutableArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
Search for a combinable shuffle across a chain ending in pshufd.
Definition: X86ISelLowering.cpp:40276
getBMIMatchingOp
static SDValue getBMIMatchingOp(unsigned Opc, SelectionDAG &DAG, SDValue OpMustEq, SDValue Op, unsigned Depth)
Definition: X86ISelLowering.cpp:49760
getTargetVShiftByConstNode
static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG)
Handle vector element shifts where the shift amount is a constant.
Definition: X86ISelLowering.cpp:25189
getPack
static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS, bool PackHiHalf=false)
Returns a node that packs the LHS + RHS nodes together at half width.
Definition: X86ISelLowering.cpp:4619
combineMOVDQ2Q
static SDValue combineMOVDQ2Q(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:57870
matchUnaryShuffle
static bool matchUnaryShuffle(MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue V1, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT)
Definition: X86ISelLowering.cpp:37908
lowerFPToIntToFP
static SDValue lowerFPToIntToFP(SDValue CastToFP, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Given a scalar cast to FP with a cast to integer operand (almost an ftrunc), try to vectorize the cas...
Definition: X86ISelLowering.cpp:19169
combineX86SubCmpForFlags
static SDValue combineX86SubCmpForFlags(SDNode *N, SDValue Flag, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST)
Definition: X86ISelLowering.cpp:49824
getHalfShuffleMask
static bool getHalfShuffleMask(ArrayRef< int > Mask, MutableArrayRef< int > HalfMask, int &HalfIdx1, int &HalfIdx2)
If the input shuffle mask results in a vector that is undefined in all upper or lower half elements a...
Definition: X86ISelLowering.cpp:15239
BrMergingBaseCostThresh
static cl::opt< int > BrMergingBaseCostThresh("x86-br-merging-base-cost", cl::init(2), cl::desc("Sets the cost threshold for when multiple conditionals will be merged " "into one branch versus be split in multiple branches. Merging " "conditionals saves branches at the cost of additional instructions. " "This value sets the instruction cost limit, below which conditionals " "will be merged, and above which conditionals will be split. Set to -1 " "to never merge branches."), cl::Hidden)
getFauxShuffleMask
static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
Definition: X86ISelLowering.cpp:5807
hasBZHI
static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT)
Definition: X86ISelLowering.cpp:49583
emitLockedStackOp
static SDValue emitLockedStackOp(SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue Chain, const SDLoc &DL)
Emit a locked operation on a stack location which does not change any memory location,...
Definition: X86ISelLowering.cpp:31032
matchShuffleWithSHUFPD
static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2, bool &ForceV1Zero, bool &ForceV2Zero, unsigned &ShuffleImm, ArrayRef< int > Mask, const APInt &Zeroable)
Definition: X86ISelLowering.cpp:15640
lowerV8F16Shuffle
static SDValue lowerV8F16Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Lower 8-lane 16-bit floating point shuffles.
Definition: X86ISelLowering.cpp:14077
LowerFSINCOS
static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:31895
lowerAtomicArithWithLOCK
static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:31681
lowerShuffleAsBitBlend
static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
Try to emit a blend instruction for a shuffle using bit math.
Definition: X86ISelLowering.cpp:10540
reduceMaskedLoadToScalarLoad
static SDValue reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and ve...
Definition: X86ISelLowering.cpp:51487
lower1BitShuffleAsKSHIFTR
static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:17280
combineVectorPack
static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48572
expandIntrinsicWChainHelper
static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, unsigned TargetOpcode, unsigned SrcReg, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX.
Definition: X86ISelLowering.cpp:26649
matchUnaryPermuteShuffle
static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
Definition: X86ISelLowering.cpp:38062
shouldExpandCmpArithRMWInIR
static bool shouldExpandCmpArithRMWInIR(AtomicRMWInst *AI)
Definition: X86ISelLowering.cpp:30787
combineVSelectToBLENDV
static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
If this is a dynamic select (non-constant condition) and we can match this node with one of the varia...
Definition: X86ISelLowering.cpp:45784
combineVectorCompareAndMaskUnaryOp
static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:54928
LowerBuildVectorAsInsert
static SDValue LowerBuildVectorAsInsert(SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:6500
isRepeatedTargetShuffleMask
static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, unsigned EltSizeInBits, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
Test whether a target shuffle mask is equivalent within each sub-lane.
Definition: X86ISelLowering.cpp:9518
combineVectorSizedSetCCEquality
static SDValue combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Try to map a 128-bit or larger integer comparison to vector instructions before type legalization spl...
Definition: X86ISelLowering.cpp:22075
isLaneCrossingShuffleMask
static bool isLaneCrossingShuffleMask(unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
Test whether there are elements crossing LaneSizeInBits lanes in this shuffle mask.
Definition: X86ISelLowering.cpp:9408
FixupMMXIntrinsicTypes
static SDValue FixupMMXIntrinsicTypes(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:57904
getScatterNode
static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:26588
LowerAndToBT
static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, X86::CondCode &X86CC)
Result of 'and' is compared against zero.
Definition: X86ISelLowering.cpp:22979
combinePMULH
static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:52500
lowerShuffleAsZeroOrAnyExtend
static SDValue lowerShuffleAsZeroOrAnyExtend(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower a vector shuffle as a zero extension on any microarch.
Definition: X86ISelLowering.cpp:12059
combineBasicSADPattern
static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:44679
supportedVectorShiftWithBaseAmnt
static bool supportedVectorShiftWithBaseAmnt(EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
Definition: X86ISelLowering.cpp:29188
combineVPMADD
static SDValue combineVPMADD(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:57493
LowerMULO
static SDValue LowerMULO(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:28853
lowerShuffleWithSHUFPD
static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:15685
LowerHorizontalByteSum
static SDValue LowerHorizontalByteSum(SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Compute the horizontal sum of bytes in V for the elements of VT.
Definition: X86ISelLowering.cpp:31251
LowerFP16_TO_FP
static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:21647
lowerV32I16Shuffle
static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 32-lane 16-bit integer shuffles.
Definition: X86ISelLowering.cpp:17044
combineBitcastToBoolVector
static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned Depth=0)
Definition: X86ISelLowering.cpp:43888
getPrefetchNode
static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:26619
MarkEHGuard
static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:26752
combineX86CloadCstore
static SDValue combineX86CloadCstore(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:56029
computeInLaneShuffleMask
static void computeInLaneShuffleMask(const ArrayRef< int > &Mask, int LaneSize, SmallVector< int > &InLaneMask)
Helper to get compute inlane shuffle mask for a complete shuffle mask.
Definition: X86ISelLowering.cpp:14867
lowerX86CmpEqZeroToCtlzSrl
static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:50391
combineVectorInsert
static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48957
combineTESTP
static SDValue combineTESTP(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:54727
combineAnd
static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:49966
getBROADCAST_LOAD
static SDValue getBROADCAST_LOAD(unsigned Opcode, const SDLoc &DL, EVT VT, EVT MemVT, MemSDNode *Mem, unsigned Offset, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:6374
isUndefUpperHalf
static bool isUndefUpperHalf(ArrayRef< int > Mask)
Return true if the mask creates a vector whose upper half is undefined.
Definition: X86ISelLowering.cpp:3567
LowerATOMIC_STORE
static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:31781
getGFNICtrlImm
uint64_t getGFNICtrlImm(unsigned Opcode, unsigned Amt=0)
Definition: X86ISelLowering.cpp:29121
lowerShuffleWithPACK
static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:10435
LowerShiftParts
static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG)
Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shi...
Definition: X86ISelLowering.cpp:19017
combineFMulcFCMulc
static SDValue combineFMulcFCMulc(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:52271
canReduceVMulWidth
static bool canReduceVMulWidth(SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode)
Definition: X86ISelLowering.cpp:47607
getX86XALUOOp
static std::pair< SDValue, SDValue > getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:23963
matchPMADDWD_2
static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:55643
combineVectorShiftVar
static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48771
LowerAVG
static SDValue LowerAVG(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:28266
combineSetCC
static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:54321
combineCompareEqual
static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP,...
Definition: X86ISelLowering.cpp:48996
isVKClass
static bool isVKClass(const TargetRegisterClass &RC)
Check if RC is a mask register class.
Definition: X86ISelLowering.cpp:58963
canLowerByDroppingElements
static int canLowerByDroppingElements(ArrayRef< int > Mask, bool MatchEven, bool IsSingleInput)
Check whether a compaction lowering can be done by dropping even/odd elements and compute how many ti...
Definition: X86ISelLowering.cpp:10314
combineTruncatedArithmetic
static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.
Definition: X86ISelLowering.cpp:52419
lowerShuffleAsBroadcast
static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower broadcast of a single element.
Definition: X86ISelLowering.cpp:12490
combineCVTP2I_CVTTP2I
static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:53526
resolveTargetShuffleInputsAndMask
static void resolveTargetShuffleInputsAndMask(SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask)
Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
Definition: X86ISelLowering.cpp:6279
lowerV64I8Shuffle
static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 64-lane 8-bit integer shuffles.
Definition: X86ISelLowering.cpp:17108
combineShuffleToAddSubOrFMAddSub
static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.
Definition: X86ISelLowering.cpp:41720
lowerShuffleAsLanePermuteAndPermute
static SDValue lowerShuffleAsLanePermuteAndPermute(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane p...
Definition: X86ISelLowering.cpp:14764
combineScalarToVector
static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:57357
lowerV8I16Shuffle
static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Generic lowering of 8-lane i16 shuffles.
Definition: X86ISelLowering.cpp:13873
getEXTEND_VECTOR_INREG
static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue In, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:4515
canonicalizeShuffleMaskWithCommute
static bool canonicalizeShuffleMaskWithCommute(ArrayRef< int > Mask)
Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.
Definition: X86ISelLowering.cpp:17495
matchAsm
static bool matchAsm(StringRef S, ArrayRef< const char * > Pieces)
Definition: X86ISelLowering.cpp:58367
lowerUINT_TO_FP_v2i32
static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:19612
getV4X86ShuffleImm8ForMask
static SDValue getV4X86ShuffleImm8ForMask(ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:9800
splitVectorStore
static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG)
Change a vector store into a pair of half-size vector stores.
Definition: X86ISelLowering.cpp:24607
combineLogicBlendIntoPBLENDV
static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:50336
widenSubVector
static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef...
Definition: X86ISelLowering.cpp:4005
supportedVectorVarShift
static bool supportedVectorVarShift(EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
Definition: X86ISelLowering.cpp:29195
isUndefInRange
static bool isUndefInRange(ArrayRef< int > Mask, unsigned Pos, unsigned Size)
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef...
Definition: X86ISelLowering.cpp:3555
LowerToTLSGeneralDynamicModelX32
static SDValue LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
Definition: X86ISelLowering.cpp:18744
MatchVectorAllEqualTest
static SDValue MatchVectorAllEqualTest(SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC)
Definition: X86ISelLowering.cpp:22443
combineFAndn
static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Do target-specific dag combines on X86ISD::FANDN nodes.
Definition: X86ISelLowering.cpp:53372
LowerToTLSExecModel
static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC)
Definition: X86ISelLowering.cpp:18790
foldMaskedMergeImpl
static SDValue foldMaskedMergeImpl(SDValue And0_L, SDValue And0_R, SDValue And1_L, SDValue And1_R, const SDLoc &DL, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:50487
supportedVectorShiftWithImm
static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
Definition: X86ISelLowering.cpp:29159
lower1BitShuffle
static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:17355
combineToExtendBoolVectorInReg
static SDValue combineToExtendBoolVectorInReg(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:45474
splitVectorIntBinary
static SDValue splitVectorIntBinary(SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
Break a binary integer operation into 2 half sized ops and then concatenate the result back.
Definition: X86ISelLowering.cpp:4203
createMMXBuildVector
static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:43808
isBlendOrUndef
static LLVM_ATTRIBUTE_UNUSED bool isBlendOrUndef(ArrayRef< int > Mask)
Return true if every element in Mask, is an in-place blend/select mask or is undef.
Definition: X86ISelLowering.cpp:3616
getIndirectThunkSymbol
static const char * getIndirectThunkSymbol(const X86Subtarget &Subtarget, unsigned Reg)
Definition: X86ISelLowering.cpp:35693
LowerADDSUBO_CARRY
static SDValue LowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:31863
getV4X86ShuffleImm
static unsigned getV4X86ShuffleImm(ArrayRef< int > Mask)
Get a 4-lane 8-bit shuffle immediate for a mask.
Definition: X86ISelLowering.cpp:9776
combineShiftLeft
static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48216
resolveTargetShuffleFromZeroables
static void resolveTargetShuffleFromZeroables(SmallVectorImpl< int > &Mask, const APInt &KnownUndef, const APInt &KnownZero, bool ResolveKnownZeros=true)
Definition: X86ISelLowering.cpp:5735
LowerBUILD_VECTORvXi1
static SDValue LowerBUILD_VECTORvXi1(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:7634
InsertBitToMaskVector
static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Insert one bit to mask vector, like v16i1 or v8i1.
Definition: X86ISelLowering.cpp:18183
getGatherNode
static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:26550
lowerShuffleAsLanePermuteAndRepeatedMask
static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.
Definition: X86ISelLowering.cpp:15066
isSoftF16
static bool isSoftF16(T VT, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:12212
lowerV16I32Shuffle
static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 16-lane 32-bit integer shuffles.
Definition: X86ISelLowering.cpp:16947
combineExtractVectorElt
static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extra...
Definition: X86ISelLowering.cpp:45297
isSingleSHUFPSMask
static bool isSingleSHUFPSMask(ArrayRef< int > Mask)
Test whether this can be lowered with a single SHUFPS instruction.
Definition: X86ISelLowering.cpp:12400
checkBoolTestAndOrSetCCCombine
static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd)
Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
Definition: X86ISelLowering.cpp:46833
isX86LogicalCmp
static bool isX86LogicalCmp(SDValue Op)
Return true if opcode is a X86 logical comparison.
Definition: X86ISelLowering.cpp:24024
isAnyInRange
static bool isAnyInRange(ArrayRef< int > Mask, int Low, int Hi)
Return true if the value of any element in Mask falls within the specified range (L,...
Definition: X86ISelLowering.cpp:3579
combineEXTRACT_SUBVECTOR
static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:57109
combineSextInRegCmov
static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:53753
PromoteMaskArithmetic
static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT, SelectionDAG &DAG, unsigned Depth)
Definition: X86ISelLowering.cpp:49220
detectSSatPattern
static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS=false)
Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_t...
Definition: X86ISelLowering.cpp:51112
FPStateSize
const unsigned FPStateSize
Definition: X86ISelLowering.cpp:27824
matchShuffleWithUNPCK
static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2, unsigned &UnpackOpcode, bool IsUnary, ArrayRef< int > TargetMask, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:9919
combineFneg
static SDValue combineFneg(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Do target-specific dag combines on floating point negations.
Definition: X86ISelLowering.cpp:52938
combineLoad
static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:51314
combineToHorizontalAddSub
static SDValue combineToHorizontalAddSub(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:52199
combineXorSubCTLZ
static SDValue combineXorSubCTLZ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53087
insertSubVector
static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
Definition: X86ISelLowering.cpp:3967
convertIntLogicToFPLogic
static SDValue convertIntLogicToFPLogic(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
If both input operands of a logic op are being cast from floating-point types or FP compares,...
Definition: X86ISelLowering.cpp:49318
isHopBuildVector
static bool isHopBuildVector(const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned &HOpcode, SDValue &V0, SDValue &V1)
Definition: X86ISelLowering.cpp:8108
combineSelectOfTwoConstants
static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG, const SDLoc &DL)
Definition: X86ISelLowering.cpp:45707
combineFOr
static SDValue combineFOr(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
Definition: X86ISelLowering.cpp:53386
LowerINSERT_SUBVECTOR
static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:18458
combineINTRINSIC_VOID
static SDValue combineINTRINSIC_VOID(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:57967
createShuffleMaskFromVSELECT
static bool createShuffleMaskFromVSELECT(SmallVectorImpl< int > &Mask, SDValue Cond, bool IsBLENDV=false)
Definition: X86ISelLowering.cpp:5768
checkBitcastSrcVectorSize
static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size, bool AllowTruncate)
Definition: X86ISelLowering.cpp:43479
matchTruncateWithPACK
static SDValue matchTruncateWithPACK(unsigned &PackOpcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Helper to determine if In truncated to DstVT has the necessary signbits / leading zero bits to be tru...
Definition: X86ISelLowering.cpp:20448
getMaskNode
static SDValue getMaskNode(SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering maskin...
Definition: X86ISelLowering.cpp:25338
lowerV8F64Shuffle
static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 8-lane 64-bit floating point shuffles.
Definition: X86ISelLowering.cpp:16753
shouldUseHorizontalOp
static bool shouldUseHorizontalOp(bool IsSingleSource, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Horizontal vector math instructions may be slower than normal math with shuffles.
Definition: X86ISelLowering.cpp:19486
isFRClass
static bool isFRClass(const TargetRegisterClass &RC)
Check if RC is a vector register class.
Definition: X86ISelLowering.cpp:58952
splitAndLowerShuffle
static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool SimpleOnly)
Generic routine to split vector shuffle into half-sized shuffles.
Definition: X86ISelLowering.cpp:14544
LowerToTLSGeneralDynamicModel64
static SDValue LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
Definition: X86ISelLowering.cpp:18736
IsNOT
static SDValue IsNOT(SDValue V, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:5114
combineFMinFMax
static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG)
Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
Definition: X86ISelLowering.cpp:53406
combineOr
static SDValue combineOr(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:50797
LowerMGATHER
static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:32171
EmitTest
static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Emit nodes that will be selected as "test Op0,Op0", or something equivalent.
Definition: X86ISelLowering.cpp:22603
getVShift
static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl)
Return a vector logical shift node.
Definition: X86ISelLowering.cpp:6764
isFreeToSplitVector
static bool isFreeToSplitVector(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:4138
combineVPDPBUSDPattern
static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:44608
LowerVACOPY
static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:25143
combineINTRINSIC_WO_CHAIN
static SDValue combineINTRINSIC_WO_CHAIN(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:57939
lowerShuffleWithPERMV
static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:14108
lowerV4I32Shuffle
static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Lower 4-lane i32 vector shuffles.
Definition: X86ISelLowering.cpp:13175
combineX86ShuffleChain
static SDValue combineX86ShuffleChain(ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Combine an arbitrary chain of shuffles into a single instruction if possible.
Definition: X86ISelLowering.cpp:38584
widenMaskVector
static SDValue widenMaskVector(SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.
Definition: X86ISelLowering.cpp:4043
LowerAsSplatVectorLoad
static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:6776
isInRange
static bool isInRange(int Val, int Low, int Hi)
Return true if Val falls within the specified range (L, H].
Definition: X86ISelLowering.cpp:3573
combineKSHIFT
static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:57606
combineTargetShuffle
static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Try to combine x86 target specific shuffles.
Definition: X86ISelLowering.cpp:40776
LowerSIGN_EXTEND
static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:24554
splitVector
static std::pair< SDValue, SDValue > splitVector(SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
Definition: X86ISelLowering.cpp:4143
getBT
static SDValue getBT(SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG)
Helper for attempting to create a X86ISD::BT node.
Definition: X86ISelLowering.cpp:21987
EmitTruncSStore
static SDValue EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
Emit Truncating Store with signed or unsigned saturation.
Definition: X86ISelLowering.cpp:26772
ExtendToType
static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false)
Widen a vector input to a vector of NVT.
Definition: X86ISelLowering.cpp:31951
getHorizDemandedElts
static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
Definition: X86ISelLowering.cpp:5213
LowerMSTORE
static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:32129
LowerADDRSPACECAST
static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:32224
combineFMA
static SDValue combineFMA(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:54131
lowerShuffleAsBlendAndPermute
static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool ImmBlends=false)
Try to lower as a blend of elements from two inputs followed by a single-input permutation.
Definition: X86ISelLowering.cpp:10808
matchShuffleAsEXTRQ
static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx, const APInt &Zeroable)
Definition: X86ISelLowering.cpp:11746
X87StateSize
const unsigned X87StateSize
Definition: X86ISelLowering.cpp:27823
lowerV8I64Shuffle
static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 8-lane 64-bit integer shuffles.
Definition: X86ISelLowering.cpp:16874
isLegalConversion
static bool isLegalConversion(MVT VT, bool IsSigned, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:19324
isUndefOrEqual
static bool isUndefOrEqual(int Val, int CmpVal)
Val is the undef sentinel value or equal to the specified value.
Definition: X86ISelLowering.cpp:3528
lowerShuffleAsVTRUNC
static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:10210
isTargetShuffle
static bool isTargetShuffle(unsigned Opcode)
Definition: X86ISelLowering.cpp:2694
isSingleElementRepeatedMask
static bool isSingleElementRepeatedMask(ArrayRef< int > Mask)
Check if the Mask consists of the same element repeated multiple times.
Definition: X86ISelLowering.cpp:11134
LowerCVTPS2PH
static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:32265
combineShiftRightArithmetic
static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48284
combinePMULDQ
static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:57440
LowerIntVSETCC_AVX512
static SDValue LowerIntVSETCC_AVX512(SDValue Op, const SDLoc &dl, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:23140
lowerShuffleWithSSE4A
static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
Definition: X86ISelLowering.cpp:11868
lowerShuffleOfExtractsAsVperm
static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0, SDValue N1, ArrayRef< int > Mask, SelectionDAG &DAG)
If we are extracting two 128-bit halves of a vector and shuffling the result, match that to a 256-bit...
Definition: X86ISelLowering.cpp:12436
lowerV4F64Shuffle
static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 4-lane 64-bit floating point shuffles.
Definition: X86ISelLowering.cpp:15833
getAVX512Node
static SDValue getAVX512Node(unsigned Opcode, const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:4263
lowerToAddSubOrFMAddSub
static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly...
Definition: X86ISelLowering.cpp:8068
lowerV8I16GeneralSingleInputShuffle
static SDValue lowerV8I16GeneralSingleInputShuffle(const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering,...
Definition: X86ISelLowering.cpp:13316
foldXorTruncShiftIntoCmp
static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG)
Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)),...
Definition: X86ISelLowering.cpp:50960
combineX86ShufflesConstants
static SDValue combineX86ShufflesConstants(MVT VT, ArrayRef< SDValue > Ops, ArrayRef< int > Mask, bool HasVariableMask, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:39662
lowerV2F64Shuffle
static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 2-lane 64-bit floating point shuffles.
Definition: X86ISelLowering.cpp:12795
combineOrXorWithSETCC
static SDValue combineOrXorWithSETCC(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:50748
isUpperSubvectorUndef
static SDValue isUpperSubvectorUndef(SDValue V, const SDLoc &DL, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:4117
BrMergingLikelyBias
static cl::opt< int > BrMergingLikelyBias("x86-br-merging-likely-bias", cl::init(0), cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "likely, then it is likely that if the conditionals are split " "both sides will be executed, so it may be desirable to increase " "the instruction cost threshold. Set to -1 to never merge likely " "branches."), cl::Hidden)
clobbersFlagRegisters
static bool clobbersFlagRegisters(const SmallVector< StringRef, 4 > &AsmPieces)
Definition: X86ISelLowering.cpp:58385
getInvertedVectorForFMA
static SDValue getInvertedVectorForFMA(SDValue V, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:54084
IsElementEquivalent
static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp, int Idx, int ExpectedIdx)
Checks whether the vector elements referenced by two shuffle masks are equivalent.
Definition: X86ISelLowering.cpp:9564
matchShuffleAsElementRotate
static int matchShuffleAsElementRotate(SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
Try to match a vector shuffle as an element rotation.
Definition: X86ISelLowering.cpp:11343
combineVEXTRACT_STORE
static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:51975
isUndefOrZeroOrInRange
static bool isUndefOrZeroOrInRange(int Val, int Low, int Hi)
Return true if Val is undef, zero or if its value falls within the specified range (L,...
Definition: X86ISelLowering.cpp:3603
getTargetConstantFromBasePtr
static const Constant * getTargetConstantFromBasePtr(SDValue Ptr)
Definition: X86ISelLowering.cpp:4709
signExtendBitcastSrcVector
static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT, SDValue Src, const SDLoc &DL)
Definition: X86ISelLowering.cpp:43555
lowerShuffleAsBlend
static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to emit a blend instruction for a shuffle.
Definition: X86ISelLowering.cpp:10647
findEltLoadSrc
static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset)
Definition: X86ISelLowering.cpp:6851
combineConcatVectorOps
static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS ...
Definition: X86ISelLowering.cpp:56192
isUndefOrInRange
static bool isUndefOrInRange(int Val, int Low, int Hi)
Return true if Val is undef or if its value falls within the specified range (L, H].
Definition: X86ISelLowering.cpp:3590
combineAddOfPMADDWD
static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT)
Definition: X86ISelLowering.cpp:55783
collectConcatOps
static bool collectConcatOps(SDNode *N, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:4053
truncateVectorWithPACK
static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
Definition: X86ISelLowering.cpp:20306
LowerFGETSIGN
static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:21968
combineSBB
static SDValue combineSBB(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:55455
computeKnownBitsForPMADDUBSW
static void computeKnownBitsForPMADDUBSW(SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
Definition: X86ISelLowering.cpp:37227
combineSIntToFP
static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:55082
FindSingleBitChange
static std::pair< Value *, BitTestKind > FindSingleBitChange(Value *V)
Definition: X86ISelLowering.cpp:30566
combineToFPTruncExtElt
static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG)
If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector...
Definition: X86ISelLowering.cpp:54986
is128BitLaneCrossingShuffleMask
static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef< int > Mask)
Test whether there are elements crossing 128-bit lanes in this shuffle mask.
Definition: X86ISelLowering.cpp:9424
LowerI64IntToFP16
static SDValue LowerI64IntToFP16(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:19067
lowerV4I64Shuffle
static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 4-lane 64-bit integer shuffles.
Definition: X86ISelLowering.cpp:15953
EmitCmp
static SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Emit nodes that will be selected as "cmp Op0,Op1", or something equivalent.
Definition: X86ISelLowering.cpp:22713
combinevXi1ConstantToInteger
static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:43735
FPStateSizeInBits
const unsigned FPStateSizeInBits
Definition: X86ISelLowering.cpp:27825
reduceMaskedStoreToScalarStore
static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS, SelectionDAG &DAG, const X86Subtarget &Subtarget)
If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract a...
Definition: X86ISelLowering.cpp:51623
convertIntLogicToFPLogicOpcode
static unsigned convertIntLogicToFPLogicOpcode(unsigned Opcode)
Definition: X86ISelLowering.cpp:49302
lowerX86FPLogicOp
static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53042
narrowExtractedVectorSelect
static SDValue narrowExtractedVectorSelect(SDNode *Ext, const SDLoc &DL, SelectionDAG &DAG)
If we are extracting a subvector of a vector select and the select condition is composed of concatena...
Definition: X86ISelLowering.cpp:57058
combineScalarAndWithMaskSetcc
static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:49680
lowerShuffleAsLanePermuteAndSHUFP
static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:14728
isNoopShuffleMask
static bool isNoopShuffleMask(ArrayRef< int > Mask)
Tiny helper function to identify a no-op mask.
Definition: X86ISelLowering.cpp:9394
getUnpackh
static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2)
Returns a vector_shuffle node for an unpackh operation.
Definition: X86ISelLowering.cpp:4609
combineExtractFromVectorLoad
static SDValue combineExtractFromVectorLoad(SDNode *N, EVT VecVT, SDValue SrcVec, uint64_t Idx, const SDLoc &dl, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:44762
combineAndMaskToShift
static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
If this is a zero/all-bits result that is bitwise-anded with a low bits mask.
Definition: X86ISelLowering.cpp:49503
lowerShuffleAsByteShiftMask
static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower a vector shuffle as a byte shift sequence.
Definition: X86ISelLowering.cpp:11558
combineX86ShuffleChainWithExtract
static SDValue combineX86ShuffleChainWithExtract(ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:39331
isTargetShuffleVariableMask
static bool isTargetShuffleVariableMask(unsigned Opcode)
Definition: X86ISelLowering.cpp:2735
isLogicOp
static bool isLogicOp(unsigned Opcode)
Definition: X86ISelLowering.cpp:2689
lowerShuffleAsShift
static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool BitwiseOnly)
Definition: X86ISelLowering.cpp:11707
LowerBuildVectorv8i16
static SDValue LowerBuildVectorv8i16(SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Custom lower build_vector of v8i16.
Definition: X86ISelLowering.cpp:6624
matchBinaryShuffle
static bool matchBinaryShuffle(MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, bool IsUnary)
Definition: X86ISelLowering.cpp:38211
lowerShuffleAsUNPCKAndPermute
static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
Try to lower as an unpack of elements from two inputs followed by a single-input permutation.
Definition: X86ISelLowering.cpp:10847
canScaleShuffleElements
static bool canScaleShuffleElements(ArrayRef< int > Mask, unsigned NumDstElts)
Definition: X86ISelLowering.cpp:3775
LowerBITREVERSE_XOP
static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:31493
getShuffleVectorZeroOrUndef
static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Return a vector_shuffle of the specified vector of zero or undef vector.
Definition: X86ISelLowering.cpp:4686
lowerBuildVectorAsBroadcast
static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which u...
Definition: X86ISelLowering.cpp:7280
combineMulToPMULDQ
static SDValue combineMulToPMULDQ(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:47938
LowerPARITY
static SDValue LowerPARITY(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:31625
lowerV16F32Shuffle
static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 16-lane 32-bit floating point shuffles.
Definition: X86ISelLowering.cpp:16807
LowerMINMAX
static SDValue LowerMINMAX(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:28282
combineToExtendCMOV
static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:53936
isHorizontalBinOp
static bool isHorizontalBinOp(unsigned HOpcode, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsCommutative, SmallVectorImpl< int > &PostShuffleMask, bool ForceHorizOp)
Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal ope...
Definition: X86ISelLowering.cpp:52011
getTargetShuffleMaskIndices
static bool getTargetShuffleMaskIndices(SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask, APInt &UndefElts)
Definition: X86ISelLowering.cpp:5092
promoteExtBeforeAdd
static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget)
sext(add_nsw(x, C)) --> add(sext(x), C_sext) zext(add_nuw(x, C)) --> add(zext(x), C_zext) Promoting a...
Definition: X86ISelLowering.cpp:53861
getTargetConstantFromNode
static const Constant * getTargetConstantFromNode(LoadSDNode *Load)
Definition: X86ISelLowering.cpp:4716
canCombineAsMaskOperation
static bool canCombineAsMaskOperation(SDValue V, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:17557
lowerShuffleAsVALIGN
static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower a vector shuffle as a dword/qword rotation.
Definition: X86ISelLowering.cpp:11507
isProfitableToUseFlagOp
static bool isProfitableToUseFlagOp(SDValue Op)
Definition: X86ISelLowering.cpp:22591
LowerBITREVERSE
static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:31538
LowerFROUND
static SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG)
ISD::FROUND is defined to round to nearest with ties rounding away from 0.
Definition: X86ISelLowering.cpp:21816
detectUSatPattern
static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG, const SDLoc &DL)
Detect patterns of truncation with unsigned saturation:
Definition: X86ISelLowering.cpp:51072
narrowShuffle
static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source v...
Definition: X86ISelLowering.cpp:41806
TranslateX86CC
static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG)
Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code,...
Definition: X86ISelLowering.cpp:2842
matchShuffleAsShift
static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable, const X86Subtarget &Subtarget)
Try to lower a vector shuffle as a bit shift (shifts in zeros).
Definition: X86ISelLowering.cpp:11645
getFlagsOfCmpZeroFori1
static SDValue getFlagsOfCmpZeroFori1(SelectionDAG &DAG, const SDLoc &DL, SDValue Mask)
Definition: X86ISelLowering.cpp:32344
lower512BitShuffle
static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
High-level routine to lower various 512-bit x86 vector shuffles.
Definition: X86ISelLowering.cpp:17206
LowerBuildVectorv16i8
static SDValue LowerBuildVectorv16i8(SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Custom lower build_vector of v16i8.
Definition: X86ISelLowering.cpp:6542
getTargetConstantBitsFromNode
static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits, APInt &UndefElts, SmallVectorImpl< APInt > &EltBits, bool AllowWholeUndefs=true, bool AllowPartialUndefs=false)
Definition: X86ISelLowering.cpp:4734
detectExtMul
static bool detectExtMul(SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0, SDValue &Op1)
Definition: X86ISelLowering.cpp:44268
splitIntVSETCC
static SDValue splitIntVSETCC(EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SelectionDAG &DAG, const SDLoc &dl)
Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back.
Definition: X86ISelLowering.cpp:23116
combineVSelectWithAllOnesOrZeros
static SDValue combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic oper...
Definition: X86ISelLowering.cpp:45574
combineMOVMSK
static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:54610
combineLRINT_LLRINT
static SDValue combineLRINT_LLRINT(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:52398
lowerAddSubToHorizontalOp
static SDValue lowerAddSubToHorizontalOp(SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of ...
Definition: X86ISelLowering.cpp:21732
combineShiftToPMULH
static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48167
getParamsForOneTrueMaskedElt
static bool getParamsForOneTrueMaskedElt(MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, Align &Alignment, unsigned &Offset)
Given a masked memory load/store operation, return true if it has one mask bit set.
Definition: X86ISelLowering.cpp:51457
reduceVMULWidth
static SDValue reduceVMULWidth(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
When the operands of vector mul are extended from smaller size values, like i8 and i16,...
Definition: X86ISelLowering.cpp:47669
TranslateIntegerX86CC
static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode)
Definition: X86ISelLowering.cpp:2821
MarkEHRegistrationNode
static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:26734
combineBROADCAST_LOAD
static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:57743
combineFMinNumFMaxNum
static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53427
combineCMP
static SDValue combineCMP(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:55265
isFMAddSubOrFMSubAdd
static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2, unsigned ExpectedUses)
Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBAD...
Definition: X86ISelLowering.cpp:8040
combineX86AddSub
static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST)
Definition: X86ISelLowering.cpp:55412
LowerFCOPYSIGN
static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:21899
createVPDPBUSD
static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS, unsigned &LogBias, const SDLoc &DL, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:44319
LowerFMINIMUM_FMAXIMUM
static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:28298
lowerV2X128Shuffle
static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering 2-lane 128-bit shuffles.
Definition: X86ISelLowering.cpp:14946
lowerUINT_TO_FP_vec
static SDValue lowerUINT_TO_FP_vec(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:19823
getSplitVectorSrc
static SDValue getSplitVectorSrc(SDValue LHS, SDValue RHS, bool AllowCommute)
Definition: X86ISelLowering.cpp:3895
LowerFABSorFNEG
static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG)
The only differences between FABS and FNEG are the mask and the logic op.
Definition: X86ISelLowering.cpp:21838
ShrinkMode
ShrinkMode
Different mul shrinking modes.
Definition: X86ISelLowering.cpp:47605
concatSubVectors
static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG, const SDLoc &dl)
Definition: X86ISelLowering.cpp:4492
combineINTRINSIC_W_CHAIN
static SDValue combineINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:57953
canonicalizeShuffleMaskWithHorizOp
static SDValue canonicalizeShuffleMaskWithHorizOp(MutableArrayRef< SDValue > Ops, MutableArrayRef< int > Mask, unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:39453
combineConstantPoolLoads
static SDValue combineConstantPoolLoads(SDNode *N, const SDLoc &dl, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:51239
computeZeroableShuffleElements
static void computeZeroableShuffleElements(ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt &KnownUndef, APInt &KnownZero)
Compute whether each element of a shuffle is zeroable.
Definition: X86ISelLowering.cpp:5550
EmitMaskedTruncSStore
static SDValue EmitMaskedTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
Emit Masked Truncating Store with signed or unsigned saturation.
Definition: X86ISelLowering.cpp:26783
lowerVSELECTtoVectorShuffle
static SDValue lowerVSELECTtoVectorShuffle(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower a VSELECT instruction to a vector shuffle.
Definition: X86ISelLowering.cpp:17764
matchShuffleAsBlend
static bool matchShuffleAsBlend(MVT VT, SDValue V1, SDValue V2, MutableArrayRef< int > Mask, const APInt &Zeroable, bool &ForceV1Zero, bool &ForceV2Zero, uint64_t &BlendMask)
Definition: X86ISelLowering.cpp:10563
adjustBitcastSrcVectorSSE1
static SDValue adjustBitcastSrcVectorSSE1(SelectionDAG &DAG, SDValue Src, const SDLoc &DL)
Definition: X86ISelLowering.cpp:43521
LowerAVXCONCAT_VECTORS
static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:9216
combineMulSpecial
static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG, EVT VT, const SDLoc &DL)
Definition: X86ISelLowering.cpp:47738
combineBitcastvxi1
static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src, const SDLoc &DL, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:43586
getUnpackl
static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2)
Returns a vector_shuffle node for an unpackl operation.
Definition: X86ISelLowering.cpp:4601
getScalarValueForVectorElement
static SDValue getScalarValueForVectorElement(SDValue V, int Idx, SelectionDAG &DAG)
Try to get a scalar value for a specific element of a vector.
Definition: X86ISelLowering.cpp:12178
LowerZERO_EXTEND_Mask
static SDValue LowerZERO_EXTEND_Mask(SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:20231
getOpcodeForIndirectThunk
static unsigned getOpcodeForIndirectThunk(unsigned RPOpc)
Definition: X86ISelLowering.cpp:35679
lowerV16I8Shuffle
static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Generic lowering of v16i8 shuffles.
Definition: X86ISelLowering.cpp:14152
isNullFPScalarOrVectorConst
static bool isNullFPScalarOrVectorConst(SDValue V)
Definition: X86ISelLowering.cpp:53302
hasIdenticalHalvesShuffleMask
static bool hasIdenticalHalvesShuffleMask(ArrayRef< int > Mask)
Return true if a shuffle mask chooses elements identically in its top and bottom halves.
Definition: X86ISelLowering.cpp:9758
matchShuffleWithPACK
static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, unsigned &PackOpcode, ArrayRef< int > TargetMask, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned MaxStages=1)
Definition: X86ISelLowering.cpp:10364
combineVectorCompare
static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:56123
matchShuffleAsVTRUNC
static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:10061
combineBITREVERSE
static SDValue combineBITREVERSE(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53233
combineArithReduction
static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops.
Definition: X86ISelLowering.cpp:45104
combineINSERT_SUBVECTOR
static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:56898
lowerShuffleAsBitRotate
static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Lower shuffle using X86ISD::VROTLI rotations.
Definition: X86ISelLowering.cpp:11299
lowerShuffleAsDecomposedShuffleMerge
static SDValue lowerShuffleAsDecomposedShuffleMerge(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Generic routine to decompose a shuffle and blend into independent blends and permutes.
Definition: X86ISelLowering.cpp:11158
combineEXTEND_VECTOR_INREG
static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:57538
useVPTERNLOG
static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT)
Definition: X86ISelLowering.cpp:2941
combineBlendOfPermutes
static SDValue combineBlendOfPermutes(MVT VT, SDValue N0, SDValue N1, ArrayRef< int > BlendMask, const APInt &DemandedElts, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
Definition: X86ISelLowering.cpp:40461
combineSetCCAtomicArith
static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc ....
Definition: X86ISelLowering.cpp:46527
combineSetCCEFLAGS
static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value,...
Definition: X86ISelLowering.cpp:47332
isBroadcastShuffleMask
static bool isBroadcastShuffleMask(ArrayRef< int > Mask)
Definition: X86ISelLowering.cpp:11125
LowerEXTEND_VECTOR_INREG
static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:24431
combinePDEP
static SDValue combinePDEP(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:57891
combineExtSetcc
static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53983
canonicalizeShuffleWithOp
static SDValue canonicalizeShuffleWithOp(SDValue N, SelectionDAG &DAG, const SDLoc &DL)
Definition: X86ISelLowering.cpp:40566
LowerVectorCTLZ
static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:28057
scalarizeExtEltFP
static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Extracting a scalar FP value from vector element 0 is free, so extract each operand first,...
Definition: X86ISelLowering.cpp:45003
combineX86ShufflesRecursively
static SDValue combineX86ShufflesRecursively(ArrayRef< SDValue > SrcOps, int SrcOpIndex, SDValue Root, ArrayRef< int > RootMask, ArrayRef< const SDNode * > SrcNodes, unsigned Depth, unsigned MaxDepth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Fully generic combining of x86 shuffle instructions.
Definition: X86ISelLowering.cpp:39784
lowerShuffleAsByteRotate
static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:11450
getInstrStrFromOpNo
static StringRef getInstrStrFromOpNo(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo)
Definition: X86ISelLowering.cpp:32290
isSequentialOrUndefOrZeroInRange
static bool isSequentialOrUndefOrZeroInRange(ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size,...
Definition: X86ISelLowering.cpp:3638
lowerShuffleAsSplitOrBlend
static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Either split a vector in halves or decompose the shuffles and the blend/unpack.
Definition: X86ISelLowering.cpp:14673
canWidenShuffleElements
static bool canWidenShuffleElements(ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask)
Helper function to test whether a shuffle mask could be simplified by widening the elements being shu...
Definition: X86ISelLowering.cpp:3674
splitVectorIntUnary
static SDValue splitVectorIntUnary(SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
Break an unary integer operation into 2 half sized ops and then concatenate the result back.
Definition: X86ISelLowering.cpp:4187
combineSext
static SDValue combineSext(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:54027
lowerV2I64Shuffle
static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 2-lane 64-bit integer shuffles.
Definition: X86ISelLowering.cpp:12879
LowerADDSAT_SUBSAT
static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:28160
combineLogicBlendIntoConditionalNegate
static SDValue combineLogicBlendIntoConditionalNegate(EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:45892
getShuffleScalarElt
static SDValue getShuffleScalarElt(SDValue Op, unsigned Index, SelectionDAG &DAG, unsigned Depth)
Returns the scalar element that will make up the i'th element of the result of the vector shuffle.
Definition: X86ISelLowering.cpp:6399
getTargetVShiftUniformOpcode
static unsigned getTargetVShiftUniformOpcode(unsigned Opc, bool IsVariable)
Definition: X86ISelLowering.cpp:25169
foldXor1SetCC
static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG)
Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
Definition: X86ISelLowering.cpp:53073
matchShuffleAsInsertPS
static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const APInt &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:12684
isNonZeroElementsInOrder
static bool isNonZeroElementsInOrder(const APInt &Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft)
Definition: X86ISelLowering.cpp:9812
lowerUINT_TO_FP_vXi32
static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:19665
combineMul
static SDValue combineMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:47980
emitOrXorXorTree
static SDValue emitOrXorXorTree(SDValue X, const SDLoc &DL, SelectionDAG &DAG, EVT VecVT, EVT CmpVT, bool HasPT, F SToV)
Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp expansion.
Definition: X86ISelLowering.cpp:22048
truncateAVX512SetCCNoBWI
static SDValue truncateAVX512SetCCNoBWI(EVT VT, EVT OpVT, SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just pre-promote its result type since...
Definition: X86ISelLowering.cpp:54307
matchShuffleAsByteRotate
static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
Try to lower a vector shuffle as a byte rotation.
Definition: X86ISelLowering.cpp:11428
lowerShuffleAsPermuteAndUnpack
static SDValue lowerShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.
Definition: X86ISelLowering.cpp:10930
combineAndOrForCcmpCtest
static SDValue combineAndOrForCcmpCtest(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST)
Definition: X86ISelLowering.cpp:49886
narrowLoadToVZLoad
static SDValue narrowLoadToVZLoad(LoadSDNode *LN, MVT MemVT, MVT VT, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:37892
combineSetCCMOVMSK
static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:47097
isAddSubOrSubAddMask
static bool isAddSubOrSubAddMask(ArrayRef< int > Mask, bool &Op0Even)
Checks if the shuffle mask takes subsequent elements alternately from two vectors.
Definition: X86ISelLowering.cpp:41581
isCompletePermute
static bool isCompletePermute(ArrayRef< int > Mask)
Return true if every element of a single input is referenced by the shuffle mask.
Definition: X86ISelLowering.cpp:3656
recoverFramePointer
static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP)
When the MSVC runtime transfers control to us, either to an outlined function or when returning to a ...
Definition: X86ISelLowering.cpp:25447
combineX86INT_TO_FP
static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:53493
getReadTimeStampCounter
static void getReadTimeStampCounter(SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp...
Definition: X86ISelLowering.cpp:26704
LowerShiftByScalarImmediate
static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:29222
LowerVectorAllEqual
static SDValue LowerVectorAllEqual(const SDLoc &DL, SDValue LHS, SDValue RHS, ISD::CondCode CC, const APInt &OriginalMask, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC)
Definition: X86ISelLowering.cpp:22295
is128BitUnpackShuffleMask
static bool is128BitUnpackShuffleMask(ArrayRef< int > Mask, const SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:9733
isOrXorXorTree
static bool isOrXorXorTree(SDValue X, bool Root=true)
Recursive helper for combineVectorSizedSetCCEquality() to see if we have a recognizable memcmp expans...
Definition: X86ISelLowering.cpp:22036
LowerAVXExtend
static SDValue LowerAVXExtend(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:20156
LowerSCALAR_TO_VECTOR
static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:18419
combineTruncate
static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:52726
combineFAnd
static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Do target-specific dag combines on X86ISD::FAND nodes.
Definition: X86ISelLowering.cpp:53355
combineFaddCFmul
static SDValue combineFaddCFmul(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:52312
combineCONCAT_VECTORS
static SDValue combineCONCAT_VECTORS(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:56862
getTargetConstantPoolFromBasePtr
static ConstantPoolSDNode * getTargetConstantPoolFromBasePtr(SDValue Ptr)
Definition: X86ISelLowering.cpp:4701
canonicalizeLaneShuffleWithRepeatedOps
static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V, SelectionDAG &DAG, const SDLoc &DL)
Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).
Definition: X86ISelLowering.cpp:40724
isShuffleEquivalent
static bool isShuffleEquivalent(ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, SDValue V1=SDValue(), SDValue V2=SDValue())
Checks whether a shuffle mask is equivalent to an explicit list of arguments.
Definition: X86ISelLowering.cpp:9624
lowerShuffleToEXPAND
static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT, const APInt &Zeroable, ArrayRef< int > Mask, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:9896
canonicalizeBitSelect
static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:50232
lowerV8F32Shuffle
static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Handle lowering of 8-lane 32-bit floating point shuffles.
Definition: X86ISelLowering.cpp:16066
LowerI64IntToFP_AVX512DQ
static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:19025
LowerBUILD_VECTORAsVariablePermute
static SDValue LowerBUILD_VECTORAsVariablePermute(SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:8730
lowerShuffleAsByteRotateAndPermute
static SDValue lowerShuffleAsByteRotateAndPermute(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of th...
Definition: X86ISelLowering.cpp:11041
combineX86GatherScatter
static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:54742
combineVectorHADDSUB
static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48723
LowerVectorCTPOP
static SDValue LowerVectorCTPOP(SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:31362
getAVX512TruncNode
static SDValue getAVX512TruncNode(const SDLoc &DL, MVT DstVT, SDValue Src, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool ZeroUppers)
Definition: X86ISelLowering.cpp:10099
createPackShuffleMask
static void createPackShuffleMask(MVT VT, SmallVectorImpl< int > &Mask, bool Unary, unsigned NumStages=1)
Create a shuffle mask that matches the PACKSS/PACKUS truncation.
Definition: X86ISelLowering.cpp:5166
isUndefOrEqualInRange
static bool isUndefOrEqualInRange(ArrayRef< int > Mask, int CmpVal, unsigned Pos, unsigned Size)
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef...
Definition: X86ISelLowering.cpp:3542
combineFaddFsub
static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Do target-specific dag combines on floating-point adds/subs.
Definition: X86ISelLowering.cpp:52387
LowerToTLSGeneralDynamicModel32
static SDValue LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
Definition: X86ISelLowering.cpp:18722
splitVectorOp
static SDValue splitVectorOp(SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
Break an operation into 2 half sized ops and then concatenate the results.
Definition: X86ISelLowering.cpp:4162
MulConstantOptimization
static cl::opt< bool > MulConstantOptimization("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden)
getIndexFromUnindexedLoad
static SDValue getIndexFromUnindexedLoad(LoadSDNode *Ld)
Definition: X86ISelLowering.cpp:49565
isAnyZero
static bool isAnyZero(ArrayRef< int > Mask)
Return true if the value of any element in Mask is the zero sentinel value.
Definition: X86ISelLowering.cpp:3584
combineShuffle
static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:41836
truncateVectorWithPACKUS
static SDValue truncateVectorWithPACKUS(EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.
Definition: X86ISelLowering.cpp:20428
lowerINT_TO_FP_vXi64
static SDValue lowerINT_TO_FP_vXi64(SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:19216
resolveZeroablesFromTargetShuffle
static void resolveZeroablesFromTargetShuffle(const SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero)
Definition: X86ISelLowering.cpp:5752
rebuildGatherScatter
static SDValue rebuildGatherScatter(MaskedGatherScatterSDNode *GorS, SDValue Index, SDValue Base, SDValue Scale, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:54761
getPSHUFShuffleMask
static SmallVector< int, 4 > getPSHUFShuffleMask(SDValue N)
Get the PSHUF-style mask from PSHUF node.
Definition: X86ISelLowering.cpp:40234
scalarizeVectorStore
static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
Definition: X86ISelLowering.cpp:24641
LowerBUILD_VECTORvXbf16
static SDValue LowerBUILD_VECTORvXbf16(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:7620
combineShuffleToFMAddSub
static SDValue combineShuffleToFMAddSub(SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
Definition: X86ISelLowering.cpp:41678
getAVX2GatherNode
static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:26518
lowerShufflePairAsUNPCKAndPermute
static SDValue lowerShufflePairAsUNPCKAndPermute(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:15760
isUndefOrZero
static bool isUndefOrZero(int Val)
Val is either the undef or zero sentinel value.
Definition: X86ISelLowering.cpp:3549
SplitOpsAndApply
SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, F Builder, bool CheckBWI=true)
Definition: X86ISelLowering.cpp:4221
combineBEXTR
static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53285
combineFMADDSUB
static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:54219
combineCMov
static SDValue combineCMov(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].
Definition: X86ISelLowering.cpp:47355
isEFLAGSLiveAfter
static bool isEFLAGSLiveAfter(MachineBasicBlock::iterator Itr, MachineBasicBlock *BB)
Definition: X86ISelLowering.cpp:34524
extract128BitVector
static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
Generate a DAG to grab 128-bits from a vector > 128 bits.
Definition: X86ISelLowering.cpp:3953
EmitAVX512Test
static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue &X86CC)
Definition: X86ISelLowering.cpp:23704
lowerShuffleWithSHUFPS
static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower a vector shuffle using the SHUFPS instruction.
Definition: X86ISelLowering.cpp:12979
combineStore
static SDValue combineStore(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:51703
combineX86SetCC
static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:54895
combineMinMaxReduction
static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:44395
isHorizOp
static LLVM_ATTRIBUTE_UNUSED bool isHorizOp(unsigned Opcode)
Definition: X86ISelLowering.cpp:7725
combineHorizOpWithShuffle
static SDValue combineHorizOpWithShuffle(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48428
LowerVectorCTLZ_AVX512CDI
static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Lower a vector CTLZ using native supported vector CTLZ instruction.
Definition: X86ISelLowering.cpp:27945
ExtractBitFromMaskVector
static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Extract one bit from mask vector, like v16i1 or v8i1.
Definition: X86ISelLowering.cpp:17947
LowervXi8MulWithUNPCK
static SDValue LowervXi8MulWithUNPCK(SDValue A, SDValue B, const SDLoc &dl, MVT VT, bool IsSigned, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue *Low=nullptr)
Definition: X86ISelLowering.cpp:28667
lowerShuffleAsBlendOfPSHUFBs
static SDValue lowerShuffleAsBlendOfPSHUFBs(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse)
Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input i...
Definition: X86ISelLowering.cpp:13810
matchShuffleAsINSERTQ
static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx)
Definition: X86ISelLowering.cpp:11802
getBitSelect
static SDValue getBitSelect(const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS, SDValue Mask, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:4541
combineAVG
static SDValue combineAVG(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:53261
isSequentialOrUndefInRange
static bool isSequentialOrUndefInRange(ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
Return true if every element in Mask, beginning from position Pos and ending in Pos + Size,...
Definition: X86ISelLowering.cpp:3627
BrMergingUnlikelyBias
static cl::opt< int > BrMergingUnlikelyBias("x86-br-merging-unlikely-bias", cl::init(-1), cl::desc("Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "unlikely, then it is unlikely that if the conditionals are split " "both sides will be executed, so it may be desirable to decrease " "the instruction cost threshold. Set to -1 to never merge unlikely " "branches."), cl::Hidden)
createSetFPEnvNodes
static SDValue createSetFPEnvNodes(SDValue Ptr, SDValue Chain, const SDLoc &DL, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:27870
getTargetShuffleAndZeroables
static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, APInt &KnownUndef, APInt &KnownZero)
Decode a target shuffle mask and inputs and see if any values are known to be undef or zero from thei...
Definition: X86ISelLowering.cpp:5633
combinePredicateReduction
static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:44469
LowerBuildVectorv4x32
static SDValue LowerBuildVectorv4x32(SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Custom lower build_vector of v4i32 or v4f32.
Definition: X86ISelLowering.cpp:6638
isTargetShuffleEquivalent
static bool isTargetShuffleEquivalent(MVT VT, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, const SelectionDAG &DAG, SDValue V1=SDValue(), SDValue V2=SDValue())
Checks whether a target shuffle mask is equivalent to an explicit pattern.
Definition: X86ISelLowering.cpp:9656
combineBitOpWithMOVMSK
static SDValue combineBitOpWithMOVMSK(SDNode *N, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:49382
foldMaskedMerge
static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG)
Fold "masked merge" expressions like (m & x) | (~m & y) into the equivalent ((x ^ y) & m) ^ y) patter...
Definition: X86ISelLowering.cpp:50513
LowerABS
static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:28230
detectZextAbsDiff
static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1)
Definition: X86ISelLowering.cpp:44301
pushAddIntoCmovOfConsts
static SDValue pushAddIntoCmovOfConsts(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
CMOV of constants requires materializing constant operands in registers.
Definition: X86ISelLowering.cpp:55827
LowerCMP_SWAP
static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:31116
LowerToTLSLocalDynamicModel
static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool Is64Bit, bool Is64BitLP64)
Definition: X86ISelLowering.cpp:18750
combineAndNotIntoANDNP
static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG)
Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
Definition: X86ISelLowering.cpp:49103
combineTruncateWithSat
static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:51139
combineBT
static SDValue combineBT(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
Definition: X86ISelLowering.cpp:53696
expandFP_TO_UINT_SSE
static SDValue expandFP_TO_UINT_SSE(MVT VT, SDValue Src, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:20826
getUnderlyingExtractedFromVec
static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec, SDValue ExtIdx)
For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.
Definition: X86ISelLowering.cpp:7518
combineVectorShiftImm
static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
Definition: X86ISelLowering.cpp:48804
isUnaryOp
static bool isUnaryOp(unsigned Opcode)
Definition: X86ISelLowering.cpp:40554
isRepeatedShuffleMask
static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
Test whether a shuffle mask is equivalent within each sub-lane.
Definition: X86ISelLowering.cpp:9468
combineBrCond
static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
Optimize branch condition evaluation.
Definition: X86ISelLowering.cpp:54909
hasFPCMov
static bool hasFPCMov(unsigned X86CC)
Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the foll...
Definition: X86ISelLowering.cpp:2925
getOneTrueElt
static int getOneTrueElt(SDValue V)
If V is a build vector of boolean constants and exactly one of those constants is true,...
Definition: X86ISelLowering.cpp:51422
lowerShuffleWithVPMOV
static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:10160
lowerShuffleWithUNPCK
static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
Definition: X86ISelLowering.cpp:10008
Concat
static constexpr int Concat[]
Definition: X86InterleavedAccess.cpp:235
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
IsFreeTruncation
if(isa< SExtInst >(LHS)) std auto IsFreeTruncation
Definition: X86PartialReduction.cpp:81
FramePtr
static const unsigned FramePtr
Definition: XCoreFrameLowering.cpp:34
llvm::APFloat::convert
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:5337
llvm::APFloat::getAllOnesValue
static APFloat getAllOnesValue(const fltSemantics &Semantics)
Returns a float which is bitcasted from an all one value int.
Definition: APFloat.cpp:5362
llvm::APFloat::clearSign
void clearSign()
Definition: APFloat.h:1215
llvm::APFloat::next
opStatus next(bool nextDown)
Definition: APFloat.h:1171
llvm::APFloat::changeSign
void changeSign()
Definition: APFloat.h:1214
llvm::APFloat::getZero
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition: APFloat.h:994
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:78
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
llvm::APInt::clearBit
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1385
llvm::APInt::isNegatedPowerOf2
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
Definition: APInt.h:427
llvm::APInt::zext
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:981
llvm::APInt::getSignMask
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
llvm::APInt::isMinSignedValue
bool isMinSignedValue() const
Determine if this is the smallest signed value.
Definition: APInt.h:401
llvm::APInt::getZExtValue
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1498
llvm::APInt::setHighBits
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
Definition: APInt.h:1370
llvm::APInt::popcount
unsigned popcount() const
Count the number of bits set.
Definition: APInt.h:1627
llvm::APInt::setBitsFrom
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
Definition: APInt.h:1364
llvm::APInt::extractBitsAsZExtValue
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
Definition: APInt.cpp:489
llvm::APInt::zextOrTrunc
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition: APInt.cpp:1002
llvm::APInt::getActiveBits
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1470
llvm::APInt::trunc
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:906
llvm::APInt::getMaxValue
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:184
llvm::APInt::setBit
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition: APInt.h:1308
llvm::APInt::abs
APInt abs() const
Get the absolute value.
Definition: APInt.h:1751
llvm::APInt::isAllOnes
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition: APInt.h:349
llvm::APInt::getBitsSet
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition: APInt.h:236
llvm::APInt::isZero
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:358
llvm::APInt::isSignMask
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
Definition: APInt.h:444
llvm::APInt::urem
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition: APInt.cpp:1636
llvm::APInt::setSignBit
void setSignBit()
Set the sign bit to 1.
Definition: APInt.h:1318
llvm::APInt::getBitWidth
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1446
llvm::APInt::ult
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1089
llvm::APInt::getSignedMaxValue
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
llvm::APInt::getMinValue
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
Definition: APInt.h:194
llvm::APInt::isNegative
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:307
llvm::APInt::intersects
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1227
llvm::APInt::eq
bool eq(const APInt &RHS) const
Equality comparison.
Definition: APInt.h:1057
llvm::APInt::exactLogBase2
int32_t exactLogBase2() const
Definition: APInt.h:1739
llvm::APInt::clearAllBits
void clearAllBits()
Set every bit to 0.
Definition: APInt.h:1375
llvm::APInt::ashrInPlace
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:812
llvm::APInt::countr_zero
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition: APInt.h:1596
llvm::APInt::isSignedIntN
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
Definition: APInt.h:413
llvm::APInt::getNumSignBits
unsigned getNumSignBits() const
Computes the number of leading bits of this APInt that are equal to its sign bit.
Definition: APInt.h:1585
llvm::APInt::countl_zero
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition: APInt.h:1555
llvm::APInt::getSplat
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.cpp:620
llvm::APInt::getSignedMinValue
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:197
llvm::APInt::getSignificantBits
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
Definition: APInt.h:1489
llvm::APInt::flipAllBits
void flipAllBits()
Toggle every bit to its opposite value.
Definition: APInt.h:1412
llvm::APInt::countl_one
unsigned countl_one() const
Count the number of leading one bits.
Definition: APInt.h:1572
llvm::APInt::insertBits
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition: APInt.cpp:368
llvm::APInt::clearLowBits
void clearLowBits(unsigned loBits)
Set bottom loBits bits to 0.
Definition: APInt.h:1395
llvm::APInt::logBase2
unsigned logBase2() const
Definition: APInt.h:1717
llvm::APInt::setAllBits
void setAllBits()
Set every bit to 1.
Definition: APInt.h:1297
llvm::APInt::getBoolValue
bool getBoolValue() const
Convert APInt to a boolean value.
Definition: APInt.h:449
llvm::APInt::isMask
bool isMask(unsigned numBits) const
Definition: APInt.h:466
llvm::APInt::isMaxSignedValue
bool isMaxSignedValue() const
Determine if this is the largest signed value.
Definition: APInt.h:383
llvm::APInt::isNonNegative
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:312
llvm::APInt::ule
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1128
llvm::APInt::sext
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:954
llvm::APInt::setBits
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition: APInt.h:1345
llvm::APInt::shl
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:851
llvm::APInt::isSubsetOf
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1235
llvm::APInt::isPowerOf2
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
llvm::APInt::getLowBitsSet
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:284
llvm::APInt::isSignBitSet
bool isSignBitSet() const
Determine if sign bit of this APInt is set.
Definition: APInt.h:319
llvm::APInt::getHighBitsSet
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition: APInt.h:274
llvm::APInt::getZero
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:178
llvm::APInt::setLowBits
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
Definition: APInt.h:1367
llvm::APInt::extractBits
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition: APInt.cpp:453
llvm::APInt::isIntN
bool isIntN(unsigned N) const
Check if this APInt has an N-bits unsigned integer value.
Definition: APInt.h:410
llvm::APInt::isOne
bool isOne() const
Determine if this is a value of 1.
Definition: APInt.h:367
llvm::APInt::getBitsSetFrom
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:264
llvm::APInt::getOneBitSet
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:217
llvm::APInt::getSExtValue
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1520
llvm::APInt::lshrInPlace
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:836
llvm::APInt::lshr
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:829
llvm::APInt::countr_one
unsigned countr_one() const
Count the number of trailing one bits.
Definition: APInt.h:1613
llvm::APInt::uge
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1199
llvm::APInt::isMaxValue
bool isMaxValue() const
Determine if this is the largest unsigned value.
Definition: APInt.h:377
llvm::APInt::truncSSat
APInt truncSSat(unsigned width) const
Truncate to new width with signed saturation.
Definition: APInt.cpp:942
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
llvm::ArrayRef::end
iterator end() const
Definition: ArrayRef.h:154
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
llvm::ArrayRef::drop_back
ArrayRef< T > drop_back(size_t N=1) const
Drop the last N elements of the array.
Definition: ArrayRef.h:210
llvm::ArrayRef::begin
iterator begin() const
Definition: ArrayRef.h:153
llvm::ArrayRef::empty
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
llvm::ArrayRef::slice
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:195
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:635
llvm::AtomicCmpXchgInst::getStrongestFailureOrdering
static AtomicOrdering getStrongestFailureOrdering(AtomicOrdering SuccessOrdering)
Returns the strongest permitted ordering on failure, given the desired ordering on success.
Definition: Instructions.h:644
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:696
llvm::AtomicRMWInst::getAlign
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:809
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition: Instructions.h:708
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:712
llvm::AtomicRMWInst::FAdd
@ FAdd
*p = old + v
Definition: Instructions.h:733
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:726
llvm::AtomicRMWInst::Or
@ Or
*p = old | v
Definition: Instructions.h:720
llvm::AtomicRMWInst::Sub
@ Sub
*p = old - v
Definition: Instructions.h:714
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition: Instructions.h:716
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition: Instructions.h:722
llvm::AtomicRMWInst::FSub
@ FSub
*p = old - v
Definition: Instructions.h:736
llvm::AtomicRMWInst::UIncWrap
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:748
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:724
llvm::AtomicRMWInst::UMin
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:730
llvm::AtomicRMWInst::FMin
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:744
llvm::AtomicRMWInst::UMax
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:728
llvm::AtomicRMWInst::FMax
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:740
llvm::AtomicRMWInst::UDecWrap
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:752
llvm::AtomicRMWInst::Nand
@ Nand
*p = ~(old & v)
Definition: Instructions.h:718
llvm::AtomicRMWInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:852
llvm::AtomicRMWInst::getOperation
BinOp getOperation() const
Definition: Instructions.h:787
llvm::AtomicRMWInst::getSyncScopeID
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
Definition: Instructions.h:843
llvm::AtomicRMWInst::getValOperand
Value * getValOperand()
Definition: Instructions.h:856
llvm::AtomicRMWInst::getOrdering
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
Definition: Instructions.h:829
llvm::AtomicSDNode
This is an SDNode representing atomic operations.
Definition: SelectionDAGNodes.h:1510
llvm::AttributeList::hasFnAttr
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
Definition: Attributes.cpp:1646
llvm::Attribute::getValueAsString
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:392
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
llvm::BitVector::count
size_type count() const
count - Returns the number of bits which are set.
Definition: BitVector.h:162
llvm::BitVector::none
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:188
llvm::BlockAddress
The address of a basic block.
Definition: Constants.h:890
llvm::BuildVectorSDNode
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Definition: SelectionDAGNodes.h:2087
llvm::BuildVectorSDNode::getRepeatedSequence
bool getRepeatedSequence(const APInt &DemandedElts, SmallVectorImpl< SDValue > &Sequence, BitVector *UndefElements=nullptr) const
Find the shortest repeating sequence of values in the build vector.
Definition: SelectionDAG.cpp:12891
llvm::BuildVectorSDNode::getSplatValue
SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
Definition: SelectionDAG.cpp:12851
llvm::BuildVectorSDNode::isConstantSplat
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
Definition: SelectionDAG.cpp:12775
llvm::BuildVectorSDNode::getConstantSplatNode
ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
Definition: SelectionDAG.cpp:12944
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition: InstrTypes.h:1458
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1398
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:757
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:786
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:784
llvm::CmpInst::ICMP_EQ
@ ICMP_EQ
equal
Definition: InstrTypes.h:778
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition: InstrTypes.h:779
llvm::CmpInst::getPredicate
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:847
llvm::ConstantArray::get
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1292
llvm::ConstantDataVector::get
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
Definition: Constants.cpp:2966
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:81
llvm::ConstantInt::isValueValidForType
static bool isValueValidForType(Type *Ty, uint64_t V)
This static method returns true if the type Ty is big enough to represent the value V.
Definition: Constants.cpp:1575
llvm::ConstantPoolSDNode::isMachineConstantPoolEntry
bool isMachineConstantPoolEntry() const
Definition: SelectionDAGNodes.h:2013
llvm::ConstantPoolSDNode::getConstVal
const Constant * getConstVal() const
Definition: SelectionDAGNodes.h:2017
llvm::ConstantSDNode::getZExtValue
uint64_t getZExtValue() const
Definition: SelectionDAGNodes.h:1683
llvm::ConstantSDNode::getAPIntValue
const APInt & getAPIntValue() const
Definition: SelectionDAGNodes.h:1682
llvm::ConstantVector::get
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1399
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:42
llvm::Constant::getIntegerValue
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
Definition: Constants.cpp:400
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
llvm::Constant::getAggregateElement
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:432
llvm::DWARFExpression::Operation
This class represents an Operation in the Expression.
Definition: DWARFExpression.h:32
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
llvm::DataLayout::getTypeAllocSize
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:461
llvm::DenseMapBase::find
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
llvm::DenseMapBase::size
unsigned size() const
Definition: DenseMap.h:99
llvm::DenseMapBase::empty
bool empty() const
Definition: DenseMap.h:98
llvm::DenseMapBase::begin
iterator begin()
Definition: DenseMap.h:75
llvm::DenseMapBase::end
iterator end()
Definition: DenseMap.h:84
llvm::DenseMapBase::contains
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition: DenseMap.h:146
llvm::DenseMapBase::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:211
llvm::Expected
Tagged union holding either a T or a Error.
Definition: Error.h:481
llvm::FastISel
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
llvm::FixedVectorType
Class to represent fixed width SIMD vectors.
Definition: DerivedTypes.h:539
llvm::FixedVectorType::get
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:680
llvm::FunctionLoweringInfo
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Definition: FunctionLoweringInfo.h:57
llvm::FunctionType::param_iterator
Type::subtype_iterator param_iterator
Definition: DerivedTypes.h:126
llvm::Function::hasOptSize
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
Definition: Function.h:705
llvm::Function::getFnAttribute
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.cpp:769
llvm::Function::getFnAttributeAsParsedInteger
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
Definition: Function.cpp:781
llvm::Function::hasMinSize
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:702
llvm::Function::getCallingConv
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:281
llvm::Function::hasPersonalityFn
bool hasPersonalityFn() const
Check whether this function has a personality function.
Definition: Function.h:903
llvm::Function::getPersonalityFn
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1993
llvm::Function::getAttributes
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:357
llvm::Function::hasFnAttribute
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:743
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:915
llvm::GlobalAddressSDNode::getGlobal
const GlobalValue * getGlobal() const
Definition: SelectionDAGNodes.h:1870
llvm::GlobalValue::dropLLVMManglingEscape
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:567
llvm::GlobalValue::isAbsoluteSymbolRef
bool isAbsoluteSymbolRef() const
Returns whether this is a reference to an absolute symbol.
Definition: Globals.cpp:399
llvm::GlobalValue::getThreadLocalMode
ThreadLocalMode getThreadLocalMode() const
Definition: GlobalValue.h:271
llvm::GlobalValue::getParent
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
llvm::ICmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1141
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2686
llvm::InlineAsm::ConstraintInfoVector
std::vector< ConstraintInfo > ConstraintInfoVector
Definition: InlineAsm.h:121
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
llvm::Instruction::eraseFromParent
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:92
llvm::Instruction::user_back
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
Definition: Instruction.h:169
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:70
llvm::IntegerType
Class to represent integer types.
Definition: DerivedTypes.h:40
llvm::IntegerType::getBitWidth
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
Definition: DerivedTypes.h:72
llvm::IntrinsicLowering::LowerToByteSwap
static bool LowerToByteSwap(CallInst *CI)
Try to replace a call instruction with a call to a bswap intrinsic.
Definition: IntrinsicLowering.cpp:465
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
llvm::LSBaseSDNode::isIndexed
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
Definition: SelectionDAGNodes.h:2432
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:174
llvm::LoadInst::setAtomic
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this load instruction.
Definition: Instructions.h:239
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2444
llvm::LoadSDNode::getBasePtr
const SDValue & getBasePtr() const
Definition: SelectionDAGNodes.h:2463
llvm::MCAsmInfo::usesWindowsCFI
bool usesWindowsCFI() const
Definition: MCAsmInfo.h:759
llvm::MCContext::getOrCreateParentFrameOffsetSymbol
MCSymbol * getOrCreateParentFrameOffsetSymbol(const Twine &FuncName)
Definition: MCContext.cpp:242
llvm::MCContext::getOrCreateLSDASymbol
MCSymbol * getOrCreateLSDASymbol(const Twine &FuncName)
Definition: MCContext.cpp:247
llvm::MCSymbol
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
llvm::MIMetadata
Set of metadata that should be preserved when using BuildMI().
Definition: MachineInstrBuilder.h:348
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:35
llvm::MVT::getFloatingPointVT
static MVT getFloatingPointVT(unsigned BitWidth)
Definition: MachineValueType.h:421
llvm::MVT::is128BitVector
bool is128BitVector() const
Return true if this is a 128-bit vector type.
Definition: MachineValueType.h:148
llvm::MVT::INVALID_SIMPLE_VALUE_TYPE
@ INVALID_SIMPLE_VALUE_TYPE
Definition: MachineValueType.h:40
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:54
llvm::MVT::getScalarSizeInBits
uint64_t getScalarSizeInBits() const
Definition: MachineValueType.h:336
llvm::MVT::changeVectorElementType
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: MachineValueType.h:199
llvm::MVT::bitsLE
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
Definition: MachineValueType.h:415
llvm::MVT::getVectorNumElements
unsigned getVectorNumElements() const
Definition: MachineValueType.h:284
llvm::MVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: MachineValueType.h:105
llvm::MVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: MachineValueType.h:89
llvm::MVT::isScalableVector
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
Definition: MachineValueType.h:112
llvm::MVT::is32BitVector
bool is32BitVector() const
Return true if this is a 32-bit vector type.
Definition: MachineValueType.h:138
llvm::MVT::changeTypeToInteger
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition: MachineValueType.h:209
llvm::MVT::getVT
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:231
llvm::MVT::bitsLT
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
Definition: MachineValueType.h:408
llvm::MVT::is512BitVector
bool is512BitVector() const
Return true if this is a 512-bit vector type.
Definition: MachineValueType.h:158
llvm::MVT::integer_valuetypes
static auto integer_valuetypes()
Definition: MachineValueType.h:492
llvm::MVT::getSizeInBits
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
Definition: MachineValueType.h:298
llvm::MVT::fixedlen_vector_valuetypes
static auto fixedlen_vector_valuetypes()
Definition: MachineValueType.h:509
llvm::MVT::getFixedSizeInBits
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: MachineValueType.h:332
llvm::MVT::getFltSemantics
const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
Definition: ValueTypes.cpp:293
llvm::MVT::bitsGT
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
Definition: MachineValueType.h:394
llvm::MVT::is256BitVector
bool is256BitVector() const
Return true if this is a 256-bit vector type.
Definition: MachineValueType.h:153
llvm::MVT::getStoreSize
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: MachineValueType.h:346
llvm::MVT::bitsGE
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
Definition: MachineValueType.h:401
llvm::MVT::isScalarInteger
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
Definition: MachineValueType.h:99
llvm::MVT::getVectorVT
static MVT getVectorVT(MVT VT, unsigned NumElements)
Definition: MachineValueType.h:441
llvm::MVT::getVectorElementType
MVT getVectorElementType() const
Definition: MachineValueType.h:255
llvm::MVT::isFloatingPoint
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: MachineValueType.h:79
llvm::MVT::getIntegerVT
static MVT getIntegerVT(unsigned BitWidth)
Definition: MachineValueType.h:431
llvm::MVT::getDoubleNumVectorElementsVT
MVT getDoubleNumVectorElementsVT() const
Definition: MachineValueType.h:226
llvm::MVT::getHalfNumVectorElementsVT
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
Definition: MachineValueType.h:217
llvm::MVT::getScalarType
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
Definition: MachineValueType.h:251
llvm::MVT::is64BitVector
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Definition: MachineValueType.h:143
llvm::MVT::changeVectorElementTypeToInteger
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: MachineValueType.h:188
llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
Definition: MachineBasicBlock.cpp:935
llvm::MachineBasicBlock::isEHPad
bool isEHPad() const
Returns true if the block is a landing pad.
Definition: MachineBasicBlock.h:633
llvm::MachineBasicBlock::rend
reverse_iterator rend()
Definition: MachineBasicBlock.h:364
llvm::MachineBasicBlock::insert
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
Definition: MachineBasicBlock.cpp:1454
llvm::MachineBasicBlock::push_back
void push_back(MachineInstr *MI)
Definition: MachineBasicBlock.h:994
llvm::MachineBasicBlock::setCallFrameSize
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
Definition: MachineBasicBlock.h:1215
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:255
llvm::MachineBasicBlock::isLiveIn
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
Definition: MachineBasicBlock.cpp:616
llvm::MachineBasicBlock::succ_size
unsigned succ_size() const
Definition: MachineBasicBlock.h:432
llvm::MachineBasicBlock::addSuccessor
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:796
llvm::MachineBasicBlock::removeSuccessor
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:834
llvm::MachineBasicBlock::instr_iterator
Instructions::iterator instr_iterator
Definition: MachineBasicBlock.h:313
llvm::MachineBasicBlock::succ_rbegin
succ_reverse_iterator succ_rbegin()
Definition: MachineBasicBlock.h:424
llvm::MachineBasicBlock::eraseFromParent
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
Definition: MachineBasicBlock.cpp:1474
llvm::MachineBasicBlock::addLiveIn
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
Definition: MachineBasicBlock.h:455
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:310
llvm::MachineBasicBlock::erase
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
Definition: MachineBasicBlock.cpp:1441
llvm::MachineBasicBlock::successors
iterator_range< succ_iterator > successors()
Definition: MachineBasicBlock.h:443
llvm::MachineBasicBlock::insertAfter
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
Definition: MachineBasicBlock.h:1022
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:1101
llvm::MachineBasicBlock::iterator
MachineInstrBundleIterator< MachineInstr > iterator
Definition: MachineBasicBlock.h:318
llvm::MachineBasicBlock::succ_rend
succ_reverse_iterator succ_rend()
Definition: MachineBasicBlock.h:428
llvm::MachineBasicBlock::setMachineBlockAddressTaken
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
Definition: MachineBasicBlock.h:296
llvm::MachineBasicBlock::setIsEHPad
void setIsEHPad(bool V=true)
Indicates the block is a landing pad.
Definition: MachineBasicBlock.h:637
llvm::MachineFrameInfo
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Definition: MachineFrameInfo.h:106
llvm::MachineFrameInfo::CreateFixedObject
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
Definition: MachineFrameInfo.cpp:83
llvm::MachineFrameInfo::CreateStackObject
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Definition: MachineFrameInfo.cpp:51
llvm::MachineFrameInfo::setFrameAddressIsTaken
void setFrameAddressIsTaken(bool T)
Definition: MachineFrameInfo.h:374
llvm::MachineFrameInfo::setReturnAddressIsTaken
void setReturnAddressIsTaken(bool s)
Definition: MachineFrameInfo.h:380
llvm::MachineFrameInfo::setHasCopyImplyingStackAdjustment
void setHasCopyImplyingStackAdjustment(bool B)
Definition: MachineFrameInfo.h:633
llvm::MachineFrameInfo::isFixedObjectIndex
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
Definition: MachineFrameInfo.h:700
llvm::MachineFrameInfo::setObjectAlignment
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
Definition: MachineFrameInfo.h:499
llvm::MachineFrameInfo::getFunctionContextIndex
int getFunctionContextIndex() const
Return the index for the function context object.
Definition: MachineFrameInfo.h:366
llvm::MachineFunction::getWinEHFuncInfo
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
Definition: MachineFunction.h:762
llvm::MachineFunction::getFunctionNumber
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
Definition: MachineFunction.h:693
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:718
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:502
llvm::MachineFunction::getFrameInfo
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Definition: MachineFunction.h:734
llvm::MachineFunction::push_back
void push_back(MachineBasicBlock *MBB)
Definition: MachineFunction.h:944
llvm::MachineFunction::getContext
MCContext & getContext() const
Definition: MachineFunction.h:670
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:728
llvm::MachineFunction::getDataLayout
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Definition: MachineFunction.cpp:309
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:684
llvm::MachineFunction::getTarget
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Definition: MachineFunction.h:714
llvm::MachineFunction::shouldSplitStack
bool shouldSplitStack() const
Should we be emitting segmented stack stuff for the function.
Definition: MachineFunction.cpp:329
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:816
llvm::MachineFunction::moveCallSiteInfo
void moveCallSiteInfo(const MachineInstr *Old, const MachineInstr *New)
Move the call site info from Old to \New call site info.
Definition: MachineFunction.cpp:936
llvm::MachineFunction::CreateMachineBasicBlock
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Definition: MachineFunction.cpp:463
llvm::MachineFunction::insert
void insert(iterator MBBI, MachineBasicBlock *MBB)
Definition: MachineFunction.h:946
llvm::MachineInstrBuilder::setMemRefs
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
Definition: MachineInstrBuilder.h:210
llvm::MachineInstrBuilder::addExternalSymbol
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:186
llvm::MachineInstrBuilder::setMIFlag
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
Definition: MachineInstrBuilder.h:280
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:133
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition: MachineInstrBuilder.h:226
llvm::MachineInstrBuilder::addRegMask
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
Definition: MachineInstrBuilder.h:199
llvm::MachineInstrBuilder::addGlobalAddress
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:179
llvm::MachineInstrBuilder::addDisp
const MachineInstrBuilder & addDisp(const MachineOperand &Disp, int64_t off, unsigned char TargetFlags=0) const
Definition: MachineInstrBuilder.h:291
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:99
llvm::MachineInstrBuilder::addJumpTableIndex
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:173
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:148
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition: MachineInstrBuilder.h:125
llvm::MachineInstrBuilder::getInstr
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Definition: MachineInstrBuilder.h:91
llvm::MachineInstrBuilder::addDef
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Definition: MachineInstrBuilder.h:118
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:69
llvm::MachineInstr::killsRegister
bool killsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr kills the specified register.
Definition: MachineInstr.h:1500
llvm::MachineInstr::eraseFromParent
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Definition: MachineInstr.cpp:761
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:579
llvm::MachineJumpTableInfo::createJumpTableIndex
unsigned createJumpTableIndex(const std::vector< MachineBasicBlock * > &DestBBs)
createJumpTableIndex - Create a new jump table.
Definition: MachineFunction.cpp:1307
llvm::MachineJumpTableInfo::EK_LabelDifference32
@ EK_LabelDifference32
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table.
Definition: MachineJumpTableInfo.h:68
llvm::MachineJumpTableInfo::EK_BlockAddress
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
Definition: MachineJumpTableInfo.h:49
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition: MachineMemOperand.h:129
llvm::MachineMemOperand::Flags
Flags
Flags values. These may be or'd together.
Definition: MachineMemOperand.h:132
llvm::MachineMemOperand::MOVolatile
@ MOVolatile
The memory access is volatile.
Definition: MachineMemOperand.h:140
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:136
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:138
llvm::MachineMemOperand::getFlags
Flags getFlags() const
Return the raw flags of the source value,.
Definition: MachineMemOperand.h:223
llvm::MachineMemOperand::getValue
const Value * getValue() const
Return the base address of the memory access.
Definition: MachineMemOperand.h:212
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:556
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:329
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:369
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:51
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:157
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:3042
llvm::MaskedGatherSDNode
This class is used to represent an MGATHER node.
Definition: SelectionDAGNodes.h:2941
llvm::MaskedGatherScatterSDNode
This is a base class used to represent MGATHER and MSCATTER nodes.
Definition: SelectionDAGNodes.h:2903
llvm::MaskedLoadSDNode
This class is used to represent an MLOAD node.
Definition: SelectionDAGNodes.h:2754
llvm::MaskedLoadStoreSDNode
This base class is used to represent MLOAD and MSTORE nodes.
Definition: SelectionDAGNodes.h:2712
llvm::MaskedLoadStoreSDNode::getMask
const SDValue & getMask() const
Definition: SelectionDAGNodes.h:2731
llvm::MaskedLoadStoreSDNode::getAddressingMode
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
Definition: SelectionDAGNodes.h:2737
llvm::MaskedScatterSDNode
This class is used to represent an MSCATTER node.
Definition: SelectionDAGNodes.h:2966
llvm::MaskedStoreSDNode
This class is used to represent an MSTORE node.
Definition: SelectionDAGNodes.h:2783
llvm::MaskedStoreSDNode::isCompressingStore
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
Definition: SelectionDAGNodes.h:2804
llvm::MaskedStoreSDNode::getOffset
const SDValue & getOffset() const
Definition: SelectionDAGNodes.h:2808
llvm::MaskedStoreSDNode::getBasePtr
const SDValue & getBasePtr() const
Definition: SelectionDAGNodes.h:2807
llvm::MaskedStoreSDNode::getMask
const SDValue & getMask() const
Definition: SelectionDAGNodes.h:2809
llvm::MaskedStoreSDNode::getValue
const SDValue & getValue() const
Definition: SelectionDAGNodes.h:2806
llvm::MaskedStoreSDNode::isTruncatingStore
bool isTruncatingStore() const
Return true if the op does a truncation before store.
Definition: SelectionDAGNodes.h:2798
llvm::MemIntrinsicSDNode
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
Definition: SelectionDAGNodes.h:1582
llvm::MemSDNode
This is an abstract virtual class for memory operations.
Definition: SelectionDAGNodes.h:1336
llvm::MemSDNode::getAddressSpace
unsigned getAddressSpace() const
Return the address space for the associated pointer.
Definition: SelectionDAGNodes.h:1427
llvm::MemSDNode::getAlign
Align getAlign() const
Definition: SelectionDAGNodes.h:1354
llvm::MemSDNode::isVolatile
bool isVolatile() const
Definition: SelectionDAGNodes.h:1375
llvm::MemSDNode::getAAInfo
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Definition: SelectionDAGNodes.h:1384
llvm::MemSDNode::getSyncScopeID
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID for this memory operation.
Definition: SelectionDAGNodes.h:1390
llvm::MemSDNode::getOriginalAlign
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
Definition: SelectionDAGNodes.h:1353
llvm::MemSDNode::readMem
bool readMem() const
Definition: SelectionDAGNodes.h:1349
llvm::MemSDNode::isSimple
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
Definition: SelectionDAGNodes.h:1413
llvm::MemSDNode::getSuccessOrdering
AtomicOrdering getSuccessOrdering() const
Return the atomic ordering requirements for this memory operation.
Definition: SelectionDAGNodes.h:1395
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1420
llvm::MemSDNode::getBasePtr
const SDValue & getBasePtr() const
Definition: SelectionDAGNodes.h:1441
llvm::MemSDNode::getPointerInfo
const MachinePointerInfo & getPointerInfo() const
Definition: SelectionDAGNodes.h:1422
llvm::MemSDNode::getChain
const SDValue & getChain() const
Definition: SelectionDAGNodes.h:1439
llvm::MemSDNode::isNonTemporal
bool isNonTemporal() const
Definition: SelectionDAGNodes.h:1376
llvm::MemSDNode::getMemoryVT
EVT getMemoryVT() const
Return the type of the in-memory value.
Definition: SelectionDAGNodes.h:1416
llvm::Metadata
Root of the metadata hierarchy.
Definition: Metadata.h:62
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::Module::getModuleFlag
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Definition: Module.cpp:319
llvm::MutableArrayRef
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
llvm::PointerType::getUnqual
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::Register::isValid
constexpr bool isValid() const
Definition: Register.h:116
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1165
llvm::SDNode::use_iterator
This class provides iterator support for SDUse operands that use a specific SDNode.
Definition: SelectionDAGNodes.h:791
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:491
llvm::SDNode::isStrictFPOpcode
bool isStrictFPOpcode()
Test if this node is a strict floating point pseudo-op.
Definition: SelectionDAGNodes.h:719
llvm::SDNode::ops
ArrayRef< SDUse > ops() const
Definition: SelectionDAGNodes.h:982
llvm::SDNode::getAsAPIntVal
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
Definition: SelectionDAGNodes.h:1717
llvm::SDNode::getOpcode
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Definition: SelectionDAGNodes.h:682
llvm::SDNode::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this node.
Definition: SelectionDAGNodes.h:758
llvm::SDNode::isOnlyUserOf
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
Definition: SelectionDAG.cpp:12219
llvm::SDNode::op_values
iterator_range< value_op_iterator > op_values() const
Definition: SelectionDAGNodes.h:996
llvm::SDNode::uses
iterator_range< use_iterator > uses()
Definition: SelectionDAGNodes.h:852
llvm::SDNode::getFlags
SDNodeFlags getFlags() const
Definition: SelectionDAGNodes.h:1024
llvm::SDNode::getSimpleValueType
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
Definition: SelectionDAGNodes.h:1051
llvm::SDNode::getAsZExtVal
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
Definition: SelectionDAGNodes.h:1709
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:1042
llvm::SDNode::getNumOperands
unsigned getNumOperands() const
Return the number of values used by this operation.
Definition: SelectionDAGNodes.h:954
llvm::SDNode::getVTList
SDVTList getVTList() const
Definition: SelectionDAGNodes.h:1001
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:973
llvm::SDNode::getConstantOperandVal
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
Definition: SelectionDAGNodes.h:1705
llvm::SDNode::areOnlyUsersOf
static bool areOnlyUsersOf(ArrayRef< const SDNode * > Nodes, const SDNode *N)
Return true if all the users of N are contained in Nodes.
Definition: SelectionDAG.cpp:12232
llvm::SDNode::use_begin
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
Definition: SelectionDAGNodes.h:846
llvm::SDNode::hasAnyUseOfValue
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
Definition: SelectionDAG.cpp:12208
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:1045
llvm::SDNode::isUndef
bool isUndef() const
Return true if the type of the node type undefined.
Definition: SelectionDAGNodes.h:705
llvm::SDNode::hasNUsesOfValue
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
Definition: SelectionDAG.cpp:12190
llvm::SDNode::setFlags
void setFlags(SDNodeFlags NewFlags)
Definition: SelectionDAGNodes.h:1025
llvm::SDNode::use_end
static use_iterator use_end()
Definition: SelectionDAGNodes.h:850
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDValue::isUndef
bool isUndef() const
Definition: SelectionDAGNodes.h:1236
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SDValue::hasOneUse
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
Definition: SelectionDAGNodes.h:1244
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1200
llvm::SDValue::getValueSizeInBits
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
Definition: SelectionDAGNodes.h:199
llvm::SDValue::getOperand
const SDValue & getOperand(unsigned i) const
Definition: SelectionDAGNodes.h:1208
llvm::SDValue::getConstantOperandAPInt
const APInt & getConstantOperandAPInt(unsigned i) const
Definition: SelectionDAGNodes.h:1216
llvm::SDValue::getScalarValueSizeInBits
uint64_t getScalarValueSizeInBits() const
Definition: SelectionDAGNodes.h:203
llvm::SDValue::getResNo
unsigned getResNo() const
get the index which selects a specific result in the SDNode
Definition: SelectionDAGNodes.h:156
llvm::SDValue::getConstantOperandVal
uint64_t getConstantOperandVal(unsigned i) const
Definition: SelectionDAGNodes.h:1212
llvm::SDValue::getSimpleValueType
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:190
llvm::SDValue::getOpcode
unsigned getOpcode() const
Definition: SelectionDAGNodes.h:1196
llvm::SDValue::getNumOperands
unsigned getNumOperands() const
Definition: SelectionDAGNodes.h:1204
llvm::SelectionDAG::FlagInserter
Help to insert SDNodeFlags automatically in transforming.
Definition: SelectionDAG.h:364
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:226
llvm::SelectionDAG::willNotOverflowAdd
bool willNotOverflowAdd(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 nodes can never overflow.
Definition: SelectionDAG.h:1973
llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG
static unsigned getOpcode_EXTEND_VECTOR_INREG(unsigned Opcode)
Convert *_EXTEND to *_EXTEND_VECTOR_INREG opcode.
Definition: SelectionDAG.h:938
llvm::SelectionDAG::getShiftAmountOperand
SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
Definition: SelectionDAG.cpp:2421
llvm::SelectionDAG::getExtLoad
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:9006
llvm::SelectionDAG::getTargetGlobalAddress
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:736
llvm::SelectionDAG::getExtOrTrunc
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
Definition: SelectionDAG.h:968
llvm::SelectionDAG::getSplatSourceVector
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
Definition: SelectionDAG.cpp:2963
llvm::SelectionDAG::ComputeMaxSignificantBits
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
Definition: SelectionDAG.cpp:5138
llvm::SelectionDAG::getMaskedGather
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
Definition: SelectionDAG.cpp:9759
llvm::SelectionDAG::getAddrSpaceCast
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
Definition: SelectionDAG.cpp:2393
llvm::SelectionDAG::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:489
llvm::SelectionDAG::getMergeValues
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
Definition: SelectionDAG.cpp:8753
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:10423
llvm::SelectionDAG::getShiftAmountConstant
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
Definition: SelectionDAG.cpp:1770
llvm::SelectionDAG::getAllOnesConstant
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
Definition: SelectionDAG.cpp:1759
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:10861
llvm::SelectionDAG::getFreeze
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
Definition: SelectionDAG.cpp:2415
llvm::SelectionDAG::getMemcpy
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
Definition: SelectionDAG.cpp:8296
llvm::SelectionDAG::getConstantPool
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
Definition: SelectionDAG.cpp:1921
llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
Definition: SelectionDAG.cpp:13134
llvm::SelectionDAG::makeEquivalentMemoryOrdering
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
Definition: SelectionDAG.cpp:11856
llvm::SelectionDAG::getJumpTableDebugInfo
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
Definition: SelectionDAG.cpp:1914
llvm::SelectionDAG::getSetCC
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:1238
llvm::SelectionDAG::getConstantFP
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1826
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:8989
llvm::SelectionDAG::getAtomic
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Definition: SelectionDAG.cpp:8710
llvm::SelectionDAG::shouldOptForSize
bool shouldOptForSize() const
Definition: SelectionDAG.cpp:1358
llvm::SelectionDAG::SplitVectorOperand
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
Definition: SelectionDAG.h:2279
llvm::SelectionDAG::getNOT
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
Definition: SelectionDAG.cpp:1580
llvm::SelectionDAG::getTargetLoweringInfo
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:493
llvm::SelectionDAG::isEqualTo
bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
Definition: SelectionDAG.cpp:5693
llvm::SelectionDAG::MaxRecursionDepth
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:451
llvm::SelectionDAG::expandVACopy
SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
Definition: SelectionDAG.cpp:2463
llvm::SelectionDAG::GetSplitDestVTs
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
Definition: SelectionDAG.cpp:12656
llvm::SelectionDAG::getTargetJumpTable
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
Definition: SelectionDAG.h:746
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1118
llvm::SelectionDAG::getCALLSEQ_END
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
Definition: SelectionDAG.h:1095
llvm::SelectionDAG::getBuildVector
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:842
llvm::SelectionDAG::isSplatValue
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
Definition: SelectionDAG.cpp:2763
llvm::SelectionDAG::getBitcast
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
Definition: SelectionDAG.cpp:2386
llvm::SelectionDAG::getSelect
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:1267
llvm::SelectionDAG::getNegative
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
Definition: SelectionDAG.cpp:1575
llvm::SelectionDAG::simplifySelect
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
Definition: SelectionDAG.cpp:9944
llvm::SelectionDAG::getZeroExtendInReg
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
Definition: SelectionDAG.cpp:1526
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:487
llvm::SelectionDAG::expandVAArg
SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
Definition: SelectionDAG.cpp:2429
llvm::SelectionDAG::doesNodeExist
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
Definition: SelectionDAG.cpp:11023
llvm::SelectionDAG::areNonVolatileConsecutiveLoads
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
Definition: SelectionDAG.cpp:12577
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1624
llvm::SelectionDAG::getMemBasePlusOffset
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
Definition: SelectionDAG.cpp:7765
llvm::SelectionDAG::getTruncStore
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:9090
llvm::SelectionDAG::ReplaceAllUsesWith
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
Definition: SelectionDAG.cpp:11355
llvm::SelectionDAG::getCommutedVectorShuffle
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
Definition: SelectionDAG.cpp:2271
llvm::SelectionDAG::SplitVector
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
Definition: SelectionDAG.cpp:12701
llvm::SelectionDAG::isGuaranteedNotToBeUndefOrPoison
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
Definition: SelectionDAG.cpp:5151
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:9039
llvm::SelectionDAG::getSignedConstant
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Definition: SelectionDAG.cpp:1750
llvm::SelectionDAG::InferPtrAlign
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
Definition: SelectionDAG.cpp:12605
llvm::SelectionDAG::getCALLSEQ_START
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
Definition: SelectionDAG.h:1083
llvm::SelectionDAG::SignBitIsZero
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
Definition: SelectionDAG.cpp:2703
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2281
llvm::SelectionDAG::getTargetExtractSubreg
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
Definition: SelectionDAG.cpp:10979
llvm::SelectionDAG::getSExtOrTrunc
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
Definition: SelectionDAG.cpp:1460
llvm::SelectionDAG::isKnownNeverZero
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
Definition: SelectionDAG.cpp:5532
llvm::SelectionDAG::FoldConstantArithmetic
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
Definition: SelectionDAG.cpp:6390
llvm::SelectionDAG::getMaskedStore
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
Definition: SelectionDAG.cpp:9710
llvm::SelectionDAG::getExternalSymbol
SDValue getExternalSymbol(const char *Sym, EVT VT)
Definition: SelectionDAG.cpp:2005
llvm::SelectionDAG::getTarget
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:488
llvm::SelectionDAG::getStrictFPExtendOrRound
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
Definition: SelectionDAG.cpp:1441
llvm::SelectionDAG::getAnyExtOrTrunc
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
Definition: SelectionDAG.cpp:1454
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:787
llvm::SelectionDAG::getSelectCC
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
Definition: SelectionDAG.h:1277
llvm::SelectionDAG::isKnownNeverZeroFloat
bool isKnownNeverZeroFloat(SDValue Op) const
Test whether the given floating point SDValue is known to never be positive or negative zero.
Definition: SelectionDAG.cpp:5523
llvm::SelectionDAG::getIntPtrConstant
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1765
llvm::SelectionDAG::getValueType
SDValue getValueType(EVT)
Definition: SelectionDAG.cpp:1991
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:10046
llvm::SelectionDAG::getFPExtendOrRound
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
Definition: SelectionDAG.cpp:1433
llvm::SelectionDAG::isKnownNeverNaN
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
Definition: SelectionDAG.cpp:5393
llvm::SelectionDAG::getTargetConstant
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:690
llvm::SelectionDAG::ComputeNumSignBits
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
Definition: SelectionDAG.cpp:4460
llvm::SelectionDAG::MaskedVectorIsZero
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
Definition: SelectionDAG.cpp:2727
llvm::SelectionDAG::getTargetBlockAddress
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:782
llvm::SelectionDAG::isBaseWithConstantOffset
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
Definition: SelectionDAG.cpp:5388
llvm::SelectionDAG::getVectorIdxConstant
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1783
llvm::SelectionDAG::ReplaceAllUsesOfValueWith
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
Definition: SelectionDAG.cpp:11516
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:482
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:813
llvm::SelectionDAG::getSplatBuildVector
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
Definition: SelectionDAG.h:859
llvm::SelectionDAG::getFrameIndex
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Definition: SelectionDAG.cpp:1878
llvm::SelectionDAG::computeKnownBits
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
Definition: SelectionDAG.cpp:3149
llvm::SelectionDAG::getZExtOrTrunc
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
Definition: SelectionDAG.cpp:1466
llvm::SelectionDAG::getCondCode
SDValue getCondCode(ISD::CondCode Cond)
Definition: SelectionDAG.cpp:2032
llvm::SelectionDAG::MaskedValueIsZero
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
Definition: SelectionDAG.cpp:2711
llvm::SelectionDAG::getValidShiftAmount
std::optional< uint64_t > getValidShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has a uniform shift amount that is less than the element bit-width of the shi...
Definition: SelectionDAG.cpp:3083
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:500
llvm::SelectionDAG::getMemIntrinsicNode
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
Definition: SelectionDAG.cpp:8764
llvm::SelectionDAG::getTargetExternalSymbol
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
Definition: SelectionDAG.cpp:2022
llvm::SelectionDAG::getMCSymbol
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
Definition: SelectionDAG.cpp:2013
llvm::SelectionDAG::CreateStackTemporary
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
Definition: SelectionDAG.cpp:2510
llvm::SelectionDAG::UpdateNodeOperands
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Definition: SelectionDAG.cpp:10513
llvm::SelectionDAG::getNodeIfExists
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
Definition: SelectionDAG.cpp:11007
llvm::SelectionDAG::getTargetConstantPool
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:753
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:570
llvm::SelectionDAG::getMaskedLoad
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
Definition: SelectionDAG.cpp:9664
llvm::SelectionDAG::SplitScalar
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
Definition: SelectionDAG.cpp:12641
llvm::SelectionDAG::getOpcode_EXTEND
static unsigned getOpcode_EXTEND(unsigned Opcode)
Convert *_EXTEND_VECTOR_INREG to *_EXTEND opcode.
Definition: SelectionDAG.h:922
llvm::SelectionDAG::matchBinOpReduction
SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, ArrayRef< ISD::NodeType > CandidateBinOps, bool AllowPartials=false)
Match a binop + shuffle pyramid that represents a horizontal reduction over the elements of a vector ...
Definition: SelectionDAG.cpp:12314
llvm::SelectionDAG::getVectorShuffle
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
Definition: SelectionDAG.cpp:2100
llvm::SelectionDAG::getMaskedScatter
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
Definition: SelectionDAG.cpp:9806
llvm::ShuffleVectorInst::isBitRotateMask
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
Definition: Instructions.cpp:2425
llvm::ShuffleVectorSDNode
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
Definition: SelectionDAGNodes.h:1608
llvm::ShuffleVectorSDNode::getMaskElt
int getMaskElt(unsigned Idx) const
Definition: SelectionDAGNodes.h:1626
llvm::ShuffleVectorSDNode::getMask
ArrayRef< int > getMask() const
Definition: SelectionDAGNodes.h:1621
llvm::ShuffleVectorSDNode::commuteMask
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
Definition: SelectionDAGNodes.h:1649
llvm::SmallBitVector
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
Definition: SmallBitVector.h:35
llvm::SmallBitVector::resize
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
Definition: SmallBitVector.h:332
llvm::SmallBitVector::count
size_type count() const
Returns the number of bits which are set.
Definition: SmallBitVector.h:200
llvm::SmallPtrSetImplBase::reserve
void reserve(size_type NumEntries)
Definition: SmallPtrSet.h:113
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:367
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:502
llvm::SmallSet
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
llvm::SmallSet::insert
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
llvm::SmallSet::size
size_type size() const
Definition: SmallSet.h:161
llvm::SmallVectorBase::empty
bool empty() const
Definition: SmallVector.h:94
llvm::SmallVectorBase::size
size_t size() const
Definition: SmallVector.h:91
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
llvm::SmallVectorImpl::assign
void assign(size_type NumElts, ValueParamT Elt)
Definition: SmallVector.h:717
llvm::SmallVectorImpl::erase
iterator erase(const_iterator CI)
Definition: SmallVector.h:750
llvm::SmallVectorImpl::const_iterator
typename SuperClass::const_iterator const_iterator
Definition: SmallVector.h:591
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:651
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition: SmallVector.h:426
llvm::SmallVectorTemplateCommon::data
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:299
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:290
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2472
llvm::StoreSDNode::getBasePtr
const SDValue & getBasePtr() const
Definition: SelectionDAGNodes.h:2494
llvm::StoreSDNode::getValue
const SDValue & getValue() const
Definition: SelectionDAGNodes.h:2493
llvm::StoreSDNode::isTruncatingStore
bool isTruncatingStore() const
Return true if the op does a truncation before store.
Definition: SelectionDAGNodes.h:2488
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::StringRef::substr
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:556
llvm::StringRef::starts_with
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:250
llvm::StringRef::empty
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
llvm::StringRef::size_type
size_t size_type
Definition: StringRef.h:56
llvm::StringRef::size
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
llvm::StringRef::ends_with
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
Definition: StringRef.h:262
llvm::StringRef::npos
static constexpr size_t npos
Definition: StringRef.h:52
llvm::StringRef::equals_insensitive
bool equals_insensitive(StringRef RHS) const
Check for string equality, ignoring case.
Definition: StringRef.h:163
llvm::StringRef::find_first_not_of
size_t find_first_not_of(char C, size_t From=0) const
Find the first character in the string that is not C or npos if not found.
Definition: StringRef.cpp:251
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:44
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:69
llvm::StringSwitch::Default
R Default(T Value)
Definition: StringSwitch.h:182
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:361
llvm::TargetFrameLowering
Information about stack frame layout on the target.
Definition: TargetFrameLowering.h:45
llvm::TargetFrameLowering::hasFP
virtual bool hasFP(const MachineFunction &MF) const =0
hasFP - Return true if the specified function should have a dedicated frame pointer register.
llvm::TargetFrameLowering::getStackAlign
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Definition: TargetFrameLowering.h:105
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:112
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:278
llvm::TargetLoweringBase::isOperationExpand
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
Definition: TargetLowering.h:1426
llvm::TargetLoweringBase::setBooleanVectorContents
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
Definition: TargetLowering.h:2468
llvm::TargetLoweringBase::setOperationAction
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
Definition: TargetLowering.h:2537
llvm::TargetLoweringBase::setMaxDivRemBitWidthSupported
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
Definition: TargetLowering.h:2747
llvm::TargetLoweringBase::PredictableSelectIsExpensive
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
Definition: TargetLowering.h:3754
llvm::TargetLoweringBase::getValueType
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
Definition: TargetLowering.h:1660
llvm::TargetLoweringBase::LegalizeAction
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
Definition: TargetLowering.h:199
llvm::TargetLoweringBase::getLibcallCallingConv
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Definition: TargetLowering.h:3463
llvm::TargetLoweringBase::MaxStoresPerMemcpyOptSize
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3715
llvm::TargetLoweringBase::emitPatchPoint
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
Definition: TargetLoweringBase.cpp:1140
llvm::TargetLoweringBase::getRegClassFor
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
Definition: TargetLowering.h:1026
llvm::TargetLoweringBase::ShiftLegalizationStrategy
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
Definition: TargetLowering.h:1061
llvm::TargetLoweringBase::getTargetMachine
const TargetMachine & getTargetMachine() const
Definition: TargetLowering.h:362
llvm::TargetLoweringBase::MaxLoadsPerMemcmp
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
Definition: TargetLowering.h:3734
llvm::TargetLoweringBase::setOperationPromotedToType
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
Definition: TargetLowering.h:2691
llvm::TargetLoweringBase::LegalizeTypeAction
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
Definition: TargetLowering.h:209
llvm::TargetLoweringBase::areJTsAllowed
virtual bool areJTsAllowed(const Function *Fn) const
Return true if lowering to a jump table is allowed.
Definition: TargetLowering.h:1369
llvm::TargetLoweringBase::setLibcallCallingConv
void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC)
Set the CallingConv that should be used for the specified libcall.
Definition: TargetLowering.h:3458
llvm::TargetLoweringBase::isOperationLegalOrPromote
bool isOperationLegalOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal using promotion.
Definition: TargetLowering.h:1337
llvm::TargetLoweringBase::addBypassSlowDiv
void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth)
Tells the code generator which bitwidths to bypass.
Definition: TargetLowering.h:2513
llvm::TargetLoweringBase::setMaxLargeFPConvertBitWidthSupported
void setMaxLargeFPConvertBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum fp convert the backend supports.
Definition: TargetLowering.h:2753
llvm::TargetLoweringBase::isTruncStoreLegal
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
Definition: TargetLowering.h:1498
llvm::TargetLoweringBase::isLoadBitCastBeneficial
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
Definition: TargetLoweringBase.cpp:2201
llvm::TargetLoweringBase::setPrefLoopAlignment
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
Definition: TargetLowering.h:2727
llvm::TargetLoweringBase::isCommutativeBinOp
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
Definition: TargetLowering.h:2881
llvm::TargetLoweringBase::setMaxAtomicSizeInBitsSupported
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
Definition: TargetLowering.h:2741
llvm::TargetLoweringBase::getPreferredVectorAction
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
Definition: TargetLowering.h:501
llvm::TargetLoweringBase::getStackPointerRegisterToSaveRestore
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
Definition: TargetLowering.h:1999
llvm::TargetLoweringBase::isOperationCustom
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
Definition: TargetLowering.h:1364
llvm::TargetLoweringBase::MaxStoresPerMemsetOptSize
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3700
llvm::TargetLoweringBase::setBooleanContents
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
Definition: TargetLowering.h:2454
llvm::TargetLoweringBase::MaxStoresPerMemmove
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
Definition: TargetLowering.h:3748
llvm::TargetLoweringBase::getPrefLoopAlignment
virtual Align getPrefLoopAlignment(MachineLoop *ML=nullptr) const
Return the preferred loop alignment.
Definition: TargetLoweringBase.cpp:2004
llvm::TargetLoweringBase::computeRegisterProperties
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
Definition: TargetLoweringBase.cpp:1260
llvm::TargetLoweringBase::shouldFoldConstantShiftPairToMask
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to fold a pair of shifts into a mask.
Definition: TargetLowering.h:814
llvm::TargetLoweringBase::getSetCCResultType
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
Definition: TargetLoweringBase.cpp:1508
llvm::TargetLoweringBase::getTypeToTransformTo
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:1140
llvm::TargetLoweringBase::MaxStoresPerMemmoveOptSize
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3750
llvm::TargetLoweringBase::getBooleanContents
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
Definition: TargetLowering.h:988
llvm::TargetLoweringBase::getPreferredSwitchConditionType
virtual MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const
Returns preferred type for switch condition.
Definition: TargetLoweringBase.cpp:1638
llvm::TargetLoweringBase::addRegisterClass
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
Definition: TargetLowering.h:2520
llvm::TargetLoweringBase::isTypeLegal
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Definition: TargetLowering.h:1077
llvm::TargetLoweringBase::getPointerTy
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Definition: TargetLowering.h:369
llvm::TargetLoweringBase::setLibcallName
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
Definition: TargetLowering.h:3428
llvm::TargetLoweringBase::setPrefFunctionAlignment
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
Definition: TargetLowering.h:2720
llvm::TargetLoweringBase::isOperationLegal
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
Definition: TargetLowering.h:1431
llvm::TargetLoweringBase::MaxStoresPerMemset
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
Definition: TargetLowering.h:3698
llvm::TargetLoweringBase::shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd
virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Definition: TargetLowering.h:839
llvm::TargetLoweringBase::setTruncStoreAction
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
Definition: TargetLowering.h:2600
llvm::TargetLoweringBase::BooleanContent
BooleanContent
Enum that describes how the target represents true/false values.
Definition: TargetLowering.h:234
llvm::TargetLoweringBase::preferredShiftLegalizationStrategy
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
Definition: TargetLowering.h:1067
llvm::TargetLoweringBase::isOperationLegalOrCustom
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
Definition: TargetLowering.h:1323
llvm::TargetLoweringBase::allowsMemoryAccess
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
Definition: TargetLoweringBase.cpp:1732
llvm::TargetLoweringBase::MaxLoadsPerMemcmpOptSize
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3736
llvm::TargetLoweringBase::isBinOp
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
Definition: TargetLowering.h:2925
llvm::TargetLoweringBase::setStackPointerRegisterToSaveRestore
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
Definition: TargetLowering.h:2486
llvm::TargetLoweringBase::isLoadExtLegal
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
Definition: TargetLowering.h:1451
llvm::TargetLoweringBase::AtomicExpansionKind
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
Definition: TargetLowering.h:253
llvm::TargetLoweringBase::setCondCodeAction
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
Definition: TargetLowering.h:2661
llvm::TargetLoweringBase::AndOrSETCCFoldKind
AndOrSETCCFoldKind
Enum of different potentially desirable ways to fold (and/or (setcc ...), (setcc ....
Definition: TargetLowering.h:290
llvm::TargetLoweringBase::setTargetDAGCombine
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
Definition: TargetLowering.h:2706
llvm::TargetLoweringBase::setLoadExtAction
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
Definition: TargetLowering.h:2554
llvm::TargetLoweringBase::NegatibleCost
NegatibleCost
Enum that specifies when a float negation is beneficial.
Definition: TargetLowering.h:282
llvm::TargetLoweringBase::getTypeAction
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
Definition: TargetLowering.h:1127
llvm::TargetLoweringBase::getLibcallName
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
Definition: TargetLowering.h:3437
llvm::TargetLoweringBase::ArgListTy
std::vector< ArgListEntry > ArgListTy
Definition: TargetLowering.h:327
llvm::TargetLoweringBase::MaxStoresPerMemcpy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
Definition: TargetLowering.h:3713
llvm::TargetLoweringBase::setSchedulingPreference
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
Definition: TargetLowering.h:2473
llvm::TargetLoweringBase::shouldConvertPhiType
virtual bool shouldConvertPhiType(Type *From, Type *To) const
Given a set in interconnected phis of type 'From' that are loaded/stored or bitcast to type 'To',...
Definition: TargetLowering.h:2875
llvm::TargetLoweringBase::isOperationLegalOrCustomOrPromote
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
Definition: TargetLowering.h:1351
llvm::TargetLoweringBase::getExtendForContent
static ISD::NodeType getExtendForContent(BooleanContent Content)
Definition: TargetLowering.h:332
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3777
llvm::TargetLowering::buildSDIVPow2WithCMov
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
Definition: TargetLowering.cpp:6253
llvm::TargetLowering::SimplifyDemandedVectorElts
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
Definition: TargetLowering.cpp:3054
llvm::TargetLowering::softenSetCCOperands
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
Definition: TargetLowering.cpp:291
llvm::TargetLowering::makeLibCall
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Definition: TargetLowering.cpp:146
llvm::TargetLowering::expandIndirectJTBranch
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
Definition: TargetLowering.cpp:477
llvm::TargetLowering::getCheaperNegatedExpression
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
Definition: TargetLowering.h:4443
llvm::TargetLowering::SimplifyMultipleUseDemandedBits
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
Definition: TargetLowering.cpp:669
llvm::TargetLowering::SimplifyMultipleUseDemandedVectorElts
SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all bits from only some vector eleme...
Definition: TargetLowering.cpp:943
llvm::TargetLowering::getConstraintType
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
Definition: TargetLowering.cpp:5486
llvm::TargetLowering::ShrinkDemandedConstant
bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
Check to see if the specified operand of the specified instruction is a constant integer.
Definition: TargetLowering.cpp:513
llvm::TargetLowering::LowerToTLSEmulatedModel
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
Definition: TargetLowering.cpp:10376
llvm::TargetLowering::LowerXConstraint
virtual const char * LowerXConstraint(EVT ConstraintVT) const
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
Definition: TargetLowering.cpp:5532
llvm::TargetLowering::LowerCallTo
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
Definition: SelectionDAGBuilder.cpp:10826
llvm::TargetLowering::expandDIVREMByConstant
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
Definition: TargetLowering.cpp:7772
llvm::TargetLowering::isPositionIndependent
bool isPositionIndependent() const
Definition: TargetLowering.cpp:47
llvm::TargetLowering::getNegatedExpression
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
Definition: TargetLowering.cpp:7252
llvm::TargetLowering::getSingleConstraintMatchWeight
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
Definition: TargetLowering.cpp:5926
llvm::TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode
virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
Definition: TargetLowering.cpp:3816
llvm::TargetLowering::getRegForInlineAsmConstraint
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
Definition: TargetLowering.cpp:5630
llvm::TargetLowering::SimplifyDemandedBits
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
Definition: TargetLowering.cpp:1100
llvm::TargetLowering::SimplifyDemandedBitsForTargetNode
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
Definition: TargetLowering.cpp:3803
llvm::TargetLowering::verifyReturnAddressArgumentIsConstant
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
Definition: TargetLowering.cpp:7217
llvm::TargetLowering::isSplatValueForTargetNode
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
Definition: TargetLowering.cpp:3897
llvm::TargetLowering::getVectorElementPointer
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
Definition: TargetLowering.cpp:10333
llvm::TargetLowering::LowerAsmOperandForConstraint
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Definition: TargetLowering.cpp:5548
llvm::TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
Definition: TargetLowering.cpp:3850
llvm::TargetLowering::expandShiftParts
void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const
Expand shift-by-parts.
Definition: TargetLowering.cpp:8135
llvm::TargetLowering::canCreateUndefOrPoisonForTargetNode
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Definition: TargetLowering.cpp:3871
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
llvm::TargetMachine::getTLSModel
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
Definition: TargetMachine.cpp:238
llvm::TargetMachine::useTLSDESC
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
Definition: TargetMachine.cpp:236
llvm::TargetMachine::useEmulatedTLS
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
Definition: TargetMachine.cpp:235
llvm::TargetMachine::Options
TargetOptions Options
Definition: TargetMachine.h:118
llvm::TargetMachine::getCodeModel
CodeModel::Model getCodeModel() const
Returns the code model.
Definition: TargetMachine.h:232
llvm::TargetMachine::getMCAsmInfo
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
Definition: TargetMachine.h:212
llvm::TargetOptions::NoSignedZerosFPMath
unsigned NoSignedZerosFPMath
NoSignedZerosFPMath - This flag is enabled when the -enable-no-signed-zeros-fp-math is specified on t...
Definition: TargetOptions.h:201
llvm::TargetOptions::NoNaNsFPMath
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
Definition: TargetOptions.h:190
llvm::TargetOptions::AllowFPOpFusion
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
Definition: TargetOptions.h:416
llvm::TargetRegisterClass::contains
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
Definition: TargetRegisterInfo.h:95
llvm::TargetRegisterClass::hasSuperClassEq
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
Definition: TargetRegisterInfo.h:143
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:238
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:96
llvm::Target
Target - Wrapper for Target specific information.
Definition: TargetRegistry.h:144
llvm::Triple::isOSMSVCRT
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
Definition: Triple.h:667
llvm::Triple::isOSDarwin
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Definition: Triple.h:558
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::TypeSize::getFixed
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:345
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:261
llvm::Type::getInt1Ty
static IntegerType * getInt1Ty(LLVMContext &C)
llvm::Type::getArrayElementType
Type * getArrayElementType() const
Definition: Type.h:399
llvm::Type::isFloatTy
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition: Type.h:153
llvm::Type::getArrayNumElements
uint64_t getArrayNumElements() const
llvm::Type::getIntNTy
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
llvm::Type::getScalarSizeInBits
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
llvm::Type::isHalfTy
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition: Type.h:142
llvm::Type::isDoubleTy
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition: Type.h:156
llvm::Type::isFloatingPointTy
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition: Type.h:184
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:224
llvm::Type::getPrimitiveSizeInBits
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1833
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::Value::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
llvm::Value::use_begin
use_iterator use_begin()
Definition: Value.h:360
llvm::Value::use_empty
bool use_empty() const
Definition: Value.h:344
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
llvm::VectorType::isValidElementType
static bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
Definition: Type.cpp:671
llvm::VectorType::getElementType
Type * getElementType() const
Definition: DerivedTypes.h:436
llvm::X86FrameLowering::has128ByteRedZone
bool has128ByteRedZone(const MachineFunction &MF) const
Return true if the function has a redzone (accessible bytes past the frame of the top of stack functi...
Definition: X86FrameLowering.cpp:1393
llvm::X86FrameLowering::Uses64BitFramePtr
bool Uses64BitFramePtr
True if the 64-bit frame or stack pointer should be used.
Definition: X86FrameLowering.h:48
llvm::X86InstrInfo::getGlobalBaseReg
unsigned getGlobalBaseReg(MachineFunction *MF) const
getGlobalBaseReg - Return a virtual register initialized with the the global base register value.
Definition: X86InstrInfo.cpp:8922
llvm::X86MachineFunctionInfo
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
Definition: X86MachineFunctionInfo.h:58
llvm::X86MachineFunctionInfo::setAMXProgModel
void setAMXProgModel(AMXProgModelEnum Model)
Definition: X86MachineFunctionInfo.h:262
llvm::X86MachineFunctionInfo::getPreallocatedArgOffsets
ArrayRef< size_t > getPreallocatedArgOffsets(const size_t Id)
Definition: X86MachineFunctionInfo.h:333
llvm::X86MachineFunctionInfo::setRestoreBasePointer
void setRestoreBasePointer(const MachineFunction *MF)
Definition: X86MachineFunctionInfo.cpp:38
llvm::X86MachineFunctionInfo::getPreallocatedStackSize
size_t getPreallocatedStackSize(const size_t Id)
Definition: X86MachineFunctionInfo.h:324
llvm::X86RegisterInfo::getPtrSizedFrameRegister
unsigned getPtrSizedFrameRegister(const MachineFunction &MF) const
Definition: X86RegisterInfo.cpp:1055
llvm::X86RegisterInfo::hasBasePointer
bool hasBasePointer(const MachineFunction &MF) const
Definition: X86RegisterInfo.cpp:767
llvm::X86RegisterInfo::getFrameRegister
Register getFrameRegister(const MachineFunction &MF) const override
Definition: X86RegisterInfo.cpp:1049
llvm::X86RegisterInfo::getDarwinTLSCallPreservedMask
const uint32_t * getDarwinTLSCallPreservedMask() const
Definition: X86RegisterInfo.cpp:541
llvm::X86RegisterInfo::getCallPreservedMask
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
Definition: X86RegisterInfo.cpp:430
llvm::X86RegisterInfo::getStackRegister
Register getStackRegister() const
Definition: X86RegisterInfo.h:167
llvm::X86RegisterInfo::getSlotSize
unsigned getSlotSize() const
Definition: X86RegisterInfo.h:175
llvm::X86RegisterInfo::getBaseRegister
Register getBaseRegister() const
Definition: X86RegisterInfo.h:168
llvm::X86RegisterInfo::getNoPreservedMask
const uint32_t * getNoPreservedMask() const override
Definition: X86RegisterInfo.cpp:537
llvm::X86Subtarget::canExtendTo512BW
bool canExtendTo512BW() const
Definition: X86Subtarget.h:236
llvm::X86Subtarget::hasAnyFMA
bool hasAnyFMA() const
Definition: X86Subtarget.h:203
llvm::X86Subtarget::isOSWindows
bool isOSWindows() const
Definition: X86Subtarget.h:322
llvm::X86Subtarget::isTargetMachO
bool isTargetMachO() const
Definition: X86Subtarget.h:288
llvm::X86Subtarget::useIndirectThunkBranches
bool useIndirectThunkBranches() const
Definition: X86Subtarget.h:221
llvm::X86Subtarget::hasSSE1
bool hasSSE1() const
Definition: X86Subtarget.h:193
llvm::X86Subtarget::isPICStyleGOT
bool isPICStyleGOT() const
Definition: X86Subtarget.h:328
llvm::X86Subtarget::hasSSE42
bool hasSSE42() const
Definition: X86Subtarget.h:198
llvm::X86Subtarget::getTargetLowering
const X86TargetLowering * getTargetLowering() const override
Definition: X86Subtarget.h:118
llvm::X86Subtarget::hasMFence
bool hasMFence() const
Use mfence if we have SSE2 or we're on x86-64 (even if we asked for no-sse2).
Definition: X86Subtarget.h:276
llvm::X86Subtarget::canUseCMOV
bool canUseCMOV() const
Definition: X86Subtarget.h:192
llvm::X86Subtarget::isPICStyleStubPIC
bool isPICStyleStubPIC() const
Definition: X86Subtarget.h:331
llvm::X86Subtarget::isTargetWindowsMSVC
bool isTargetWindowsMSVC() const
Definition: X86Subtarget.h:300
llvm::X86Subtarget::canUseCMPXCHG8B
bool canUseCMPXCHG8B() const
Definition: X86Subtarget.h:185
llvm::X86Subtarget::isTargetDarwin
bool isTargetDarwin() const
Definition: X86Subtarget.h:280
llvm::X86Subtarget::isTargetWin64
bool isTargetWin64() const
Definition: X86Subtarget.h:324
llvm::X86Subtarget::isTarget64BitLP64
bool isTarget64BitLP64() const
Is this x86_64 with the LP64 programming model (standard AMD64, no x32)?
Definition: X86Subtarget.h:178
llvm::X86Subtarget::getTargetTriple
const Triple & getTargetTriple() const
Definition: X86Subtarget.h:278
llvm::X86Subtarget::getInstrInfo
const X86InstrInfo * getInstrInfo() const override
Definition: X86Subtarget.h:122
llvm::X86Subtarget::useAVX512Regs
bool useAVX512Regs() const
Definition: X86Subtarget.h:253
llvm::X86Subtarget::hasSSE3
bool hasSSE3() const
Definition: X86Subtarget.h:195
llvm::X86Subtarget::isCallingConvWin64
bool isCallingConvWin64(CallingConv::ID CC) const
Definition: X86Subtarget.h:337
llvm::X86Subtarget::hasAVX512
bool hasAVX512() const
Definition: X86Subtarget.h:201
llvm::X86Subtarget::canExtendTo512DQ
bool canExtendTo512DQ() const
Definition: X86Subtarget.h:232
llvm::X86Subtarget::hasSSE41
bool hasSSE41() const
Definition: X86Subtarget.h:197
llvm::X86Subtarget::isTargetELF
bool isTargetELF() const
Definition: X86Subtarget.h:286
llvm::X86Subtarget::hasSSEPrefetch
bool hasSSEPrefetch() const
Definition: X86Subtarget.h:209
llvm::X86Subtarget::canUseCMPXCHG16B
bool canUseCMPXCHG16B() const
Definition: X86Subtarget.h:186
llvm::X86Subtarget::classifyGlobalReference
unsigned char classifyGlobalReference(const GlobalValue *GV, const Module &M) const
Definition: X86Subtarget.cpp:125
llvm::X86Subtarget::hasSSE2
bool hasSSE2() const
Definition: X86Subtarget.h:194
llvm::X86Subtarget::hasSSSE3
bool hasSSSE3() const
Definition: X86Subtarget.h:196
llvm::X86Subtarget::hasInt256
bool hasInt256() const
Definition: X86Subtarget.h:202
llvm::X86Subtarget::isPICStyleRIPRel
bool isPICStyleRIPRel() const
Definition: X86Subtarget.h:329
llvm::X86Subtarget::isTargetCygMing
bool isTargetCygMing() const
Definition: X86Subtarget.h:320
llvm::X86Subtarget::classifyLocalReference
unsigned char classifyLocalReference(const GlobalValue *GV) const
Classify a global variable reference for the current subtarget according to how we should reference i...
Definition: X86Subtarget.cpp:72
llvm::X86Subtarget::classifyBlockAddressReference
unsigned char classifyBlockAddressReference() const
Classify a blockaddress reference for the current subtarget according to how we should reference it i...
Definition: X86Subtarget.cpp:60
llvm::X86Subtarget::isTargetPS
bool isTargetPS() const
Definition: X86Subtarget.h:284
llvm::X86Subtarget::getRegisterInfo
const X86RegisterInfo * getRegisterInfo() const override
Definition: X86Subtarget.h:132
llvm::X86Subtarget::hasAVX
bool hasAVX() const
Definition: X86Subtarget.h:199
llvm::X86Subtarget::isTargetWindowsGNU
bool isTargetWindowsGNU() const
Definition: X86Subtarget.h:312
llvm::X86Subtarget::getPreferVectorWidth
unsigned getPreferVectorWidth() const
Definition: X86Subtarget.h:225
llvm::X86Subtarget::isTargetWindowsItanium
bool isTargetWindowsItanium() const
Definition: X86Subtarget.h:316
llvm::X86Subtarget::isTargetNaCl64
bool isTargetNaCl64() const
Definition: X86Subtarget.h:296
llvm::X86Subtarget::getFrameLowering
const X86FrameLowering * getFrameLowering() const override
Definition: X86Subtarget.h:124
llvm::X86Subtarget::useBWIRegs
bool useBWIRegs() const
Definition: X86Subtarget.h:262
llvm::X86Subtarget::classifyGlobalFunctionReference
unsigned char classifyGlobalFunctionReference(const GlobalValue *GV, const Module &M) const
Classify a global function reference for the current subtarget.
Definition: X86Subtarget.cpp:192
llvm::X86Subtarget::hasAVX2
bool hasAVX2() const
Definition: X86Subtarget.h:200
llvm::X86TargetLowering::shouldFormOverflowOp
bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const override
Overflow nodes should get combined/lowered to optimal instructions (they should allow eliminating exp...
Definition: X86ISelLowering.cpp:3232
llvm::X86TargetLowering::getPrefLoopAlignment
Align getPrefLoopAlignment(MachineLoop *ML) const override
Return the preferred loop alignment.
Definition: X86ISelLowering.cpp:59666
llvm::X86TargetLowering::isLegalAddImmediate
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
Definition: X86ISelLowering.cpp:34297
llvm::X86TargetLowering::preferSextInRegOfTruncate
bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT, EVT ExtVT) const override
Definition: X86ISelLowering.cpp:58185
llvm::X86TargetLowering::PerformDAGCombine
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
Definition: X86ISelLowering.cpp:57981
llvm::X86TargetLowering::preferABDSToABSWithNSW
bool preferABDSToABSWithNSW(EVT VT) const override
Definition: X86ISelLowering.cpp:58180
llvm::X86TargetLowering::isCheapToSpeculateCtlz
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Definition: X86ISelLowering.cpp:3247
llvm::X86TargetLowering::getJumpTableEncoding
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
Definition: X86ISelLoweringCall.cpp:415
llvm::X86TargetLowering::BuildFILD
std::pair< SDValue, SDValue > BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer, MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const
Definition: X86ISelLowering.cpp:19442
llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode
bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded vector elements, returning true on success...
Definition: X86ISelLowering.cpp:41966
llvm::X86TargetLowering::LowerAsmOutputForConstraint
SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, const SDLoc &DL, const AsmOperandInfo &Constraint, SelectionDAG &DAG) const override
Handle Lowering flag assembly outputs.
Definition: X86ISelLowering.cpp:58754
llvm::X86TargetLowering::LowerXConstraint
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
Definition: X86ISelLowering.cpp:58742
llvm::X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode
SDValue SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const override
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
Definition: X86ISelLowering.cpp:43269
llvm::X86TargetLowering::isSplatValueForTargetNode
bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth) const override
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
Definition: X86ISelLowering.cpp:43458
llvm::X86TargetLowering::convertSelectOfConstantsToMath
bool convertSelectOfConstantsToMath(EVT VT) const override
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
Definition: X86ISelLowering.cpp:3158
llvm::X86TargetLowering::getConstraintType
ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint letter, return the type of constraint for this target.
Definition: X86ISelLowering.cpp:58513
llvm::X86TargetLowering::getExceptionSelectorRegister
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Definition: X86ISelLowering.cpp:27426
llvm::X86TargetLowering::isVectorShiftByScalarCheap
bool isVectorShiftByScalarCheap(Type *Ty) const override
This is used to enable splatted operand transforms for vector shifts and vector funnel shifts.
Definition: X86ISelLowering.cpp:34211
llvm::X86TargetLowering::preferredShiftLegalizationStrategy
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
Definition: X86ISelLowering.cpp:3492
llvm::X86TargetLowering::LowerOperation
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
Provide custom lowering hooks for some operations.
Definition: X86ISelLowering.cpp:32393
llvm::X86TargetLowering::isLegalStoreImmediate
bool isLegalStoreImmediate(int64_t Imm) const override
Return true if the specified immediate is legal for the value input of a store instruction.
Definition: X86ISelLowering.cpp:34302
llvm::X86TargetLowering::visitMaskedStore
SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const override
Definition: X86ISelLowering.cpp:32375
llvm::X86TargetLowering::getNegatedExpression
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, NegatibleCost &Cost, unsigned Depth) const override
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
Definition: X86ISelLowering.cpp:52975
llvm::X86TargetLowering::isTypeDesirableForOp
bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
Definition: X86ISelLowering.cpp:58190
llvm::X86TargetLowering::isCtlzFast
bool isCtlzFast() const override
Return true if ctlz instruction is fast.
Definition: X86ISelLowering.cpp:3300
llvm::X86TargetLowering::getRegisterByName
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
Definition: X86ISelLowering.cpp:27379
llvm::X86TargetLowering::shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Definition: X86ISelLowering.cpp:3347
llvm::X86TargetLowering::isNarrowingProfitable
bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
Definition: X86ISelLowering.cpp:34447
llvm::X86TargetLowering::supportSwiftError
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
Definition: X86ISelLowering.cpp:59543
llvm::X86TargetLowering::isCheapToSpeculateCttz
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
Definition: X86ISelLowering.cpp:3240
llvm::X86TargetLowering::shouldSplatInsEltVarIndex
bool shouldSplatInsEltVarIndex(EVT VT) const override
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
Definition: X86ISelLowering.cpp:3501
llvm::X86TargetLowering::shouldSinkOperands
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Return true if sinking I's operands to the same basic block as I is profitable, e....
Definition: X86ISelLowering.cpp:34348
llvm::X86TargetLowering::isInlineAsmTargetBranch
bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const override
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
Definition: X86ISelLowering.cpp:32333
llvm::X86TargetLowering::hasFastEqualityCompare
MVT hasFastEqualityCompare(unsigned NumBits) const override
Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
Definition: X86ISelLowering.cpp:3507
llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle
bool SimplifyDemandedVectorEltsForTargetShuffle(SDValue Op, const APInt &DemandedElts, unsigned MaskIndex, TargetLoweringOpt &TLO, unsigned Depth) const
Definition: X86ISelLowering.cpp:41899
llvm::X86TargetLowering::isLegalICmpImmediate
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
Definition: X86ISelLowering.cpp:34293
llvm::X86TargetLowering::shouldFoldSelectWithIdentityConstant
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
Definition: X86ISelLowering.cpp:34452
llvm::X86TargetLowering::hasInlineStackProbe
bool hasInlineStackProbe(const MachineFunction &MF) const override
Returns true if stack probing through inline assembly is requested.
Definition: X86ISelLowering.cpp:59618
llvm::X86TargetLowering::EmitInstrWithCustomInserter
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition: X86ISelLowering.cpp:36605
llvm::X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand
unsigned preferedOpcodeForCmpEqPiecesOfOperand(EVT VT, unsigned ShiftOpc, bool MayTransformRotate, const APInt &ShiftOrRotateAmt, const std::optional< APInt > &AndMask) const override
Definition: X86ISelLowering.cpp:3369
llvm::X86TargetLowering::isXAndYEqZeroPreferableToXAndYEqY
bool isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond, EVT VT) const override
Definition: X86ISelLowering.cpp:22791
llvm::X86TargetLowering::canMergeStoresTo
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
Definition: X86ISelLowering.cpp:3282
llvm::X86TargetLowering::ExpandInlineAsm
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
Definition: X86ISelLowering.cpp:58401
llvm::X86TargetLowering::hasAndNot
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
Definition: X86ISelLowering.cpp:3325
llvm::X86TargetLowering::shouldReduceLoadWidth
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const override
Return true if we believe it is correct and profitable to reduce the load node to a smaller type.
Definition: X86ISelLowering.cpp:3103
llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
Definition: X86ISelLowering.cpp:42755
llvm::X86TargetLowering::shouldConvertConstantLoadToIntImm
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
Definition: X86ISelLowering.cpp:3139
llvm::X86TargetLowering::preferScalarizeSplat
bool preferScalarizeSplat(SDNode *N) const override
Definition: X86ISelLowering.cpp:3454
llvm::X86TargetLowering::getRegForInlineAsmConstraint
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
Definition: X86ISelLowering.cpp:58978
llvm::X86TargetLowering::isFPImmLegal
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
Definition: X86ISelLowering.cpp:3095
llvm::X86TargetLowering::shouldFoldConstantShiftPairToMask
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
Definition: X86ISelLowering.cpp:3458
llvm::X86TargetLowering::EmitKCFICheck
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
Definition: X86ISelLowering.cpp:59548
llvm::X86TargetLowering::isLoadBitCastBeneficial
bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const override
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
Definition: X86ISelLowering.cpp:3264
llvm::X86TargetLowering::useLoadStackGuardNode
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Definition: X86ISelLowering.cpp:2600
llvm::X86TargetLowering::hasAndNotCompare
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
Definition: X86ISelLowering.cpp:3309
llvm::X86TargetLowering::reduceSelectOfFPConstantLoads
bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
Definition: X86ISelLowering.cpp:3149
llvm::X86TargetLowering::getStackProbeSymbolName
StringRef getStackProbeSymbolName(const MachineFunction &MF) const override
Returns the name of the symbol used to emit stack probes or the empty string if not applicable.
Definition: X86ISelLowering.cpp:59636
llvm::X86TargetLowering::hasBitTest
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
Definition: X86ISelLowering.cpp:3342
llvm::X86TargetLowering::allowTruncateForTailCall
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
Definition: X86ISelLowering.cpp:34279
llvm::X86TargetLowering::isShuffleMaskLegal
bool isShuffleMaskLegal(ArrayRef< int > Mask, EVT VT) const override
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
Definition: X86ISelLowering.cpp:34470
llvm::X86TargetLowering::useStackGuardXorFP
bool useStackGuardXorFP() const override
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
Definition: X86ISelLowering.cpp:2604
llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine the number of bits in the operation that are sign bits.
Definition: X86ISelLowering.cpp:37715
llvm::X86TargetLowering::shouldScalarizeBinop
bool shouldScalarizeBinop(SDValue) const override
Scalar ops always have equal or better analysis/performance/power than the vector equivalent,...
Definition: X86ISelLowering.cpp:3213
llvm::X86TargetLowering::isTruncateFree
bool isTruncateFree(Type *Ty1, Type *Ty2) const override
Return true if it's free to truncate a value of type Ty1 to type Ty2.
Definition: X86ISelLowering.cpp:34271
llvm::X86TargetLowering::decomposeMulByConstant
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
Definition: X86ISelLowering.cpp:3167
llvm::X86TargetLowering::areJTsAllowed
bool areJTsAllowed(const Function *Fn) const override
Returns true if lowering to a jump table is allowed.
Definition: X86ISelLowering.cpp:34499
llvm::X86TargetLowering::isCommutativeBinOp
bool isCommutativeBinOp(unsigned Opcode) const override
Returns true if the opcode is a commutative binary operation.
Definition: X86ISelLowering.cpp:34254
llvm::X86TargetLowering::isScalarFPTypeInSSEReg
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
Definition: X86ISelLowering.cpp:3259
llvm::X86TargetLowering::getTargetNodeName
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Definition: X86ISelLowering.cpp:33703
llvm::X86TargetLowering::getPreferredSwitchConditionType
MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const override
Returns preferred type for switch condition.
Definition: X86ISelLowering.cpp:34508
llvm::X86TargetLowering::visitMaskedLoad
SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const override
Definition: X86ISelLowering.cpp:32355
llvm::X86TargetLowering::isFMAFasterThanFMulAndFAdd
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
Definition: X86ISelLowering.cpp:34424
llvm::X86TargetLowering::isExtractSubvectorCheap
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
Definition: X86ISelLowering.cpp:3199
llvm::X86TargetLowering::isVectorClearMaskLegal
bool isVectorClearMaskLegal(ArrayRef< int > Mask, EVT VT) const override
Similar to isShuffleMaskLegal.
Definition: X86ISelLowering.cpp:34487
llvm::X86TargetLowering::getSingleConstraintMatchWeight
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &Info, const char *Constraint) const override
Examine constraint string and operand type and determine a weight value.
Definition: X86ISelLowering.cpp:58592
llvm::X86TargetLowering::isIntDivCheap
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
Definition: X86ISelLowering.cpp:59482
llvm::X86TargetLowering::getPreferredVectorAction
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Customize the preferred legalization strategy for certain types.
Definition: X86ISelLowering.cpp:2618
llvm::X86TargetLowering::shouldConvertPhiType
bool shouldConvertPhiType(Type *From, Type *To) const override
Given a set in interconnected phis of type 'From' that are loaded/stored or bitcast to type 'To',...
Definition: X86ISelLowering.cpp:34405
llvm::X86TargetLowering::hasStackProbeSymbol
bool hasStackProbeSymbol(const MachineFunction &MF) const override
Returns true if stack probing through a function call is requested.
Definition: X86ISelLowering.cpp:59613
llvm::X86TargetLowering::isZExtFree
bool isZExtFree(Type *Ty1, Type *Ty2) const override
Return true if any actual instruction that defines a value of type Ty1 implicit zero-extends the valu...
Definition: X86ISelLowering.cpp:34314
llvm::X86TargetLowering::allowsMemoryAccess
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
This function returns true if the memory access is aligned or if the target allows this specific unal...
Definition: X86ISelLoweringCall.cpp:374
llvm::X86TargetLowering::isLegalAddressingMode
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
Definition: X86ISelLowering.cpp:34159
llvm::X86TargetLowering::emitStackGuardXorFP
SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const override
Definition: X86ISelLowering.cpp:2609
llvm::X86TargetLowering::isDesirableToCombineLogicOpOfSETCC
TargetLowering::AndOrSETCCFoldKind isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const override
Return prefered fold type, Abs if this is a vector, AddAnd if its an integer, None otherwise.
Definition: X86ISelLowering.cpp:58259
llvm::X86TargetLowering::shouldFoldMaskToVariableShiftPair
bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
Definition: X86ISelLowering.cpp:3477
llvm::X86TargetLowering::addressingModeSupportsTLS
bool addressingModeSupportsTLS(const GlobalValue &GV) const override
Returns true if the targets addressing mode can target thread local storage (TLS).
Definition: X86ISelLowering.cpp:18990
llvm::X86TargetLowering::getReturnAddressFrameIndex
SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const
Definition: X86ISelLowering.cpp:2754
llvm::X86TargetLowering::targetShrinkDemandedConstant
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
Definition: X86ISelLowering.cpp:37097
llvm::X86TargetLowering::getExceptionPointerRegister
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
Definition: X86ISelLowering.cpp:27418
llvm::X86TargetLowering::expandIndirectJTBranch
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expansion.
Definition: X86ISelLowering.cpp:58240
llvm::X86TargetLowering::createFastISel
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
Definition: X86ISelLowering.cpp:2635
llvm::X86TargetLowering::computeKnownBitsForTargetNode
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
Definition: X86ISelLowering.cpp:37276
llvm::X86TargetLowering::isBinOp
bool isBinOp(unsigned Opcode) const override
Add x86-specific opcodes to the default list.
Definition: X86ISelLowering.cpp:34234
llvm::X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode
bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const override
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
Definition: X86ISelLowering.cpp:43389
llvm::X86TargetLowering::IsDesirableToPromoteOp
bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
Definition: X86ISelLowering.cpp:58281
llvm::X86TargetLowering::unwrapAddress
SDValue unwrapAddress(SDValue N) const override
Definition: X86ISelLowering.cpp:37884
llvm::X86TargetLowering::getJumpConditionMergingParams
CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs) const override
Definition: X86ISelLowering.cpp:3437
llvm::X86TargetLowering::getSetCCResultType
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the value type to use for ISD::SETCC.
Definition: X86ISelLoweringCall.cpp:204
llvm::X86TargetLowering::X86TargetLowering
X86TargetLowering(const X86TargetMachine &TM, const X86Subtarget &STI)
Definition: X86ISelLowering.cpp:127
llvm::X86TargetLowering::isVectorLoadExtDesirable
bool isVectorLoadExtDesirable(SDValue) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
Definition: X86ISelLowering.cpp:34411
llvm::X86TargetLowering::getTgtMemIntrinsic
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
Definition: X86ISelLowering.cpp:2946
llvm::X86TargetLowering::getTargetConstantFromLoad
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
Definition: X86ISelLowering.cpp:4728
llvm::X86TargetLowering::getTypeToTransformTo
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override
For types supported by the target, this is an identity function.
Definition: X86ISelLowering.h:1635
llvm::X86TargetLowering::canCreateUndefOrPoisonForTargetNode
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
Definition: X86ISelLowering.cpp:43425
llvm::X86TargetLowering::getStackProbeSize
unsigned getStackProbeSize(const MachineFunction &MF) const
Definition: X86ISelLowering.cpp:59659
llvm::X86TargetLowering::ShouldShrinkFPConstant
bool ShouldShrinkFPConstant(EVT VT) const override
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
Definition: X86ISelLowering.cpp:3252
llvm::X86TargetLowering::ReplaceNodeResults
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
Replace the results of node with an illegal result type with new values built out of custom code.
Definition: X86ISelLowering.cpp:32553
llvm::X86TargetLowering::isMaskAndCmp0FoldingBeneficial
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
Definition: X86ISelLowering.cpp:3304
llvm::X86TargetLowering::LowerAsmOperandForConstraint
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
Definition: X86ISelLowering.cpp:58781
llvm::X86TargetLowering::needsFixedCatchObjects
bool needsFixedCatchObjects() const override
Definition: X86ISelLowering.cpp:27434
llvm::details::FixedOrScalableQuantity::getFixedValue
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:202
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::ilist_detail::node_parent_access::getParent
const ParentTy * getParent() const
Definition: ilist_node.h:32
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:132
uint16_t
uint32_t
uint64_t
INT64_MIN
#define INT64_MIN
Definition: DataTypes.h:74
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:395
llvm::AMDGPU::HSAMD::Kernel::Key::Attrs
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
Definition: AMDGPUMetadata.h:393
llvm::APIntOps::ScaleBitMask
APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
Definition: APInt.cpp:2978
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
llvm::COFF::Entry
@ Entry
Definition: COFF.h:826
llvm::CallingConv::X86_ThisCall
@ X86_ThisCall
Similar to X86_StdCall.
Definition: CallingConv.h:122
llvm::CallingConv::X86_StdCall
@ X86_StdCall
stdcall is mostly used by the Win32 API.
Definition: CallingConv.h:99
llvm::CallingConv::Fast
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
llvm::CallingConv::Tail
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition: CallingConv.h:76
llvm::CallingConv::SwiftTail
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
Definition: CallingConv.h:87
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::CallingConv::X86_FastCall
@ X86_FastCall
'fast' analog of X86_StdCall.
Definition: CallingConv.h:103
llvm::ISD::isConstantSplatVectorAllOnes
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
Definition: SelectionDAG.cpp:179
llvm::ISD::isNON_EXTLoad
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
Definition: SelectionDAGNodes.h:3197
llvm::ISD::NodeType
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:779
llvm::ISD::MERGE_VALUES
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:243
llvm::ISD::STACKRESTORE
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1194
llvm::ISD::STACKSAVE
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1190
llvm::ISD::CTLZ_ZERO_UNDEF
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:752
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:490
llvm::ISD::DELETED_NODE
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
llvm::ISD::MLOAD
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
Definition: ISDOpcodes.h:1355
llvm::ISD::EH_SJLJ_LONGJMP
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition: ISDOpcodes.h:153
llvm::ISD::FGETSIGN
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:511
llvm::ISD::SMUL_LOHI
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:257
llvm::ISD::ATOMIC_LOAD_NAND
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1337
llvm::ISD::INSERT_SUBVECTOR
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:573
llvm::ISD::BSWAP
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:743
llvm::ISD::VAEND
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1223
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
llvm::ISD::ATOMIC_LOAD_MAX
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1339
llvm::ISD::ATOMIC_STORE
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1309
llvm::ISD::STRICT_FCEIL
@ STRICT_FCEIL
Definition: ISDOpcodes.h:440
llvm::ISD::ATOMIC_LOAD_UMIN
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1340
llvm::ISD::FRAME_TO_ARGS_OFFSET
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
Definition: ISDOpcodes.h:130
llvm::ISD::RESET_FPENV
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1070
llvm::ISD::FMAD
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:501
llvm::ISD::STRICT_FTANH
@ STRICT_FTANH
Definition: ISDOpcodes.h:430
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:246
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1099
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:813
llvm::ISD::FMA
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:497
llvm::ISD::INTRINSIC_VOID
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:205
llvm::ISD::EH_SJLJ_SETUP_DISPATCH
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition: ISDOpcodes.h:157
llvm::ISD::GlobalAddress
@ GlobalAddress
Definition: ISDOpcodes.h:78
llvm::ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1322
llvm::ISD::STRICT_FMINIMUM
@ STRICT_FMINIMUM
Definition: ISDOpcodes.h:450
llvm::ISD::SINT_TO_FP
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:840
llvm::ISD::CONCAT_VECTORS
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:557
llvm::ISD::FADD
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:397
llvm::ISD::ABS
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:716
llvm::ISD::MEMBARRIER
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
Definition: ISDOpcodes.h:1296
llvm::ISD::ATOMIC_FENCE
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:1301
llvm::ISD::SIGN_EXTEND_VECTOR_INREG
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:870
llvm::ISD::SDIVREM
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:262
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:491
llvm::ISD::FP16_TO_FP
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:963
llvm::ISD::STRICT_FLOG2
@ STRICT_FLOG2
Definition: ISDOpcodes.h:435
llvm::ISD::ATOMIC_LOAD_OR
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1335
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:953
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:236
llvm::ISD::ATOMIC_LOAD_XOR
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1336
llvm::ISD::INIT_TRAMPOLINE
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:1267
llvm::ISD::FLDEXP
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:996
llvm::ISD::STRICT_FSQRT
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:418
llvm::ISD::BUILTIN_OP_END
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1480
llvm::ISD::GlobalTLSAddress
@ GlobalTLSAddress
Definition: ISDOpcodes.h:79
llvm::ISD::EH_LABEL
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1170
llvm::ISD::EH_RETURN
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition: ISDOpcodes.h:141
llvm::ISD::SET_ROUNDING
@ SET_ROUNDING
Set rounding mode.
Definition: ISDOpcodes.h:935
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:804
llvm::ISD::STRICT_FASIN
@ STRICT_FASIN
Definition: ISDOpcodes.h:425
llvm::ISD::AVGCEILS
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:684
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:464
llvm::ISD::SCALAR_TO_VECTOR
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:634
llvm::ISD::ADDROFRETURNADDR
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition: ISDOpcodes.h:107
llvm::ISD::BR
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:1115
llvm::ISD::STRICT_FATAN
@ STRICT_FATAN
Definition: ISDOpcodes.h:427
llvm::ISD::CTTZ_ZERO_UNDEF
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:751
llvm::ISD::PREFETCH
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1289
llvm::ISD::FSINCOS
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:1056
llvm::ISD::SETCCCARRY
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:787
llvm::ISD::FNEG
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:980
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1145
llvm::ISD::SSUBO
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:334
llvm::ISD::ATOMIC_LOAD_MIN
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1338
llvm::ISD::BR_JT
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1124
llvm::ISD::GC_TRANSITION_START
@ GC_TRANSITION_START
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
Definition: ISDOpcodes.h:1381
llvm::ISD::SSUBSAT
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:356
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:756
llvm::ISD::ATOMIC_LOAD
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1305
llvm::ISD::UNDEF
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:218
llvm::ISD::SPLAT_VECTOR
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:641
llvm::ISD::VACOPY
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1219
llvm::ISD::CopyFromReg
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:215
llvm::ISD::SADDO
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:330
llvm::ISD::STRICT_FTRUNC
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:444
llvm::ISD::GET_ROUNDING
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:930
llvm::ISD::STRICT_FP_TO_FP16
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:966
llvm::ISD::MULHU
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:673
llvm::ISD::STRICT_FP16_TO_FP
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:965
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:734
llvm::ISD::VECTOR_SHUFFLE
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:614
llvm::ISD::ATOMIC_LOAD_AND
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1333
llvm::ISD::EXTRACT_SUBVECTOR
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:587
llvm::ISD::FMINNUM_IEEE
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
Definition: ISDOpcodes.h:1041
llvm::ISD::STRICT_FMAXIMUM
@ STRICT_FMAXIMUM
Definition: ISDOpcodes.h:449
llvm::ISD::STRICT_FMAXNUM
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:438
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:549
llvm::ISD::CopyToReg
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:209
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:810
llvm::ISD::DEBUGTRAP
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1279
llvm::ISD::FP_TO_UINT_SAT
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:906
llvm::ISD::STRICT_FMINNUM
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:439
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:771
llvm::ISD::STRICT_FSINH
@ STRICT_FSINH
Definition: ISDOpcodes.h:428
llvm::ISD::ATOMIC_LOAD_UMAX
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1341
llvm::ISD::LOCAL_RECOVER
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition: ISDOpcodes.h:120
llvm::ISD::FMINNUM
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:1028
llvm::ISD::UBSANTRAP
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1283
llvm::ISD::SMULO
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:338
llvm::ISD::DYNAMIC_STACKALLOC
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1109
llvm::ISD::ConstantPool
@ ConstantPool
Definition: ISDOpcodes.h:82
llvm::ISD::ANY_EXTEND_VECTOR_INREG
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:859
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:848
llvm::ISD::SMIN
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:696
llvm::ISD::FP_EXTEND
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:938
llvm::ISD::STRICT_FROUND
@ STRICT_FROUND
Definition: ISDOpcodes.h:442
llvm::ISD::VSELECT
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:765
llvm::ISD::UADDO_CARRY
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:310
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:463
llvm::ISD::MGATHER
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
Definition: ISDOpcodes.h:1367
llvm::ISD::STRICT_BF16_TO_FP
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:974
llvm::ISD::STRICT_FFLOOR
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:441
llvm::ISD::STRICT_FROUNDEVEN
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:443
llvm::ISD::BF16_TO_FP
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:972
llvm::ISD::FRAMEADDR
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:100
llvm::ISD::ATOMIC_LOAD_ADD
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1331
llvm::ISD::STRICT_FP_TO_UINT
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:457
llvm::ISD::STRICT_FP_ROUND
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:479
llvm::ISD::STRICT_FP_TO_SINT
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:456
llvm::ISD::FMINIMUM
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:1047
llvm::ISD::ATOMIC_LOAD_SUB
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1332
llvm::ISD::FP_TO_SINT
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:886
llvm::ISD::READCYCLECOUNTER
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1250
llvm::ISD::TargetConstant
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition: ISDOpcodes.h:164
llvm::ISD::STRICT_FP_EXTEND
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:484
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:708
llvm::ISD::TRAP
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1276
llvm::ISD::INTRINSIC_WO_CHAIN
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:190
llvm::ISD::GET_FPENV_MEM
@ GET_FPENV_MEM
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1075
llvm::ISD::STRICT_FCOSH
@ STRICT_FCOSH
Definition: ISDOpcodes.h:429
llvm::ISD::STRICT_FP_TO_BF16
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:975
llvm::ISD::AVGFLOORS
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:679
llvm::ISD::STRICT_FADD
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:407
llvm::ISD::STRICT_FLOG10
@ STRICT_FLOG10
Definition: ISDOpcodes.h:434
llvm::ISD::FREEZE
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition: ISDOpcodes.h:223
llvm::ISD::INSERT_VECTOR_ELT
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:538
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
llvm::ISD::STRICT_FEXP2
@ STRICT_FEXP2
Definition: ISDOpcodes.h:432
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1330
llvm::ISD::ExternalSymbol
@ ExternalSymbol
Definition: ISDOpcodes.h:83
llvm::ISD::FFREXP
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:1001
llvm::ISD::FP_ROUND
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:919
llvm::ISD::STRICT_FLDEXP
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:421
llvm::ISD::ZERO_EXTEND_VECTOR_INREG
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:881
llvm::ISD::ADDRSPACECAST
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:957
llvm::ISD::STRICT_FNEARBYINT
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:437
llvm::ISD::FP_TO_SINT_SAT
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:905
llvm::ISD::EH_SJLJ_SETJMP
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition: ISDOpcodes.h:147
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:816
llvm::ISD::VAARG
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1214
llvm::ISD::BRCOND
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1138
llvm::ISD::BlockAddress
@ BlockAddress
Definition: ISDOpcodes.h:84
llvm::ISD::SHL_PARTS
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:793
llvm::ISD::GC_TRANSITION_END
@ GC_TRANSITION_END
Definition: ISDOpcodes.h:1382
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::ISD::FCOPYSIGN
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:507
llvm::ISD::SADDSAT
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:347
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::ISD::STRICT_FRINT
@ STRICT_FRINT
Definition: ISDOpcodes.h:436
llvm::ISD::SET_FPENV_MEM
@ SET_FPENV_MEM
Sets the current floating point environment.
Definition: ISDOpcodes.h:1080
llvm::ISD::ABDS
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition: ISDOpcodes.h:691
llvm::ISD::ADJUST_TRAMPOLINE
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:1273
llvm::ISD::SADDO_CARRY
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:320
llvm::ISD::INTRINSIC_W_CHAIN
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:198
llvm::ISD::STRICT_FACOS
@ STRICT_FACOS
Definition: ISDOpcodes.h:426
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:529
llvm::ISD::isExtVecInRegOpcode
bool isExtVecInRegOpcode(unsigned Opcode)
Definition: ISDOpcodes.h:1679
llvm::ISD::isOverflowIntrOpRes
bool isOverflowIntrOpRes(SDValue Op)
Returns true if the specified value is the overflow result from one of the overflow intrinsic nodes.
Definition: SelectionDAGNodes.h:3274
llvm::ISD::isBuildVectorOfConstantSDNodes
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
Definition: SelectionDAG.cpp:278
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:3228
llvm::ISD::matchUnaryPredicate
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
Definition: SelectionDAGNodes.h:3249
llvm::ISD::isExtOpcode
bool isExtOpcode(unsigned Opcode)
Definition: ISDOpcodes.h:1674
llvm::ISD::isConstantSplatVectorAllZeros
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
Definition: SelectionDAG.cpp:228
llvm::ISD::getSetCCInverse
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
Definition: SelectionDAG.cpp:602
llvm::ISD::isBitwiseLogicOp
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
Definition: ISDOpcodes.h:1495
llvm::ISD::isTrueWhenEqual
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
Definition: ISDOpcodes.h:1661
llvm::ISD::isUNINDEXEDLoad
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
Definition: SelectionDAGNodes.h:3221
llvm::ISD::isEXTLoad
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
Definition: SelectionDAGNodes.h:3203
llvm::ISD::isFreezeUndef
bool isFreezeUndef(const SDNode *N)
Return true if the specified node is FREEZE(UNDEF).
Definition: SelectionDAG.cpp:350
llvm::ISD::getSetCCSwappedOperands
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
Definition: SelectionDAG.cpp:579
llvm::ISD::isBuildVectorAllZeros
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
Definition: SelectionDAG.cpp:274
llvm::ISD::isSignedIntSetCC
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
Definition: ISDOpcodes.h:1636
llvm::ISD::isConstantSplatVector
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
Definition: SelectionDAG.cpp:144
llvm::ISD::isBuildVectorOfConstantFPSDNodes
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
Definition: SelectionDAG.cpp:291
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1603
llvm::ISD::isBuildVectorAllOnes
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
Definition: SelectionDAG.cpp:270
llvm::ISD::LoadExtType
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1583
llvm::ISD::isUnsignedIntSetCC
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
Definition: ISDOpcodes.h:1642
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:3190
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1539
llvm::PatternMatch::m_AllOnes
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
Definition: PatternMatch.h:524
llvm::PatternMatch::m_And
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1206
llvm::PatternMatch::m_SignMask
cst_pred_ty< is_sign_mask > m_SignMask()
Match an integer or vector with only the sign bit(s) set.
Definition: PatternMatch.h:664
llvm::PatternMatch::m_AShr
BinaryOp_match< LHS, RHS, Instruction::AShr > m_AShr(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1236
llvm::PatternMatch::m_c_And
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
Definition: PatternMatch.h:2727
llvm::PatternMatch::m_SpecificInt
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
Definition: PatternMatch.h:972
llvm::PatternMatch::match
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
llvm::PatternMatch::m_Specific
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:875
llvm::PatternMatch::m_SMin
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
Definition: PatternMatch.h:2302
llvm::PatternMatch::m_c_Xor
BinaryOp_match< LHS, RHS, Instruction::Xor, true > m_c_Xor(const LHS &L, const RHS &R)
Matches an Xor with LHS and RHS in either order.
Definition: PatternMatch.h:2741
llvm::PatternMatch::m_ZeroInt
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
Definition: PatternMatch.h:599
llvm::PatternMatch::m_ICmp
CmpClass_match< LHS, RHS, ICmpInst, ICmpInst::Predicate > m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R)
Definition: PatternMatch.h:1589
llvm::PatternMatch::m_OneUse
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
llvm::PatternMatch::m_c_ICmp
CmpClass_match< LHS, RHS, ICmpInst, ICmpInst::Predicate, true > m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R)
Matches an ICmp with a predicate over LHS and RHS in either order.
Definition: PatternMatch.h:2693
llvm::PatternMatch::m_c_Add
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
Definition: PatternMatch.h:2713
llvm::PatternMatch::m_SpecificICmp
SpecificCmpClass_match< LHS, RHS, ICmpInst, ICmpInst::Predicate > m_SpecificICmp(ICmpInst::Predicate MatchPred, const LHS &L, const RHS &R)
Definition: PatternMatch.h:1654
llvm::PatternMatch::m_SMax
MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty > m_SMax(const LHS &L, const RHS &R)
Definition: PatternMatch.h:2296
llvm::PatternMatch::m_Value
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
llvm::PatternMatch::m_Shl
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1224
llvm::PatternMatch::m_Not
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Definition: PatternMatch.h:2395
llvm::PatternMatch::m_c_Or
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
Definition: PatternMatch.h:2734
llvm::PatternMatch::m_Sub
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1104
llvm::PatternMatch::m_UMin
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
Definition: PatternMatch.h:2314
llvm::RTLIB::getSINTTOFP
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:296
llvm::RTLIB::getUINTTOFP
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:342
llvm::RTLIB::Libcall
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Definition: RuntimeLibcalls.h:33
llvm::RTLIB::getFPTOUINT
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:247
llvm::RTLIB::getFPTOSINT
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:198
llvm::RTLIB::getFPROUND
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:155
llvm::RegState::Implicit
@ Implicit
Not emitted register (e.g. carry, or temporary result).
Definition: MachineInstrBuilder.h:48
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:52
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:46
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:50
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:54
llvm::SyncScope::SingleThread
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition: LLVMContext.h:54
llvm::SyncScope::System
@ System
Synchronized with respect to all concurrently executing threads.
Definition: LLVMContext.h:57
llvm::TLSModel::GeneralDynamic
@ GeneralDynamic
Definition: CodeGen.h:46
llvm::WinEH::EncodingType::X86
@ X86
Windows x64, Windows Itanium (IA-64)
llvm::X86AS::PTR32_UPTR
@ PTR32_UPTR
Definition: X86.h:213
llvm::X86AS::FS
@ FS
Definition: X86.h:210
llvm::X86AS::PTR64
@ PTR64
Definition: X86.h:214
llvm::X86AS::PTR32_SPTR
@ PTR32_SPTR
Definition: X86.h:212
llvm::X86AS::GS
@ GS
Definition: X86.h:209
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:614
llvm::X86II::MO_TLSLD
@ MO_TLSLD
MO_TLSLD - On a symbol operand this indicates that the immediate is the offset of the GOT entry with ...
Definition: X86BaseInfo.h:411
llvm::X86II::MO_GOTPCREL_NORELAX
@ MO_GOTPCREL_NORELAX
MO_GOTPCREL_NORELAX - Same as MO_GOTPCREL except that R_X86_64_GOTPCREL relocations are guaranteed to...
Definition: X86BaseInfo.h:391
llvm::X86II::MO_COFFSTUB
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
Definition: X86BaseInfo.h:488
llvm::X86II::MO_NTPOFF
@ MO_NTPOFF
MO_NTPOFF - On a symbol operand this indicates that the immediate is the negative thread-pointer offs...
Definition: X86BaseInfo.h:450
llvm::X86II::MO_INDNTPOFF
@ MO_INDNTPOFF
MO_INDNTPOFF - On a symbol operand this indicates that the immediate is the absolute address of the G...
Definition: X86BaseInfo.h:432
llvm::X86II::MO_GOTNTPOFF
@ MO_GOTNTPOFF
MO_GOTNTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry w...
Definition: X86BaseInfo.h:456
llvm::X86II::MO_TPOFF
@ MO_TPOFF
MO_TPOFF - On a symbol operand this indicates that the immediate is the thread-pointer offset for the...
Definition: X86BaseInfo.h:438
llvm::X86II::MO_TLVP_PIC_BASE
@ MO_TLVP_PIC_BASE
MO_TLVP_PIC_BASE - On a symbol operand this indicates that the immediate is some TLS offset from the ...
Definition: X86BaseInfo.h:476
llvm::X86II::MO_TLSGD
@ MO_TLSGD
MO_TLSGD - On a symbol operand this indicates that the immediate is the offset of the GOT entry with ...
Definition: X86BaseInfo.h:403
llvm::X86II::MO_NO_FLAG
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
Definition: X86BaseInfo.h:363
llvm::X86II::MO_TLVP
@ MO_TLVP
MO_TLVP - On a symbol operand this indicates that the immediate is some TLS offset.
Definition: X86BaseInfo.h:472
llvm::X86II::MO_DLLIMPORT
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand "FOO", this indicates that the reference is actually to the "__imp...
Definition: X86BaseInfo.h:460
llvm::X86II::MO_GOTTPOFF
@ MO_GOTTPOFF
MO_GOTTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry wi...
Definition: X86BaseInfo.h:425
llvm::X86II::MO_SECREL
@ MO_SECREL
MO_SECREL - On a symbol operand this indicates that the immediate is the offset from beginning of sec...
Definition: X86BaseInfo.h:480
llvm::X86II::MO_DTPOFF
@ MO_DTPOFF
MO_DTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry with...
Definition: X86BaseInfo.h:444
llvm::X86II::MO_TLSLDM
@ MO_TLSLDM
MO_TLSLDM - On a symbol operand this indicates that the immediate is the offset of the GOT entry with...
Definition: X86BaseInfo.h:419
llvm::X86II::MO_GOTPCREL
@ MO_GOTPCREL
MO_GOTPCREL - On a symbol operand this indicates that the immediate is offset to the GOT entry for th...
Definition: X86BaseInfo.h:387
llvm::X86ISD::FST
@ FST
This instruction implements a truncating store from FP stack slots.
Definition: X86ISelLowering.h:915
llvm::X86ISD::CMPM
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
Definition: X86ISelLowering.h:395
llvm::X86ISD::FMAX
@ FMAX
Floating point max and min.
Definition: X86ISelLowering.h:264
llvm::X86ISD::BT
@ BT
X86 bit-test instructions.
Definition: X86ISelLowering.h:91
llvm::X86ISD::HADD
@ HADD
Integer horizontal add/sub.
Definition: X86ISelLowering.h:253
llvm::X86ISD::MOVQ2DQ
@ MOVQ2DQ
Copies a 64-bit value from an MMX vector to the low word of an XMM vector, with the high word zero fi...
Definition: X86ISelLowering.h:157
llvm::X86ISD::BLENDI
@ BLENDI
Blend where the selector is an immediate.
Definition: X86ISelLowering.h:203
llvm::X86ISD::CMP
@ CMP
X86 compare and logical compare instructions.
Definition: X86ISelLowering.h:85
llvm::X86ISD::BLENDV
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
Definition: X86ISelLowering.h:210
llvm::X86ISD::ADDSUB
@ ADDSUB
Combined add and sub on an FP vector.
Definition: X86ISelLowering.h:213
llvm::X86ISD::STRICT_FCMP
@ STRICT_FCMP
X86 strict FP compare instructions.
Definition: X86ISelLowering.h:790
llvm::X86ISD::STRICT_CMPM
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
Definition: X86ISelLowering.h:798
llvm::X86ISD::FHADD
@ FHADD
Floating point horizontal add/sub.
Definition: X86ISelLowering.h:257
llvm::X86ISD::BSR
@ BSR
Bit scan reverse.
Definition: X86ISelLowering.h:33
llvm::X86ISD::SETCC
@ SETCC
X86 SetCC.
Definition: X86ISelLowering.h:95
llvm::X86ISD::NT_BRIND
@ NT_BRIND
BRIND node with NoTrack prefix.
Definition: X86ISelLowering.h:127
llvm::X86ISD::SELECTS
@ SELECTS
X86 Select.
Definition: X86ISelLowering.h:98
llvm::X86ISD::FSETCCM
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
Definition: X86ISelLowering.h:111
llvm::X86ISD::PEXTRB
@ PEXTRB
Extract an 8-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRB.
Definition: X86ISelLowering.h:173
llvm::X86ISD::FXOR
@ FXOR
Bitwise logical XOR of floating point values.
Definition: X86ISelLowering.h:52
llvm::X86ISD::BRCOND
@ BRCOND
X86 conditional branches.
Definition: X86ISelLowering.h:123
llvm::X86ISD::FSETCC
@ FSETCC
X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.
Definition: X86ISelLowering.h:107
llvm::X86ISD::PINSRB
@ PINSRB
Insert the lower 8-bits of a 32-bit value to a vector, corresponds to X86::PINSRB.
Definition: X86ISelLowering.h:185
llvm::X86ISD::INSERTPS
@ INSERTPS
Insert any element of a 4 x float vector into any element of a destination 4 x floatvector.
Definition: X86ISelLowering.h:181
llvm::X86ISD::PSHUFB
@ PSHUFB
Shuffle 16 8-bit values within a vector.
Definition: X86ISelLowering.h:192
llvm::X86ISD::PEXTRW
@ PEXTRW
Extract a 16-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRW.
Definition: X86ISelLowering.h:177
llvm::X86ISD::AADD
@ AADD
RAO arithmetic instructions.
Definition: X86ISelLowering.h:856
llvm::X86ISD::FANDN
@ FANDN
Bitwise logical ANDNOT of floating point values.
Definition: X86ISelLowering.h:56
llvm::X86ISD::GlobalBaseReg
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
Definition: X86ISelLowering.h:144
llvm::X86ISD::FMAXC
@ FMAXC
Commutative FMIN and FMAX.
Definition: X86ISelLowering.h:268
llvm::X86ISD::EXTRQI
@ EXTRQI
SSE4A Extraction and Insertion.
Definition: X86ISelLowering.h:520
llvm::X86ISD::FLD
@ FLD
This instruction implements an extending load to FP stack slots.
Definition: X86ISelLowering.h:909
llvm::X86ISD::PSADBW
@ PSADBW
Compute Sum of Absolute Differences.
Definition: X86ISelLowering.h:195
llvm::X86ISD::FOR
@ FOR
Bitwise logical OR of floating point values.
Definition: X86ISelLowering.h:48
llvm::X86ISD::FIST
@ FIST
This instruction implements a fp->int store from FP stack slots.
Definition: X86ISelLowering.h:903
llvm::X86ISD::FP_TO_INT_IN_MEM
@ FP_TO_INT_IN_MEM
This instruction implements FP_TO_SINT with the integer destination in memory and a FP reg source.
Definition: X86ISelLowering.h:890
llvm::X86ISD::LADD
@ LADD
LOCK-prefixed arithmetic read-modify-write instructions.
Definition: X86ISelLowering.h:842
llvm::X86ISD::MMX_MOVW2D
@ MMX_MOVW2D
Copies a GPR into the low 32-bit word of a MMX vector and zero out the high word.
Definition: X86ISelLowering.h:169
llvm::X86ISD::Wrapper
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
Definition: X86ISelLowering.h:149
llvm::X86ISD::PINSRW
@ PINSRW
Insert the lower 16-bits of a 32-bit value to a vector, corresponds to X86::PINSRW.
Definition: X86ISelLowering.h:189
llvm::X86ISD::CMPCCXADD
@ CMPCCXADD
Compare and Add if Condition is Met.
Definition: X86ISelLowering.h:947
llvm::X86ISD::MMX_MOVD2W
@ MMX_MOVD2W
Copies a 32-bit value from the low word of a MMX vector to a GPR.
Definition: X86ISelLowering.h:165
llvm::X86ISD::FILD
@ FILD
This instruction implements SINT_TO_FP with the integer source in memory and FP reg result.
Definition: X86ISelLowering.h:897
llvm::X86ISD::MOVDQ2Q
@ MOVDQ2Q
Copies a 64-bit value from the low word of an XMM vector to an MMX vector.
Definition: X86ISelLowering.h:161
llvm::X86ISD::ANDNP
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
Definition: X86ISelLowering.h:200
llvm::X86ISD::BSF
@ BSF
Bit scan forward.
Definition: X86ISelLowering.h:31
llvm::X86ISD::VAARG_64
@ VAARG_64
These instructions grab the address of the next argument from a va_list.
Definition: X86ISelLowering.h:919
llvm::X86ISD::FAND
@ FAND
Bitwise logical AND of floating point values.
Definition: X86ISelLowering.h:44
llvm::X86ISD::CMOV
@ CMOV
X86 conditional moves.
Definition: X86ISelLowering.h:117
llvm::X86ISD::WrapperRIP
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
Definition: X86ISelLowering.h:153
llvm::X86ISD::FSHL
@ FSHL
X86 funnel/double shift i16 instructions.
Definition: X86ISelLowering.h:39
llvm::X86ISD::FRSQRT
@ FRSQRT
Floating point reciprocal-sqrt and reciprocal approximation.
Definition: X86ISelLowering.h:278
llvm::X86::TO_NEAREST_INT
@ TO_NEAREST_INT
Definition: X86BaseInfo.h:42
llvm::X86::CUR_DIRECTION
@ CUR_DIRECTION
Definition: X86BaseInfo.h:46
llvm::X86::mayFoldLoadIntoBroadcastFromMem
bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into a vector splat instruction as a memory oper...
Definition: X86ISelLowering.cpp:2663
llvm::X86::isZeroNode
bool isZeroNode(SDValue Elt)
Returns true if Elt is a constant zero or floating point constant +0.0.
Definition: X86ISelLowering.cpp:3781
llvm::X86::AddrNumOperands
@ AddrNumOperands
Definition: X86BaseInfo.h:36
llvm::X86::GetOppositeBranchCondition
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
Definition: X86InstrInfo.cpp:3299
llvm::X86::mayFoldIntoZeroExtend
bool mayFoldIntoZeroExtend(SDValue Op)
Check if Op is an operation that could be folded into a zero extend x86 instruction.
Definition: X86ISelLowering.cpp:2681
llvm::X86::mayFoldIntoStore
bool mayFoldIntoStore(SDValue Op)
Check if Op is a value that could be used to fold a store into some other x86 instruction as a memory...
Definition: X86ISelLowering.cpp:2677
llvm::X86::isExtendedSwiftAsyncFrameSupported
bool isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget, const MachineFunction &MF)
True if the target supports the extended frame for async Swift functions.
Definition: X86ISelLowering.cpp:26793
llvm::X86::getCCMPCondFlagsFromCondCode
int getCCMPCondFlagsFromCondCode(CondCode CC)
Definition: X86InstrInfo.cpp:3223
llvm::X86::mayFoldLoad
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
Definition: X86ISelLowering.cpp:2644
llvm::X86::createFastISel
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
Definition: X86FastISel.cpp:4068
llvm::X86::isOffsetSuitableForCodeModel
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
Definition: X86ISelLowering.cpp:2772
llvm::X86::isConstantSplat
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
Definition: X86ISelLowering.cpp:5065
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::dwarf::toString
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
Definition: DWARFFormValue.h:177
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:47
llvm::rdf::Func
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::DecodeZeroExtendMask
void DecodeZeroExtendMask(unsigned SrcScalarBits, unsigned DstScalarBits, unsigned NumDstElts, bool IsAnyExtend, SmallVectorImpl< int > &ShuffleMask)
Decode a zero extension instruction as a shuffle mask.
Definition: X86ShuffleDecode.cpp:370
llvm::next_nodbg
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
Definition: MachineBasicBlock.h:1432
llvm::isGlobalStubReference
static bool isGlobalStubReference(unsigned char TargetFlag)
isGlobalStubReference - Return true if the specified TargetFlag operand is a reference to a stub for ...
Definition: X86InstrInfo.h:121
llvm::drop_begin
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
llvm::ThreadPriority::Low
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
llvm::Offset
@ Offset
Definition: DWP.cpp:480
llvm::Length
@ Length
Definition: DWP.cpp:480
llvm::DecodeMOVHLPSMask
void DecodeMOVHLPSMask(unsigned NElts, SmallVectorImpl< int > &ShuffleMask)
Decode a MOVHLPS instruction as a v2f64/v4f32 shuffle mask.
Definition: X86ShuffleDecode.cpp:60
llvm::find
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1742
llvm::isGlobalRelativeToPICBase
static bool isGlobalRelativeToPICBase(unsigned char TargetFlag)
isGlobalRelativeToPICBase - Return true if the specified global value reference is relative to a 32-b...
Definition: X86InstrInfo.h:139
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1722
llvm::popcount
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1680
llvm::DecodeZeroMoveLowMask
void DecodeZeroMoveLowMask(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)
Decode a move lower and zero upper instruction as a shuffle mask.
Definition: X86ShuffleDecode.cpp:384
llvm::SM_SentinelUndef
@ SM_SentinelUndef
Definition: X86ShuffleDecode.h:28
llvm::SM_SentinelZero
@ SM_SentinelZero
Definition: X86ShuffleDecode.h:28
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:373
llvm::DecodeVPERMILPMask
void DecodeVPERMILPMask(unsigned NumElts, unsigned ScalarBits, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERMILPD/VPERMILPS variable mask from a raw array of constants.
Definition: X86ShuffleDecode.cpp:476
llvm::isNullConstant
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
Definition: SelectionDAG.cpp:11909
llvm::isAllOnesOrAllOnesSplat
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
Definition: Utils.cpp:1553
llvm::DecodePSHUFLWMask
void DecodePSHUFLWMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Decodes the shuffle masks for pshuflw.
Definition: X86ShuffleDecode.cpp:177
llvm::getIntrinsicWithChain
static const IntrinsicData * getIntrinsicWithChain(unsigned IntNo)
Definition: X86IntrinsicsInfo.h:347
llvm::peekThroughBitcasts
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
Definition: SelectionDAG.cpp:12001
llvm::enumerate
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition: STLExtras.h:2431
llvm::Log2_64_Ceil
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
Definition: MathExtras.h:359
llvm::getX86SubSuperRegister
MCRegister getX86SubSuperRegister(MCRegister Reg, unsigned Size, bool High=false)
Definition: X86MCTargetDesc.cpp:761
llvm::AddressSpace
AddressSpace
Definition: NVPTXBaseInfo.h:21
llvm::ExceptionHandling::SjLj
@ SjLj
setjmp/longjmp based exceptions
llvm::isIntOrFPConstant
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
Definition: SelectionDAGNodes.h:1852
llvm::setDirectAddressInInstr
static void setDirectAddressInInstr(MachineInstr *MI, unsigned Operand, unsigned Reg)
Replace the address used in the instruction with the direct memory reference.
Definition: X86InstrBuilder.h:132
llvm::DecodeVPERMV3Mask
void DecodeVPERMV3Mask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERMT2 W/D/Q/PS/PD mask from a raw array of constants.
Definition: X86ShuffleDecode.cpp:558
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:76
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:656
llvm::DecodeBLENDMask
void DecodeBLENDMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Decode a BLEND immediate mask into a shuffle mask.
Definition: X86ShuffleDecode.cpp:314
llvm::decodeVSHUF64x2FamilyMask
void decodeVSHUF64x2FamilyMask(unsigned NumElts, unsigned ScalarSize, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Decode a shuffle packed values at 128-bit granularity (SHUFF32x4/SHUFF64x2/SHUFI32x4/SHUFI64x2) immed...
Definition: X86ShuffleDecode.cpp:263
llvm::DecodeVPERMMask
void DecodeVPERMMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Decodes the shuffle masks for VPERMQ/VPERMPD.
Definition: X86ShuffleDecode.cpp:363
llvm::addFrameReference
static const MachineInstrBuilder & addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset=0, bool mem=true)
addFrameReference - This function is used to add a reference to the base of an abstract object on the...
Definition: PPCInstrBuilder.h:32
llvm::DecodeEXTRQIMask
void DecodeEXTRQIMask(unsigned NumElts, unsigned EltSize, int Len, int Idx, SmallVectorImpl< int > &ShuffleMask)
Decode a SSE4A EXTRQ instruction as a shuffle mask.
Definition: X86ShuffleDecode.cpp:399
llvm::addFullAddress
static const MachineInstrBuilder & addFullAddress(const MachineInstrBuilder &MIB, const X86AddressMode &AM)
Definition: X86InstrBuilder.h:172
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:296
llvm::getIntrinsicWithoutChain
static const IntrinsicData * getIntrinsicWithoutChain(unsigned IntNo)
Definition: X86IntrinsicsInfo.h:2147
llvm::unique
auto unique(Range &&R, Predicate P)
Definition: STLExtras.h:2038
llvm::isNullOrNullSplat
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition: Utils.cpp:1535
llvm::DecodePSRLDQMask
void DecodePSRLDQMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Definition: X86ShuffleDecode.cpp:111
llvm::bit_ceil
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Definition: bit.h:342
llvm::copy_if
OutputIt copy_if(R &&Range, OutputIt Out, UnaryPredicate P)
Provide wrappers to std::copy_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1768
llvm::Log2_64
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:346
llvm::isMinSignedConstant
bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
Definition: SelectionDAG.cpp:11933
llvm::isConstOrConstSplatFP
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
Definition: SelectionDAG.cpp:12081
llvm::DecodeVPERM2X128Mask
void DecodeVPERM2X128Mask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Definition: X86ShuffleDecode.cpp:280
llvm::countr_zero
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
llvm::M1
unsigned M1(unsigned Val)
Definition: VE.h:376
llvm::DecodeVPERMIL2PMask
void DecodeVPERMIL2PMask(unsigned NumElts, unsigned ScalarBits, unsigned M2Z, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERMIL2PD/VPERMIL2PS variable mask from a raw array of constants.
Definition: X86ShuffleDecode.cpp:498
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
llvm::DecodeMOVLHPSMask
void DecodeMOVLHPSMask(unsigned NElts, SmallVectorImpl< int > &ShuffleMask)
Decode a MOVLHPS instruction as a v2f64/v4f32 shuffle mask.
Definition: X86ShuffleDecode.cpp:69
llvm::getShuffleDemandedElts
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
Definition: VectorUtils.cpp:312
llvm::Log2_32
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:340
llvm::countl_zero
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition: bit.h:281
llvm::isBitwiseNot
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
Definition: SelectionDAG.cpp:12025
llvm::reverse
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:419
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:291
llvm::getHorizDemandedEltsForFirstOperand
void getHorizDemandedEltsForFirstOperand(unsigned VectorBitWidth, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
Compute the demanded elements mask of horizontal binary operations.
Definition: VectorUtils.cpp:570
llvm::get
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
Definition: PointerIntPair.h:270
llvm::addRegOffset
static const MachineInstrBuilder & addRegOffset(const MachineInstrBuilder &MIB, unsigned Reg, bool isKill, int Offset)
addRegOffset - This function is used to add a memory reference of the form [Reg + Offset],...
Definition: X86InstrBuilder.h:157
llvm::createUnpackShuffleMask
void createUnpackShuffleMask(EVT VT, SmallVectorImpl< int > &Mask, bool Lo, bool Unary)
Generate unpacklo/unpackhi shuffle mask.
Definition: X86ISelLowering.cpp:4548
llvm::Hi_32
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition: MathExtras.h:154
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1736
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
llvm::DecodeINSERTQIMask
void DecodeINSERTQIMask(unsigned NumElts, unsigned EltSize, int Len, int Idx, SmallVectorImpl< int > &ShuffleMask)
Decode a SSE4A INSERTQ instruction as a shuffle mask.
Definition: X86ShuffleDecode.cpp:436
llvm::peekThroughOneUseBitcasts
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
Definition: SelectionDAG.cpp:12007
llvm::classifyEHPersonality
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Definition: EHPersonalities.cpp:23
llvm::Lo_32
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition: MathExtras.h:159
llvm::DecodeVPERMVMask
void DecodeVPERMVMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERM W/D/Q/PS/PD mask from a raw array of constants.
Definition: X86ShuffleDecode.cpp:544
llvm::verifyIntrinsicTables
static void verifyIntrinsicTables()
Definition: X86IntrinsicsInfo.h:2154
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition: AtomicOrdering.h:56
llvm::ModRefInfo::Mod
@ Mod
The access may modify the value stored in memory.
llvm::createSplat2ShuffleMask
void createSplat2ShuffleMask(MVT VT, SmallVectorImpl< int > &Mask, bool Lo)
Similar to unpacklo/unpackhi, but without the 128-bit lane limitation imposed by AVX and specific to ...
Definition: X86ISelLowering.cpp:4568
llvm::IRMemLocation::First
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
llvm::isFuncletEHPersonality
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
Definition: EHPersonalities.h:65
llvm::DecodeVALIGNMask
void DecodeVALIGNMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Definition: X86ShuffleDecode.cpp:138
llvm::CombineLevel
CombineLevel
Definition: DAGCombine.h:15
llvm::lower_bound
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1961
llvm::narrowShuffleMaskElts
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
Definition: VectorUtils.cpp:349
llvm::RecurKind::UMin
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
llvm::RecurKind::Or
@ Or
Bitwise or logical OR of integers.
llvm::RecurKind::Mul
@ Mul
Product of integers.
llvm::RecurKind::Xor
@ Xor
Bitwise or logical XOR of integers.
llvm::RecurKind::SMax
@ SMax
Signed integer max implemented in terms of select(cmp()).
llvm::RecurKind::And
@ And
Bitwise or logical AND of integers.
llvm::RecurKind::SMin
@ SMin
Signed integer min implemented in terms of select(cmp()).
llvm::RecurKind::Add
@ Add
Sum of integers.
llvm::DecodeScalarMoveMask
void DecodeScalarMoveMask(unsigned NumElts, bool IsLoad, SmallVectorImpl< int > &ShuffleMask)
Decode a scalar float move instruction as a shuffle mask.
Definition: X86ShuffleDecode.cpp:390
llvm::isNullConstantOrUndef
bool isNullConstantOrUndef(SDValue V)
Returns true if V is a constant integer zero or an UNDEF node.
Definition: SelectionDAG.cpp:11914
llvm::count
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1921
llvm::getAddressFromInstr
static X86AddressMode getAddressFromInstr(const MachineInstr *MI, unsigned Operand)
Compute the addressing mode from an machine instruction starting with the given operand.
Definition: X86InstrBuilder.h:92
llvm::DecodeVPPERMMask
void DecodeVPPERMMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPPERM mask from a raw array of constants such as from BUILD_VECTOR.
Definition: X86ShuffleDecode.cpp:324
llvm::Op
DWARFExpression::Operation Op
Definition: DWARFExpression.cpp:22
llvm::DecodePALIGNRMask
void DecodePALIGNRMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Definition: X86ShuffleDecode.cpp:124
llvm::DecodeMOVSLDUPMask
void DecodeMOVSLDUPMask(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)
Definition: X86ShuffleDecode.cpp:77
llvm::RoundingMode
RoundingMode
Rounding mode.
Definition: FloatingPointMode.h:37
llvm::RoundingMode::TowardZero
@ TowardZero
roundTowardZero.
llvm::RoundingMode::NearestTiesToEven
@ NearestTiesToEven
roundTiesToEven.
llvm::RoundingMode::TowardPositive
@ TowardPositive
roundTowardPositive.
llvm::RoundingMode::TowardNegative
@ TowardNegative
roundTowardNegative.
llvm::M0
unsigned M0(unsigned Val)
Definition: VE.h:375
llvm::isAsynchronousEHPersonality
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
Definition: EHPersonalities.h:50
llvm::isConstOrConstSplat
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
Definition: SelectionDAG.cpp:12035
llvm::DecodeINSERTPSMask
void DecodeINSERTPSMask(unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Decode a 128-bit INSERTPS instruction as a v4f32 shuffle mask.
Definition: X86ShuffleDecode.cpp:26
llvm::BitWidth
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
llvm::DecodeUNPCKLMask
void DecodeUNPCKLMask(unsigned NumElts, unsigned ScalarBits, SmallVectorImpl< int > &ShuffleMask)
Decodes the shuffle masks for unpcklps/unpcklpd and punpckl*.
Definition: X86ShuffleDecode.cpp:233
llvm::DecodePSLLDQMask
void DecodePSLLDQMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Definition: X86ShuffleDecode.cpp:99
llvm::count_if
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:1928
llvm::DecodeUNPCKHMask
void DecodeUNPCKHMask(unsigned NumElts, unsigned ScalarBits, SmallVectorImpl< int > &ShuffleMask)
Decodes the shuffle masks for unpckhps/unpckhpd and punpckh*.
Definition: X86ShuffleDecode.cpp:217
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1749
llvm::isOneConstant
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Definition: SelectionDAG.cpp:11928
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1886
llvm::commonAlignment
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
llvm::isNullFPConstant
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
Definition: SelectionDAG.cpp:11918
llvm::all_equal
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2070
llvm::TRUNCATE_TO_MEM_VI16
@ TRUNCATE_TO_MEM_VI16
Definition: X86IntrinsicsInfo.h:70
llvm::INTR_TYPE_SCALAR_MASK_SAE
@ INTR_TYPE_SCALAR_MASK_SAE
Definition: X86IntrinsicsInfo.h:60
llvm::INTR_TYPE_1OP_SAE
@ INTR_TYPE_1OP_SAE
Definition: X86IntrinsicsInfo.h:50
llvm::TRUNCATE_TO_MEM_VI32
@ TRUNCATE_TO_MEM_VI32
Definition: X86IntrinsicsInfo.h:71
llvm::INTR_TYPE_2OP_SAE
@ INTR_TYPE_2OP_SAE
Definition: X86IntrinsicsInfo.h:51
llvm::TRUNCATE_TO_REG
@ TRUNCATE_TO_REG
Definition: X86IntrinsicsInfo.h:64
llvm::INTR_TYPE_3OP_SCALAR_MASK_SAE
@ INTR_TYPE_3OP_SCALAR_MASK_SAE
Definition: X86IntrinsicsInfo.h:62
llvm::INTR_TYPE_3OP_MASK_SAE
@ INTR_TYPE_3OP_MASK_SAE
Definition: X86IntrinsicsInfo.h:54
llvm::INTR_TYPE_2OP_MASK
@ INTR_TYPE_2OP_MASK
Definition: X86IntrinsicsInfo.h:56
llvm::TRUNCATE_TO_MEM_VI8
@ TRUNCATE_TO_MEM_VI8
Definition: X86IntrinsicsInfo.h:69
llvm::TRUNCATE2_TO_REG
@ TRUNCATE2_TO_REG
Definition: X86IntrinsicsInfo.h:65
llvm::CVTNEPS2BF16_MASK
@ CVTNEPS2BF16_MASK
Definition: X86IntrinsicsInfo.h:23
llvm::CMP_MASK_SCALAR_CC
@ CMP_MASK_SCALAR_CC
Definition: X86IntrinsicsInfo.h:43
llvm::INTR_TYPE_1OP_MASK_SAE
@ INTR_TYPE_1OP_MASK_SAE
Definition: X86IntrinsicsInfo.h:52
llvm::FIXUPIMM_MASKZ
@ FIXUPIMM_MASKZ
Definition: X86IntrinsicsInfo.h:73
llvm::INTR_TYPE_SCALAR_MASK
@ INTR_TYPE_SCALAR_MASK
Definition: X86IntrinsicsInfo.h:59
llvm::INTR_TYPE_3OP_IMM8
@ INTR_TYPE_3OP_IMM8
Definition: X86IntrinsicsInfo.h:39
llvm::INTR_TYPE_2OP_MASK_SAE
@ INTR_TYPE_2OP_MASK_SAE
Definition: X86IntrinsicsInfo.h:53
llvm::INTR_TYPE_SCALAR_MASK_RND
@ INTR_TYPE_SCALAR_MASK_RND
Definition: X86IntrinsicsInfo.h:61
llvm::INTR_TYPE_1OP_MASK
@ INTR_TYPE_1OP_MASK
Definition: X86IntrinsicsInfo.h:55
llvm::COMPRESS_EXPAND_IN_REG
@ COMPRESS_EXPAND_IN_REG
Definition: X86IntrinsicsInfo.h:63
llvm::INTR_TYPE_CAST_MMX
@ INTR_TYPE_CAST_MMX
Definition: X86IntrinsicsInfo.h:78
llvm::INTR_TYPE_4OP_IMM8
@ INTR_TYPE_4OP_IMM8
Definition: X86IntrinsicsInfo.h:38
llvm::DecodePSHUFMask
void DecodePSHUFMask(unsigned NumElts, unsigned ScalarBits, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Decodes the shuffle masks for pshufd/pshufw/vpermilpd/vpermilps.
Definition: X86ShuffleDecode.cpp:147
llvm::DecodeMOVDDUPMask
void DecodeMOVDDUPMask(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)
Definition: X86ShuffleDecode.cpp:91
llvm::array_pod_sort
void array_pod_sort(IteratorTy Start, IteratorTy End)
array_pod_sort - This sorts an array with the specified start and end extent.
Definition: STLExtras.h:1607
llvm::DecodeVectorBroadcast
void DecodeVectorBroadcast(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)
Decodes a broadcast of the first element of a vector.
Definition: X86ShuffleDecode.cpp:249
llvm::DecodeSHUFPMask
void DecodeSHUFPMask(unsigned NumElts, unsigned ScalarBits, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Decodes the shuffle masks for shufp*.
Definition: X86ShuffleDecode.cpp:200
llvm::DecodePSHUFHWMask
void DecodePSHUFHWMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)
Decodes the shuffle masks for pshufhw.
Definition: X86ShuffleDecode.cpp:163
llvm::DecodeMOVSHDUPMask
void DecodeMOVSHDUPMask(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)
Definition: X86ShuffleDecode.cpp:84
llvm::maxUIntN
uint64_t maxUIntN(uint64_t N)
Gets the maximum value for a N-bit unsigned integer.
Definition: MathExtras.h:219
llvm::isAllOnesConstant
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
Definition: SelectionDAG.cpp:11923
llvm::DecodePSHUFBMask
void DecodePSHUFBMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a PSHUFB mask from a raw array of constants such as from BUILD_VECTOR.
Definition: X86ShuffleDecode.cpp:292
llvm::getSplatIndex
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
Definition: VectorUtils.cpp:229
llvm::addDirectMem
static const MachineInstrBuilder & addDirectMem(const MachineInstrBuilder &MIB, unsigned Reg)
addDirectMem - This function is used to add a direct memory reference to the current instruction – th...
Definition: X86InstrBuilder.h:124
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
N
#define N
EQ
#define EQ(a, b)
Definition: regexec.c:112
MaskOps
This is used by foldAnyOrAllBitsSet() to capture a source value (Root) and the bit indexes (Mask) nee...
Definition: AggressiveInstCombine.cpp:184
llvm::APFloatBase::IEEEsingle
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:281
llvm::APFloatBase::rmNearestTiesToEven
static constexpr roundingMode rmNearestTiesToEven
Definition: APFloat.h:254
llvm::APFloatBase::rmTowardZero
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:258
llvm::APFloatBase::x87DoubleExtended
static const fltSemantics & x87DoubleExtended() LLVM_READNONE
Definition: APFloat.cpp:300
llvm::APFloatBase::IEEEquad
static const fltSemantics & IEEEquad() LLVM_READNONE
Definition: APFloat.cpp:283
llvm::APFloatBase::semanticsPrecision
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:329
llvm::APFloatBase::IEEEdouble
static const fltSemantics & IEEEdouble() LLVM_READNONE
Definition: APFloat.cpp:282
llvm::APFloatBase::IEEEhalf
static const fltSemantics & IEEEhalf() LLVM_READNONE
Definition: APFloat.cpp:279
llvm::APFloatBase::BFloat
static const fltSemantics & BFloat() LLVM_READNONE
Definition: APFloat.cpp:280
llvm::APFloatBase::opStatus
opStatus
IEEE-754R 7: Default exception handling.
Definition: APFloat.h:270
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::Align::value
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:35
llvm::EVT::changeVectorElementTypeToInteger
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:94
llvm::EVT::getStoreSize
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:381
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:137
llvm::EVT::getVectorVT
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:74
llvm::EVT::changeTypeToInteger
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition: ValueTypes.h:121
llvm::EVT::bitsLT
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:291
llvm::EVT::isFloatingPoint
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:147
llvm::EVT::getVectorElementCount
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:341
llvm::EVT::getDoubleNumVectorElementsVT
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:449
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:359
llvm::EVT::isByteSized
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:234
llvm::EVT::getVectorMinNumElements
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:350
llvm::EVT::getScalarSizeInBits
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:371
llvm::EVT::getEVT
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:275
llvm::EVT::getStoreSizeInBits
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition: ValueTypes.h:398
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:307
llvm::EVT::is128BitVector
bool is128BitVector() const
Return true if this is a 128-bit vector type.
Definition: ValueTypes.h:204
llvm::EVT::getIntegerVT
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:65
llvm::EVT::getFixedSizeInBits
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:367
llvm::EVT::is512BitVector
bool is512BitVector() const
Return true if this is a 512-bit vector type.
Definition: ValueTypes.h:214
llvm::EVT::getFloatingPointVT
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Definition: ValueTypes.h:59
llvm::EVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:168
llvm::EVT::getScalarType
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:314
llvm::EVT::is256BitVector
bool is256BitVector() const
Return true if this is a 256-bit vector type.
Definition: ValueTypes.h:209
llvm::EVT::getTypeForEVT
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:204
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:319
llvm::EVT::isScalarInteger
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:157
llvm::EVT::changeVectorElementType
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:102
llvm::EVT::getFltSemantics
const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
Definition: ValueTypes.cpp:306
llvm::EVT::getVectorNumElements
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:327
llvm::EVT::getHalfNumVectorElementsVT
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:439
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:152
llvm::EVT::is64BitVector
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Definition: ValueTypes.h:199
llvm::KnownBits::makeConstant
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition: KnownBits.h:290
llvm::KnownBits::sadd_sat
static KnownBits sadd_sat(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from llvm.sadd.sat(LHS, RHS)
Definition: KnownBits.cpp:753
llvm::KnownBits::eq
static std::optional< bool > eq(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_EQ result.
Definition: KnownBits.cpp:488
llvm::KnownBits::anyextOrTrunc
KnownBits anyextOrTrunc(unsigned BitWidth) const
Return known bits for an "any" extension or truncation of the value we're tracking.
Definition: KnownBits.h:175
llvm::KnownBits::isNonNegative
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition: KnownBits.h:97
llvm::KnownBits::isZero
bool isZero() const
Returns true if value is all zero.
Definition: KnownBits.h:76
llvm::KnownBits::countMinTrailingZeros
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:231
llvm::KnownBits::isUnknown
bool isUnknown() const
Returns true if we don't know any bits.
Definition: KnownBits.h:62
llvm::KnownBits::trunc
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition: KnownBits.h:150
llvm::KnownBits::countMaxPopulation
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition: KnownBits.h:278
llvm::KnownBits::setAllZero
void setAllZero()
Make all bits known to be zero and discard any previous information.
Definition: KnownBits.h:82
llvm::KnownBits::getBitWidth
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:40
llvm::KnownBits::zext
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
Definition: KnownBits.h:161
llvm::KnownBits::isConstant
bool isConstant() const
Returns true if we know the value of all bits.
Definition: KnownBits.h:50
llvm::KnownBits::resetAll
void resetAll()
Resets the known state of all bits.
Definition: KnownBits.h:70
llvm::KnownBits::abdu
static KnownBits abdu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for abdu(LHS, RHS).
Definition: KnownBits.cpp:228
llvm::KnownBits::extractBits
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
Definition: KnownBits.h:214
llvm::KnownBits::countMaxActiveBits
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
Definition: KnownBits.h:285
llvm::KnownBits::intersectWith
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
Definition: KnownBits.h:300
llvm::KnownBits::sext
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
Definition: KnownBits.h:169
llvm::KnownBits::add
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false)
Compute knownbits resulting from addition of LHS and RHS.
Definition: KnownBits.h:333
llvm::KnownBits::zextOrTrunc
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
Definition: KnownBits.h:185
llvm::KnownBits::countMinLeadingZeros
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:237
llvm::KnownBits::getMaxValue
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition: KnownBits.h:134
llvm::KnownBits::computeForAddSub
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
Definition: KnownBits.cpp:51
llvm::KnownBits::isNegative
bool isNegative() const
Returns true if this value is known to be negative.
Definition: KnownBits.h:94
llvm::KnownBits::setAllOnes
void setAllOnes()
Make all bits known to be one and discard any previous information.
Definition: KnownBits.h:88
llvm::KnownBits::mul
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
Definition: KnownBits.cpp:797
llvm::KnownBits::sgt
static std::optional< bool > sgt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGT result.
Definition: KnownBits.cpp:526
llvm::KnownBits::isAllOnes
bool isAllOnes() const
Returns true if value is all one bits.
Definition: KnownBits.h:79
llvm::KnownBits::getConstant
const APInt & getConstant() const
Returns the value when all bits have a known value.
Definition: KnownBits.h:56
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:41
llvm::MachinePointerInfo::isDereferenceable
bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
Definition: MachineOperand.cpp:1041
llvm::MachinePointerInfo::getConstantPool
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
Definition: MachineOperand.cpp:1056
llvm::MachinePointerInfo::getWithOffset
MachinePointerInfo getWithOffset(int64_t O) const
Definition: MachineMemOperand.h:81
llvm::MachinePointerInfo::getGOT
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
Definition: MachineOperand.cpp:1071
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1062
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
llvm::SDNodeFlags
These are IR-level optimization flags that may be propagated to SDNodes.
Definition: SelectionDAGNodes.h:379
llvm::SDNodeFlags::hasNoSignedZeros
bool hasNoSignedZeros() const
Definition: SelectionDAGNodes.h:447
llvm::SDVTList
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Definition: SelectionDAGNodes.h:79
llvm::SDVTList::NumVTs
unsigned int NumVTs
Definition: SelectionDAGNodes.h:81
llvm::TargetLoweringBase::AddrMode
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
Definition: TargetLowering.h:2791
llvm::TargetLowering::AsmOperandInfo
This contains information for each constraint that we are lowering.
Definition: TargetLowering.h:4968
llvm::TargetLowering::AsmOperandInfo::ConstraintVT
MVT ConstraintVT
The ValueType for the operand value.
Definition: TargetLowering.h:4984
llvm::TargetLowering::AsmOperandInfo::ConstraintCode
std::string ConstraintCode
This contains the actual string for the code, like "m".
Definition: TargetLowering.h:4972
llvm::TargetLowering::CallLoweringInfo
This structure contains all information that is necessary for lowering calls.
Definition: TargetLowering.h:4520
llvm::TargetLowering::CallLoweringInfo::setLibCallee
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
Definition: TargetLowering.h:4574
llvm::TargetLowering::CallLoweringInfo::setDebugLoc
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
Definition: TargetLowering.h:4563
llvm::TargetLowering::CallLoweringInfo::setChain
CallLoweringInfo & setChain(SDValue InChain)
Definition: TargetLowering.h:4568
llvm::TargetLowering::DAGCombinerInfo::CombineTo
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
Definition: DAGCombiner.cpp:909
llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
Definition: DAGCombiner.cpp:929
llvm::TargetLowering::TargetLoweringOpt
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
Definition: TargetLowering.h:3943
llvm::TargetLowering::TargetLoweringOpt::CombineTo
bool CombineTo(SDValue O, SDValue N)
Definition: TargetLowering.h:3957
llvm::X86AddressMode
X86AddressMode - This struct holds a generalized full x86 address mode.
Definition: X86InstrBuilder.h:42
Generated on Wed Nov 20 2024 09:20:45 for LLVM by doxygen 1.9.6